Add: Incremental Books

For new exchanges, TAIFEX and CBOE

.devcontainer/Dockerfile

@@ -87,6 +87,9 @@ RUN dpkg -i /emumba/emumba-deribit-connector-3.3.1-Linux.LICENSE_DISABLED.deb \
 # Pcap
 RUN apt-get update && apt-get install -y libpcap0.8-dev
 
+# Databento (deps)
+RUN apt-get update && apt-get -y install libzstd-dev
+
 # Set up default executables (update-alternatives)
 RUN \
    update-alternatives --install /usr/bin/clang clang /usr/bin/clang-$clang_major 100; \

.devcontainer/devcontainer.json

@@ -6,7 +6,7 @@
     "memory": "4gb",
     "storage": "6gb"
   },
-  // "postCreateCommand": "/home/bts/spark/docker/sparkd/dev/devcontainer_post_create.sh",
+  "postCreateCommand": "/home/bts/spark/docker/sparkd/dev/devcontainer_post_create.sh",
   "containerUser": "bts",
   "workspaceMount": "source=${localWorkspaceFolder},target=/home/bts/spark,type=bind",
   "workspaceFolder": "/home/bts/spark",
@@ -25,7 +25,9 @@
     "-p",
     "8002:8002",
     "-p",
-    "8003:8003"
+    "8003:8003",
+    "-p",
+    "12345:12345"
   ],
   "mounts": [
     "source=${localEnv:HOME}/.bts/devcontainer-mounts/${devcontainerId},target=/home/bts,type=bind"
@@ -45,4 +47,4 @@
       ]
     }
   }
-}
+}

.gitattributes

@@ -1,2 +1,3 @@
-*.cmd eol=crlf
-*.sh eol=lf
+*.cmd text=auto eol=crlf
+*.cs text=auto eol=crlf
+*.sh text=auto eol=lf

.vscode/launch.json

@@ -2,18 +2,10 @@
   "version": "0.2.0",
   "configurations": [
     {
-      "name": "(gdb) Launch",
-      "type": "cppdbg",
-      "request": "launch",
-      "program": "${workspaceFolder}/build/sparkd-debug-gcc/server/sparkd",
-      "args": ["run"],
-      "stopAtEntry": false,
-      "cwd": "${workspaceFolder}/build/sparkd-debug-gcc/server",
-      "environment": [],
-      "externalConsole": false,
-      "MIMode": "gdb",
+      "name": ".NET Core Attach",
+      "type": "coreclr",
+      "request": "attach"
     },
-    
     {
       "name": "DataStreamPublisher",
       "type": "coreclr",
@@ -34,11 +26,23 @@
       "stopAtEntry": false,
       "console": "integratedTerminal"
     },
+    {
+      "name": "(gdb) Launch",
+      "type": "cppdbg",
+      "request": "launch",
+      "program": "/home/bts/spark/build/sparkd-debug-clang/server/sparkd",
+      "args": ["run"],
+      "stopAtEntry": false,
+      "cwd": "/home/bts/spark/build/sparkd-debug-clang/server",
+      "environment": [],
+      "externalConsole": false,
+      "MIMode": "gdb",
+    },
     {
       "name": "DataStreamQueryAnalyzer",
       "type": "coreclr",
       "request": "launch",
-      "preLaunchTask": "Build DataStream (Debug)",
+      //"preLaunchTask": "Build DataStreamQueryAnalyzer (Debug)",
       "program": "",
       "windows": {
         "program": "DataStreamQueryAnalyzer.exe",
@@ -151,4 +155,4 @@
       "default": "http://localhost:8000/config",
     },
   ]
-}
+}

.vscode/tasks.json

@@ -189,25 +189,6 @@
         "color": "terminal.ansiCyan"
       },
     },
-    {
-      "label": "Build Sparkd -j8",
-      "group": "build",
-      "type": "shell",
-      "command": "",
-      "linux": {
-        "command": "tools/build_sparkd.sh",
-        "args": [
-          "--preset",
-          "${input:sparkdCmakePreset}",
-          "-j",
-          "8"
-        ],
-      },
-      "icon": {
-        "id": "terminal",
-        "color": "terminal.ansiCyan"
-      },
-    },
     {
       "label": "Build Sparkd (Debug)",
       "group": "build",
@@ -545,6 +526,28 @@
         "color": "terminal.ansiGreen",
       },
     },
+    {
+      "label": "Run PubSubServer (Debug | Skip Build)",
+      "type": "process",
+      "command": "",
+      "linux": {
+        "command": "PubSubServer",
+      },
+      "windows": {
+        "command": "PubSubServer.exe",
+      },
+      "args": [
+        "--config",
+        "${workspaceFolder}/src/DataStreamSuite/PubSubServer/Configurations/${input:pubSubServerConfig}.json",
+      ],
+      "options": {
+        "cwd": "${workspaceFolder}/build/Debug/bin/net7.0"
+      },
+      "icon": {
+        "id": "run",
+        "color": "terminal.ansiGreen",
+      },
+    },
     {
       "label": "Run PubSubServer (Release)",
       "dependsOn": "Build DataStream (Release)",
@@ -568,6 +571,28 @@
         "color": "terminal.ansiGreen",
       },
     },
+    {
+      "label": "Run PubSubServer (Release | Skip Build)",
+      "type": "process",
+      "command": "",
+      "linux": {
+        "command": "PubSubServer",
+      },
+      "windows": {
+        "command": "PubSubServer.exe",
+      },
+      "args": [
+        "--config",
+        "${workspaceFolder}/src/DataStreamSuite/PubSubServer/Configurations/${input:pubSubServerConfig}.json",
+      ],
+      "options": {
+        "cwd": "${workspaceFolder}/build/Release/bin/net7.0"
+      },
+      "icon": {
+        "id": "run",
+        "color": "terminal.ansiGreen",
+      },
+    },
     {
       "label": "Run Spark",
       "dependsOn": "Build Spark",
@@ -703,6 +728,36 @@
         "id": "wrench"
       },
     },
+    {
+      "label": "View PubSubServer Logs (Last Run)",
+      "type": "process",
+      "command": "",
+      "linux": {
+        "command": "tools/logging/show_last_run.sh",
+        "args": [
+          "${env:HOME}/.local/share/DataStream/data/logging.PubSubServer",
+          "Starting PubSubServer"
+        ]
+      },
+      "windows": {
+        "command": "tools/logging/show_last_run.cmd",
+        "args": [
+          "DataStream",
+          "PubSubServer.exe",
+          "Starting PubSubServer"
+        ]
+      },
+      "presentation": {
+        "reveal": "always",
+        "focus": true,
+        "panel": "shared",
+        "showReuseMessage": false,
+        "close": true,
+      },
+      "icon": {
+        "id": "wrench"
+      },
+    },
   ],
   "inputs": [
     {

CMakeLists.txt

@@ -132,6 +132,8 @@ else()
       COMMAND ln ARGS -sf ${CMAKE_BINARY_DIR}/src/server/sparkd/sparkd ${CMAKE_BINARY_DIR}/server
       COMMAND ln ARGS -sf ${CMAKE_BINARY_DIR}/data_files ${CMAKE_BINARY_DIR}/server
       COMMAND ln ARGS -sf ${CMAKE_BINARY_DIR}/src/lib/libspark.so ${CMAKE_BINARY_DIR}/server
+      COMMAND ln ARGS -sf ${EXTERNALS_ROOT}/lib/${CMAKE_BUILD_TYPE}/libonixs-fix-engine.so.4 ${CMAKE_BINARY_DIR}/server
+      
       COMMAND
         ln ARGS -sf
         ${EXTERNALS_ROOT}/lib/Release/libOnixS.IceImpactMulticastPriceFeedHandlerCpp.so.3.16.0.0

src/DataStreamSuite/DataStream/Misc/TCPConnectionTestStream.cs

@@ -39,8 +39,8 @@ namespace BCG {
 
         protected override void myOnFirstSubscriber() {
             MasterWorker.getInstance(false).addWorker(this, _send_period);
-            var server_address = new BlutoSubAddress(_ip, _port);
-            bts.AsyncTCPConnector_APM.connect_to(server_address, this);
+            var server_rci = new RemoteConnectionInfo(_ip, _port);
+            bts.AsyncTCPConnector_APM.connect_to(server_rci, this);
         }
 
         protected override void myOnLastSubscriber() {

src/DataStreamSuite/DataStream/Primitives/Distinct/DistinctRowStream.cs

@@ -1,23 +1,24 @@
+using bts;
+
 using System;
 using System.Collections;
 
-namespace BCG
-{
+namespace BCG {
 	/// <summary>
 	/// Provides a stream that with distinct rows with respect to the columns
 	/// the user specifies in the argument
 	/// </summary>
-	[DataStreamUserRespresentation ("distinct")]
-	public class DistinctRowStream : ViewStream
-	{
-		public DistinctRowStream (ViewStream stream, params string[] cols)
-		{
-         ArrayList groupByCols = new ArrayList ();
-         foreach (string col in cols)
-            foreach (string c in col.Split (' '))
-               groupByCols.Add (c);
-         mySetRealSource (new GroupByStream (stream, 
-            new DistinctRowEvaluator (groupByCols.ToArray (typeof(string)) as string[]), false));
+	[DataStreamUserRespresentation("distinct")]
+	public class DistinctRowStream : ViewStream {
+		public DistinctRowStream(ViewStream stream, params string[] cols) {
+			var g_by_eval = new DistinctRowEvaluator(cols);
+			ViewStream r = new GroupByStream(stream, g_by_eval);
+			var proj_exp = String.Empty;
+			foreach (var c in cols)
+				proj_exp += $"{c} ";
+			proj_exp = proj_exp.Trim();
+			r = new NewProjectStream(r, proj_exp);
+			mySetRealSource(r);
 		}
 	}
 }

src/DataStreamSuite/DataStream/Primitives/FileStreams/DumpLoaderStream.cs

@@ -1,49 +0,0 @@
-// using System;
-// using System.Collections.Generic;
-// using System.Text;
-// using System.IO;
-
-// using System.Runtime.Serialization.Formatters.Binary;
-
-// namespace BCG
-// {
-//    public class DumpLoaderStream : ViewStream, ISideEffectStream
-//    {
-//       //wtf
-//       //public class Exception : System.Exception
-//       //{public Exception(string msg) : base(msg) { }}
-
-//       public DumpLoaderStream(string fileName, int secondsBetweenPolls)
-//       {
-//          mySetRealSource(
-//             new PollStream(() => new LoaderStream(fileName).rows, secondsBetweenPolls, 
-//                ViewStreamDatabase.instance));
-//       }
-
-//       public DumpLoaderStream(string fileName)
-//       {
-//         if (fileName == null)
-//            throw new LoaderStream.Exception("File name cannot be null");
-
-//         for (int i = 0; i < 5; i++)
-//            try
-//            {
-//               using (FileStream fs = new FileStream(fileName, FileMode.Open, FileAccess.Read))
-//               {
-//                  IClientDeserializer deser = new ClientUniversalDeserializer();
-//                  byte[] the_ops;
-//                  Array the_rows;
-//                  deser.deserialize(new Stream[] { fs }, out the_ops, out the_rows);
-
-//                  mySetRealSource(new BlueRowCollectionStaticStream((IBlueRowCollection)the_rows.GetValue(0)));
-//               }
-//               break;
-//            }
-//            catch (IOException)
-//            {
-//               System.Threading.Thread.Sleep(50);
-//            }
-
-//       }
-//    }
-// }

src/DataStreamSuite/DataStream/Primitives/FileStreams/LoaderStream.cs

@@ -1,6 +1,7 @@
 using BCG;
 
 namespace bts {
+   // This is the other side of the SaverStream, its "loads" the stream and fires once
    [DataStreamUserRespresentation("loader")]
    public class LoaderStream : ViewStream {
 

src/DataStreamSuite/DataStream/Primitives/FileStreams/PersistorStream.cs

@@ -5,10 +5,11 @@ using System.Text;
 using System.IO;
 using Google.Protobuf;
 using System.Globalization;
-using bts;
+using BCG;
 
-namespace BCG {
-   // Writes all the udpates to a stream in a file.
+namespace bts {
+   // Writes all the udpates to a stream in a file or multiples files dependong on the rollover size (as opposed to Saver
+   // which just writes one update, as a create)
    // File naming:
    // token_epoch_yyyyMMdd_HHmmss
    // File location: directory(passed)\token\
@@ -94,10 +95,10 @@ namespace BCG {
       void _on_change(object new_value) {
          DataPacket dp = new_value as DataPacket;
          byte[] serialized_update = null;
-         bool _write_create = _serializer == null || _n_updates_in_current_file == 0;
+         bool first_write = _serializer == null || _n_updates_in_current_file == 0;
          _serializer = _serializer ?? new UniversalStreamIOSerializer();
-         if (_write_create) {
-            // new files always starts with create
+         if (first_write) {
+            // first file always starts with create
             serialized_update = _serializer.serialize(new bts.StreamIOMessage(new bts.StreamIOMessage.CreateMessage(dp.newRows)));
          } else
             serialized_update = _serializer.serialize(new bts.StreamIOMessage(new bts.StreamIOMessage.MultipleRemoveAddMessage(dp.deletedRows, dp.addedRows)));
@@ -105,8 +106,8 @@ namespace BCG {
          long utc_epoch = epoch_now();
 
          //time, length of data, data
-         bw.Write(utc_epoch);
-         bw.Write(serialized_update.Length);
+         //bw.Write(utc_epoch);
+         //bw.Write(serialized_update.Length);
          bw.Write(serialized_update);
          _out_fs.Flush();
          myFireEvent(dp);

src/DataStreamSuite/DataStream/Primitives/FileStreams/PlaybackStream.cs

@@ -1,190 +0,0 @@
-// using System;
-// using System.Collections.Generic;
-// using System.Linq;
-// using System.Text;
-// using System.IO;
-// using System.Runtime.Serialization.Formatters.Binary;
-// using BCG.Utilities.General;
-// using bts.collections;
-// using System.Reflection.Metadata;
-
-
-// namespace BCG {
-
-//    public class PlaybackStream : ViewStream, IWorker {
-//       private readonly PlaybackType _type;
-//       private BlueTable _current_table;
-//       private IClientDeserializer _deserilializer = new ClientUniversalDeserializer();
-//       // these are the packets already downloaded from the current file
-//       private List<PlaybackPacket> _packets = new List<PlaybackPacket>();
-//       //private IEnumerable<FileStream> _file_streams = new List<FileStream>();
-//       private IEnumerable<string> _file_names;
-//       private Tuple<int, FileStream> _current_fs = new Tuple<int, FileStream>(-1, null);
-//       private DateTime _playback_t0, _playback_t1;
-//       private bool _all_files_read = false;
-//       SnapshotData _snapshot_data = null; // only initialized when in snapshot mode
-
-//       public PlaybackStream(string datetime_start, string datetime_end, string directory_name, string token) {
-//          _type = PlaybackType.Streaming;
-//          _playback_t0 = PersistorStream.parse_datetime(datetime_start);
-//          _playback_t1 = PersistorStream.parse_datetime(datetime_end);
-//          string token_directory_name = $"{directory_name}\\{token}";
-//          bts.Log.Gen.info($"PlaybackStream t0: {_playback_t0} t1: {_playback_t1} dir: {token_directory_name}");
-//          List<string> files_in_range = new List<string>();
-//          if (Directory.Exists(token_directory_name)) {
-//             _file_names =
-//          Directory.GetFiles(token_directory_name).Where(f => {
-//             DateTime file_time = PersistorStream.parse_datetime_from_filename(f);
-//             return file_time > _playback_t0 && file_time < _playback_t1;
-//          });
-//          }
-//       }
-
-//       protected override void myOnFirstSubscriber() {
-//          FileStream fs = new FileStream(_snapshot_data.snapshot_file, FileMode.Open, FileAccess.Read);
-//          _snapshot_data.packets = StreamPersistorUtils.load_packets_from_file(fs);
-//          PlaybackPacket snapshot_packet = StreamPersistorUtils.find_packet_at_time(_snapshot_data.packets, _snapshot_data.snapshot_time);
-//          if (snapshot_packet != null) {
-//             foreach (PlaybackPacket p in _snapshot_data.packets) {
-//                byte[] ops;
-//                Array row_array;
-//                _deserilializer.deserialize(new Stream[] { new MemoryStream(p.data) }, out ops, out row_array);
-//                for (int i = 0; i < ops.Length; i++) {
-//                   IBlueRowCollection[] rows_args = row_array.GetValue(i) as IBlueRowCollection[];
-//                   byte op = ops[i];
-//                   if (op == StreamOpCode.create) {
-//                      IBlueRowCollection full_table = rows_args[0];
-//                      _current_table = new BlueTable(full_table);
-//                   } else if (op == StreamOpCode.multipleRemoveAdd) {
-//                      //IBlueRowCollection[] del_added = rows_args.GetValue(0) as IBlueRowCollection[];
-//                      IBlueRowCollection dels = rows_args.GetValue(0) as IBlueRowCollection;
-//                      IBlueRowCollection adds = rows_args.GetValue(1) as IBlueRowCollection;
-//                      BlueDataUtility.absorbChanges(_current_table, dels, adds);
-//                   } else {
-//                      bts.Log.assert(false, $"Oprator: {op} is not expected");
-//                   }
-//                   if (p.idx == snapshot_packet.idx) {
-//                      myFireEvent(new DataPacket(_current_table, _current_table, new BlueTable(_current_table.schema)));
-//                      break;
-//                   }
-//                }
-//             }
-//          }
-//       }
-
-
-//       private void _load_packets_from_current_file() {
-//          int curr_idx = _current_fs.Item1;
-//          FileStream curr_fs = _current_fs.Item2;
-//          _packets = new List<PlaybackPacket>();
-//          _packets.AddRange(StreamPersistorUtils.load_packets_from_file(curr_fs));
-//       }
-
-//       private bool current_file_all_replayed {
-//          get {
-//             bool rv = true;
-//             _packets.ForEach(p => rv = rv && p.replayed);
-//             return rv;
-//          }
-//       }
-
-//       protected override void myOnLastSubscriber() {
-//          var fs = _current_fs.Item2;
-//          if (fs != null) {
-//             fs.Close();
-//          }
-//          ViewStreamDatabase.instance.removeWorker(this);
-//       }
-
-//       // public event PlaybackStreamProgress onProgress;
-//       // public event Action onFileExhausted;
-//       private PlaybackPacket _locate_packet_to_replay() {
-//          foreach (var p in _packets) {
-//             // returns the first non replayed packet
-//             if (!p.replayed)
-//                return p;
-//          }
-//          //else
-//          {
-//             double secondsElapsed = new BinaryReader(myFileReader).ReadDouble();
-//             myNextPacketTime = myPlaybackStart + new TimeSpan(0, 0, 0, 0, (int)(1000 * secondsElapsed / myPlaybackSpeed));
-//             if (onProgress != null)
-//                onProgress(myFileReader.Position, myFileReader.Length);
-//          }
-//       }
-
-//       #region IWorker Members
-
-//       public void doWork() {
-//          if (_all_files_read)
-//             return;
-//          if (current_file_all_replayed) {
-//             _load_next_file();
-//          }
-//          PlaybackPacket curr_packet = _locate_packet_to_replay();
-//          if (curr_packet.idx == 0) { // first packet in the file, fire it rightaway 
-//             byte[] ops;
-//             Array row_array;
-//             // what a stupid interface!
-//             _deserilializer.deserialize(new Stream[] { new MemoryStream(curr_packet.data) }, out ops, out row_array);
-//             _fire_packet(ops, row_array);
-//             curr_packet.replay_timestamp = PersistorStream.epoch_now();
-//          } else {
-//             PlaybackPacket prev_packet = _get_previous_packet(curr_packet);
-//             // this is the time we need to fire (replay) this packet
-//             long time_to_fire = prev_packet.replay_timestamp + curr_packet.offset_from_previous;
-//             long now_epoch = PersistorStream.epoch_now();
-//             // fire when we are past the 
-//             if (now_epoch >= time_to_fire) {
-//                byte[] ops;
-//                Array row_array;
-//                _deserilializer.deserialize(new Stream[] { new MemoryStream(curr_packet.data) }, out ops, out row_array);
-//                _fire_packet(ops, row_array);
-//                curr_packet.replay_timestamp = PersistorStream.epoch_now();
-//             }
-//          }
-//       }
-
-//       private void _fire_packet(byte[] ops, Array row_array) {
-//          for (int i = 0; i < ops.Length; i++) {
-//             IBlueRowCollection[] rows_args = row_array.GetValue(i) as IBlueRowCollection[];
-//             byte op = ops[i];
-//             if (op == StreamOpCode.create) {
-//                IBlueRowCollection full_table = rows_args[0];
-//                _current_table = new BlueTable(full_table);
-//                //_current_table.addRows(full_table);
-//                myFireEvent(new DataPacket(full_table, full_table, myLastValue == null ? new BlueTable(full_table.schema) :
-//                                           (myLastValue as DataPacket).newRows));
-//             } else if (op == StreamOpCode.multipleRemoveAdd) {
-//                //IBlueRowCollection[] del_added = rows_args.GetValue(0) as IBlueRowCollection[];
-//                IBlueRowCollection dels = rows_args.GetValue(0) as IBlueRowCollection;
-//                IBlueRowCollection adds = rows_args.GetValue(1) as IBlueRowCollection;
-//                BlueDataUtility.absorbChanges(_current_table, dels, adds);
-//                myFireEvent(new DataPacket(_current_table, adds, dels));
-//             } else {
-//                bts.Log.assert(false, $"Oprator: {op} is not expected");
-//             }
-
-//          }
-//       }
-
-//       private void _load_next_file() {
-//          var new_idx = _current_fs.Item1 + 1;
-//          if (new_idx >= _file_names.Count()) {
-//             _all_files_read = true;
-//             bts.Log.Gen.info("All files read");
-//             return;
-//          }
-//          var prev_fs = _current_fs.Item2;
-//          if (prev_fs != null)
-//             prev_fs.Close();
-//          var new_filename = _file_names.ElementAt(new_idx);
-//          bts.Log.Gen.info("PlaybackStream_load_next_file new_filename=: {new_filename}");
-//          var new_fs = new FileStream(new_filename, FileMode.Open, FileAccess.Read);
-//          _current_fs = new Tuple<int, FileStream>(new_idx, new_fs);
-//          _load_packets_from_current_file();
-//       }
-
-//       #endregion
-//    }
-// }

src/DataStreamSuite/DataStream/Primitives/FileStreams/RecorderStream.cs

@@ -1,68 +0,0 @@
-// using System;
-// using System.Collections.Generic;
-// using System.Linq;
-// using System.Text;
-// using System.IO;
-
-// namespace BCG
-// {
-//    public class RecorderStream : ViewStream
-//    {
-//       private ViewStream mySource;
-//       private DateTime myStartTime;
-//       private FileStream myOutFile;
-//       private string myFileName;
-//       private IBlueDataSerializer mySerializer;
-
-//       public RecorderStream(ViewStream src, string recordFile)
-//       {
-//          mySource = src;
-//          myFileName = recordFile;
-
-//          myFileName = myFileName.Replace("%DATE%", DateTime.Now.ToString("yyyyMMdd_HHmmss")); 
-//       }
-
-//       protected override void myOnFirstSubscriber()
-//       {
-//          myStartTime = DateTime.Now;         
-//          mySource.onChange += new ChangeDelegate(mySource_onChange);
-//          myOutFile = new FileStream(myFileName, FileMode.OpenOrCreate);
-//       }
-
-//       protected override void myOnLastSubscriber()
-//       {
-//          mySource.onChange -= mySource_onChange;
-//          myOutFile.Close();
-//       }
-
-//       void mySource_onChange(object newValue)
-//       {
-//          DataPacket dp = (DataPacket)newValue;         
-
-//          BlueTable toWrite = new BlueTable(dp.schema);
-
-//          if (mySerializer == null)
-//          {
-//             mySerializer = new UniversalSerializer(dp.schema);
-//          }
-//          foreach (IBlueRow r in dp.newRows)
-//             toWrite.addRow(r);
-
-//          byte[] result;
-
-//          if (myLastValue == null)
-//             mySerializer.serialize(StreamOpCode.create, new IBlueRowCollection[] { dp.newRows }, out result);
-//          else
-//             mySerializer.serialize(StreamOpCode.multipleRemoveAdd, new IBlueRowCollection[] { dp.deletedRows, dp.addedRows }, out result);
-
-//          BinaryWriter bw = new BinaryWriter(myOutFile);
-//          //write down what time this happened
-//          bw.Write((DateTime.Now - myStartTime).TotalSeconds);
-//          bw.Write(result.Length);
-//          bw.Write(result);         
-//          myOutFile.Flush();
-
-//          myFireEvent(dp);
-//       }
-//    }
-// }

src/DataStreamSuite/DataStream/Primitives/FileStreams/SaverStream.cs

@@ -5,7 +5,8 @@ using System.IO;
 using BCG;
 
 namespace bts {
-   // this saves the total state in the filr provided in the cons
+   // this saves the updates as a Create on the same file, each update calls -save, which
+   // reqrites the whole data to the packet. The other side of this stream is the Loader
    [DataStreamUserRespresentation("saver")]
    public class SaverStream : ViewStream {
       private ViewStream _stream;
@@ -31,7 +32,6 @@ namespace bts {
          StreamIOMessage.CreateMessage cr_msg = new StreamIOMessage.CreateMessage(dp.newRows);
          byte[] str_bytes = _serializer.serialize(new StreamIOMessage(cr_msg));
          StreamSerializationUtils.write_to_file(str_bytes, _filename);
-
       }
    }
 }

src/DataStreamSuite/DataStream/Primitives/FileStreams/StreamPersistorUtils.cs

@@ -3,6 +3,7 @@ using System.Collections.Generic;
 using System.Linq;
 using System.Globalization;
 using System.IO;
+using bts;
 
 
 namespace BCG {

src/DataStreamSuite/DataStream/Primitives/Misc/PubSubTestStream.cs

@@ -8,26 +8,33 @@ using System;
 using System.IO;
 using System.Linq;
 
-namespace bts {
-    public class PubSubTestingStream : ViewStream, IWorker {
-        public class PayloadMessage {
+namespace bts
+{
+    public class PubSubTestingStream : ViewStream, IWorker
+    {
+        public class PayloadMessage
+        {
             public int msg_id = 0;
             public string guid;
             public Int64 travel_epoch_ms = 0;
             public byte[] payload = null;
-            public PayloadMessage(int msg_id_args, Int64 send_epoch_args, int payload_size, byte payload_byte) {
+            public PayloadMessage(int msg_id_args, Int64 send_epoch_args, int payload_size, byte payload_byte)
+            {
                 msg_id = msg_id_args;
                 travel_epoch_ms = send_epoch_args;
                 payload = new byte[payload_size];
-                for (int i = 0; i < payload_size; i++) {
+                for (int i = 0; i < payload_size; i++)
+                {
                     payload[i] = payload_byte;
                 }
             }
             private PayloadMessage() { }
             public PayloadMessage(int msg_id, int payload_size, byte payload_byte) :
-            this(msg_id, PubSubTestingStream.epoch_now(), payload_size, payload_byte) { }
+            this(msg_id, PubSubTestingStream.epoch_now(), payload_size, payload_byte)
+            { }
 
-            public byte[] serialize() {
+            public byte[] serialize()
+            {
                 var str = new MemoryStream();
                 BinaryWriter bw = new BinaryWriter(str);
                 bw.Write(msg_id);
@@ -40,7 +47,8 @@ namespace bts {
                 return bytes;
             }
 
-            public static PayloadMessage deserialize(byte[] ser_bytes) {
+            public static PayloadMessage deserialize(byte[] ser_bytes)
+            {
                 var str = new MemoryStream(ser_bytes);
                 BinaryReader br = new BinaryReader(str);
                 int msg_id = br.ReadInt32();
@@ -48,7 +56,8 @@ namespace bts {
                 Int64 epoch = br.ReadInt64();
                 int len = br.ReadInt32();
                 var payload = br.ReadBytes(len);
-                return new PayloadMessage() {
+                return new PayloadMessage()
+                {
                     msg_id = msg_id,
                     guid = guid,
                     travel_epoch_ms = epoch,
@@ -73,7 +82,8 @@ namespace bts {
         private readonly int _payload_sz;
         private const byte _payload_byte = (byte)'A';
         private readonly string _tag_name;
-        public PubSubTestingStream(int publish_period_msecs, int max_pubs, int payload_size, string tag_name, ViewStreamDatabase vsd) {
+        public PubSubTestingStream(int publish_period_msecs, int max_pubs, int payload_size, string tag_name, ViewStreamDatabase vsd)
+        {
             _vsd = vsd;
             _payload_sz = payload_size;
             var schema = BlueDataUtility.form_schema("PubSubTest",
@@ -96,43 +106,53 @@ namespace bts {
         public PubSubTestingStream(int publish_period_seconds, int max_pubs, int msg_size, string tag_name)
         : this(publish_period_seconds, max_pubs, msg_size, tag_name, ViewStreamDatabase.instance) { }
 
-        protected override void myOnFirstSubscriber() {
+        protected override void myOnFirstSubscriber()
+        {
             _vsd.addWorker(this);
             System.Threading.Thread th = new System.Threading.Thread(_subscribe_thread_func);
             th.IsBackground = true;
             th.Start();
         }
 
-        protected override void myOnLastSubscriber() {
+        protected override void myOnLastSubscriber()
+        {
             _vsd.removeWorker(this);
             _continue_sub_thread = false;
         }
 
-        void _log_updates(IEnumerable<PubSubServerMessage> updates) {
+        void _log_updates(IEnumerable<PubSubServerMessage> updates)
+        {
             var idx = 0;
-            foreach (var msg in updates) {
+            foreach (var msg in updates)
+            {
                 bts.Log.Gen.info($"PubSubTesStream._subscribe_thread_func idx: {idx} msg: {msg}");
                 idx++;
             }
         }
 
-        private void _subscribe_thread_func() {
+        private void _subscribe_thread_func()
+        {
             // subscription is done in another thread
             SubscriberCore sub_core = new SubscriberCore();
             sub_core.subscribe(_tag_name);
-            while (_continue_sub_thread) {
+            while (_continue_sub_thread)
+            {
                 // TODO recv_id
                 var updates = sub_core.get_updates(TimeSpan.FromMilliseconds(250));
                 if (updates == null || updates.Count() == 0)
                     continue;
                 _log_updates(updates);
                 _n_updates += updates.Count();
-                if (updates.Count() > 0) {
+                if (updates.Count() > 0)
+                {
                     var server_msg = updates.Last();
-                    if (server_msg.data != null && server_msg.data.Length > 0) {
+                    if (server_msg.data != null && server_msg.data.Length > 0)
+                    {
                         _last_recvd_msg = PayloadMessage.deserialize(server_msg.data);
                         _last_recvd_msg.travel_epoch_ms = epoch_now();
-                    } else {
+                    }
+                    else
+                    {
                         // why are we getting empty or null data ? (server is sending data regardless?)
                     }
                     _fire_dirty = true;
@@ -140,28 +160,36 @@ namespace bts {
             }
         }
 
-        private void _publish_next_packet() {
-            if (_last_sent_msg == null) {
-                _last_sent_msg = new PayloadMessage(0, _payload_sz, _payload_byte) {
+        private void _publish_next_packet()
+        {
+            if (_last_sent_msg == null)
+            {
+                _last_sent_msg = new PayloadMessage(0, _payload_sz, _payload_byte)
+                {
                     guid = Guid.NewGuid().ToString()
                 };
-            } else {
+            }
+            else
+            {
                 _last_sent_msg.msg_id++;
                 _last_sent_msg.guid = Guid.NewGuid().ToString();
                 _last_sent_msg.travel_epoch_ms = epoch_now();
             }
             var ser_bytes = _last_sent_msg.serialize();
             _n_pubs++;
-            _pub_core.publish(String.Empty, _tag_name, ser_bytes);
+            _pub_core.publish("Blob", _tag_name, ser_bytes);
+            _fire_dirty = true;
         }
 
-        private void _fire_dp() {
+        private void _fire_dp()
+        {
             object last_sent_id = _last_sent_msg == null ? (object)DBNull.Value : (object)_last_sent_msg.msg_id;
             object last_recv_id = _last_recvd_msg == null ? (object)DBNull.Value : (object)_last_recvd_msg.msg_id;
             object echo_time = DBNull.Value;
             object last_sent_guid = _last_sent_msg == null ? (object)DBNull.Value : (object)_last_sent_msg.guid;
             object last_recv_guid = _last_recvd_msg == null ? (object)DBNull.Value : (object)_last_recvd_msg.guid;
-            if (_last_sent_msg != null && _last_recvd_msg != null) {
+            if (_last_sent_msg != null && _last_recvd_msg != null)
+            {
                 echo_time = _last_recvd_msg.travel_epoch_ms - _last_sent_msg.travel_epoch_ms;
             }
             var match = last_sent_id.Equals(last_recv_id) && last_sent_guid.Equals(last_recv_guid);
@@ -173,21 +201,25 @@ namespace bts {
             myFireEvent(dp);
         }
 
-        public void doWork() {
+        public void doWork()
+        {
             if (DateTime.Now - _last_publish_time > _publish_period)
-                if (_n_pubs <= _max_pubs) {
+                if (_n_pubs <= _max_pubs)
+                {
                     _publish_next_packet();
                     _last_publish_time = DateTime.Now;
                 }
             // publish 
             // TODO: send id
-            if (_fire_dirty) {
+            if (_fire_dirty)
+            {
                 _fire_dirty = false;
                 _fire_dp();
             }
         }
 
-        public static long epoch_now() {
+        public static long epoch_now()
+        {
             return DateTimeOffset.UtcNow.ToUnixTimeMilliseconds();
         }
 

src/DataStreamSuite/DataStream/Primitives/Misc/PubsubEchoTestStream.cs

@@ -0,0 +1,227 @@
+using BCG;
+using BCG.Utilities.General;
+using System.Collections.Generic;
+
+
+using System;
+using System.IO;
+using System.Linq;
+using System.Runtime.CompilerServices;
+using bts.api;
+
+namespace bts
+{
+   [DataStreamUserRespresentation("pss_echo")]
+   public class PubsubEchoTestStream : ViewStream, IWorker
+   {
+      public class PayloadMessage
+      {
+         public readonly string msg_id;
+         public readonly Int64 timestamp = 0;
+         public readonly byte[] payload = null;
+
+         public static byte[] create_data(int payload_size, byte payload_byte)
+         {
+            var rv = new byte[payload_size];
+            for (int i = 0; i < payload_size; i++)
+            {
+               rv[i] = payload_byte;
+            }
+            return rv;
+         }
+
+         // public PayloadMessage(string guid_, Int64 timestamp_, int payload_size, byte payload_byte)
+         // {
+         //    guid = guid_;
+         //    timestamp = timestamp_;
+         //    payload = new byte[payload_size];
+         //    for (int i = 0; i < payload_size; i++)
+         //    {
+         //       payload[i] = payload_byte;
+         //    }
+         // }
+         public PayloadMessage(string guid_, Int64 timestamp_, byte[] payload_)
+         {
+            msg_id = guid_;
+            timestamp = timestamp_;
+            payload = payload_;
+         }
+
+         private PayloadMessage() { }
+         public byte[] serialize()
+         {
+            var str = new MemoryStream();
+            BinaryWriter bw = new BinaryWriter(str);
+            bw.Write(msg_id);
+            bw.Write(timestamp);
+            bw.Write(payload.Length);
+            bw.Write(payload);
+            str.Position = 0;
+            var bytes = str.ToArray();
+            return bytes;
+         }
+
+         public static PayloadMessage deserialize(byte[] ser_bytes)
+         {
+            var str = new MemoryStream(ser_bytes);
+            BinaryReader br = new BinaryReader(str);
+            string guid = br.ReadString();
+            Int64 timestamp_ = br.ReadInt64();
+            int len = br.ReadInt32();
+            var payload = br.ReadBytes(len);
+            return new PayloadMessage(guid, timestamp_, payload);
+         }
+      }
+      //public void
+      private readonly int _msg_size;
+      private readonly TimeSpan _publish_period;
+      private DateTime _last_publish_time = DateTime.MinValue;
+      private PublisherCore _pub_core;
+      private ISubscriberCore _sub_core;
+      private BlueTable _table;
+      private int _n_pubs = 0;
+      private readonly int _max_pubs;
+      private int _n_updates = 0;
+      private ViewStreamDatabase _vsd;
+      private PayloadMessage _last_sent_msg, _last_recvd_msg;
+      private const byte _payload_byte = (byte)'A';
+      private readonly string _tag_name;
+      private readonly int _payload_sz;
+      private readonly IPubSubRouter _router;
+      private enum IOState
+      {
+         SEND = 0,
+         RECV = 1
+      };
+      IOState _state = IOState.SEND;
+
+      public PubsubEchoTestStream(string server_url, TimeSpan publish_period, int payload_size, string tag_name, ViewStreamDatabase vsd)
+      {
+         var route = new PubSubRoute(server_url, ".*");
+         _router = new PubSubRouter(route);
+         _vsd = vsd;
+         _payload_sz = payload_size;
+         var schema = BlueDataUtility.form_schema("PubSubTest",
+         Tuple.Create("id", TypeCode.Int32),
+         Tuple.Create("sent_id", TypeCode.String),
+         Tuple.Create("sent_time", TypeCode.Int64),
+         Tuple.Create("recv_id", TypeCode.String),
+         Tuple.Create("recv_time", TypeCode.Int64),
+         Tuple.Create("match", TypeCode.Boolean),
+         Tuple.Create("echo_time", TypeCode.Int32));
+         _table = new BlueTable(schema);
+         _publish_period = publish_period;
+         _tag_name = tag_name;
+      }
+
+      public PubsubEchoTestStream(string server_url, int publish_period_msecs, int payload_size, string tag_name)
+      : this(server_url, TimeSpan.FromMilliseconds(publish_period_msecs), payload_size, tag_name, ViewStreamDatabase.instance) { }
+
+      protected override void myOnFirstSubscriber()
+      {
+         _pub_core = new PublisherCore(_router);
+         _sub_core = new SubscriberCore(_router);
+         _sub_core.subscribe(_tag_name);
+         _vsd.addWorker(this);
+
+      }
+
+      protected override void myOnLastSubscriber()
+      {
+         _vsd.removeWorker(this);
+         _sub_core.unsubscribe(_tag_name);
+      }
+
+      private void _fire_dp_after_send()
+      {
+         // as soon as the data is published, its reflected in the stream state
+         Log.assert(_last_sent_msg != null);
+         BlueRow r = new BlueRow(_table.schema, _table.Count, _last_sent_msg.msg_id, _last_sent_msg.timestamp, DBNull.Value, DBNull.Value,
+            DBNull.Value, DBNull.Value);
+         _table.addRow(r);
+         var dp_out = BlueDataUtility.build_dp_inc(_table, myLastValue as DataPacket);
+         myFireEvent(dp_out);
+      }
+
+      private void _fire_dp_after_recv()
+      {
+         Log.assert(_last_recvd_msg != null);
+         // update the last publish row
+         int id = _table.Count - 1;
+         IBlueRow last_row = _table[id];
+         _table.deleteRow(last_row);
+         var recv_ts = epoch_now();
+         BlueRow nr = new BlueRow(_table.schema, id, _last_sent_msg.msg_id, _last_sent_msg.timestamp, _last_recvd_msg.msg_id, recv_ts,
+                                 String.Compare(_last_recvd_msg.msg_id, _last_sent_msg.msg_id) == 0, recv_ts - _last_recvd_msg.timestamp);
+         _table.addRow(nr);
+         var dp_out = BlueDataUtility.build_dp_inc(_table, myLastValue as DataPacket);
+         myFireEvent(dp_out);
+      }
+
+      private string _get_next_id() {
+         return _table.Count.ToString();
+      }
+
+      private string _get_next_id()
+      {
+         return _table.Count.ToString();
+      }
+
+      private void _publish_next_packet()
+      {
+         var bytes = PayloadMessage.create_data(_payload_sz, _payload_byte);
+         _last_sent_msg = new PayloadMessage(_get_next_id(), epoch_now(), bytes);
+         var ser_bytes = _last_sent_msg.serialize();
+         _n_pubs++;
+         _pub_core.publish("Blob", _tag_name, ser_bytes);
+         _last_publish_time = DateTime.Now;
+      }
+
+      public void doWork()
+      {
+         Func<IEnumerable<PubSubServerMessage>, PayloadMessage> find_packet = (updates) =>
+         {
+            foreach (var u in updates)
+            {
+               var recv_msg = PayloadMessage.deserialize(u.data);
+               if (recv_msg.msg_id == _last_sent_msg.msg_id)
+               {
+                  return recv_msg;
+               }
+            }
+            return null;
+         };
+         switch (_state)
+         {
+            case IOState.SEND:
+               if (DateTime.Now - _last_publish_time >= _publish_period)
+               {
+                  _publish_next_packet();
+                  _last_publish_time = DateTime.Now;
+                  _fire_dp_after_send();
+                  _state = IOState.RECV;
+                  return;
+               }
+               break;
+            case IOState.RECV:
+               var updates = _sub_core.get_updates(TimeSpan.Zero);
+               updates = updates.Where(u => u.tag == _tag_name);
+               var recv_msg = find_packet(updates);
+               if (recv_msg != null)
+               {
+                  _last_recvd_msg = recv_msg;
+                  _fire_dp_after_recv();
+                  _state = IOState.SEND;
+                  break;
+               }
+               break;
+         }
+      }
+
+      public static long epoch_now()
+      {
+         return DateTimeOffset.UtcNow.ToUnixTimeMilliseconds();
+      }
+
+   }
+}

src/DataStreamSuite/DataStream/Primitives/Misc/SimpleTCPEchoStream.cs

@@ -14,11 +14,14 @@ namespace bts {
       private BlueTable _table;
       private SyncQueue<string> _action_queue = new SyncQueue<string>();
 
-      public SimpleTCPEchoStream(string ip, int port, int msg_sz, int send_wait_period_ms) {
+      public SimpleTCPEchoStream(string ip, int port, int msg_sz, int send_wait_period_ms)
+      : this(ip, port, msg_sz, send_wait_period_ms, Int32.MaxValue) { }
+
+      public SimpleTCPEchoStream(string ip, int port, int msg_sz, int send_wait_period_ms, int max_n_msgs) {
          _ip = ip;
          _port = port;
          _table = new BlueTable(_schema);
-         _tcp_client = new SimpleTCPClient(_ip, _port, Convert.ToUInt32(msg_sz), send_wait_period_ms);
+         _tcp_client = new SimpleTCPClient(_ip, _port, Convert.ToUInt32(msg_sz), send_wait_period_ms, max_n_msgs);
       }
 
       protected override void myOnFirstSubscriber() {

src/DataStreamSuite/DataStream/Primitives/Misc/TestStream.cs

@@ -43,11 +43,11 @@ namespace BCG {
             Tuple.Create("exception", TypeCode.String)
             );
             // register tests here
-            // _tests.Add(new Func<TestResult>(_run_sub_msg_test));
-            // _tests.Add(new Func<TestResult>(_run_pub_msg_test));
-            // _tests.Add(new Func<TestResult>(_run_pubsub_server_msg_test));
-            // _tests.Add(new Func<TestResult>(_run_pubsub_client_msg_test));
-            // _tests.Add(new Func<TestResult>(_run_columninfo_serialization_test));
+            _tests.Add(new Func<TestResult>(_run_sub_msg_test));
+            _tests.Add(new Func<TestResult>(_run_pub_msg_test));
+            _tests.Add(new Func<TestResult>(_run_pubsub_server_msg_test));
+            _tests.Add(new Func<TestResult>(_run_pubsub_client_msg_test));
+            _tests.Add(new Func<TestResult>(_run_columninfo_serialization_test));
             // _tests.Add(new Func<TestResult>(_run_schema_serialization_test));
             // _tests.Add(new Func<TestResult>(_row_serialization_tests_no_dbnulls));
             //  _tests.Add(new Func<TestResult>(_row_serialization_tests_with_dbnulls));

src/DataStreamSuite/DataStream/Primitives/Misc/TickingTestStream.cs

@@ -0,0 +1,31 @@
+namespace BCG {
+   using System;
+
+   public class TickingTestStream : ViewStream {
+      public TickingTestStream(int ticking_period_msecs)
+      :
+      this(ViewStreamDatabase.instance, TimeSpan.FromSeconds(ticking_period_msecs)) { }
+
+      public TickingTestStream(ViewStreamDatabase vsd, TimeSpan ticking_period_msecs) {
+         ViewStream vs = new DisplayableMillisecondStream(Convert.ToInt32(ticking_period_msecs.TotalMilliseconds), 10);
+         vs = new MathStream(vs, "x=id*123 y=x*x");
+         mySetRealSource(vs);
+         //vs.onChange += _on_str_update;
+      }
+
+      // private void _on_str_update(object dp_obj) {
+      //    DataPacket dp = dp_obj as DataPacket;
+      //    IBlueTableSchema out_schema = BlueDataUtility.add_column(dp.schema, "time_txt", typeof(string));
+      //    BlueTable state = new BlueTable(out_schema);
+      //    foreach (IBlueRow r in dp.newRows) {
+      //       MutableBlueRow nr = new MutableBlueRow(out_schema, r);
+      //       DateTime time = r.get_datetime("time");
+      //       var time_txt = time.ToString("yyyy-MM-dd HH:mm:ss.fff");
+      //       nr.setValue("time_txt", time_txt);
+      //       state.addRow(nr);
+      //    }
+      //    var dp_out = BlueDataUtility.build_dp_inc(state, myLastValue == null ? null : myLastValue as DataPacket);
+      //    myFireEvent(dp_out);
+      // }
+   }
+}

src/DataStreamSuite/DataStream/Primitives/MongoDB/MongoDBStream.cs

@@ -7,122 +7,127 @@ using MongoDB.Driver;
 using System;
 using System.Collections.Generic;
 using System.Diagnostics;
+using System.Threading.Tasks;
+using System.Threading;
 
 namespace bts {
-    [DataStreamUserRespresentation("mongodb_sql")]
-    public class MongoDBSQLStream : ViewStream, IWorker {
-        private ViewStreamDatabase _vsd;
-        private readonly string _connection_string, _db_name, _collection_name;
-        private MongoClient _client;
-        // make this a sorted table
-        private BlueTable _table;
+    // [DataStreamUserRespresentation("mongodb_sql")]
+    // public class MongoDBSQLStream : ViewStream, IWorker {
+    //     private ViewStreamDatabase _vsd;
+    //     private readonly string _connection_string, _db_name, _collection_name;
+    //     private MongoClient _client;
+    //     // make this a sorted table
+    //     private BlueTable _table;
 
 
-        // Access the database
-        private IMongoDatabase _database;
-        private IDictionary<string, int> _id_map = new Dictionary<string, int>();
-        // mongodb://192.168.1.80:27017
+    //     // Access the database
+    //     private IMongoDatabase _database;
+    //     private IDictionary<string, int> _id_map = new Dictionary<string, int>();
+    //     // mongodb://192.168.1.80:27017
 
-        public MongoDBSQLStream(string connection_string, string db_name, string collection_name)
-        : this(connection_string, db_name, collection_name, ViewStreamDatabase.instance) {
+    //     public MongoDBSQLStream(string connection_string, string db_name, string collection_name)
+    //     : this(connection_string, db_name, collection_name, ViewStreamDatabase.instance) {
 
-        }
-        public MongoDBSQLStream(string connection_string, string db_name, string collection_name, ViewStreamDatabase vsd) {
-            bts.Log.Gen.info($"MongoDBStream constructed connection_string: {connection_string} " +
-                                    $"db_name: {db_name}");
-            _connection_string = connection_string;
-            _db_name = db_name;
-            _collection_name = collection_name;
-            _vsd = vsd;
-            _client = new MongoClient(_connection_string);
-            _database = _client.GetDatabase(_db_name);
-        }
+    //     }
+    //     public MongoDBSQLStream(string connection_string, string db_name, string collection_name, ViewStreamDatabase vsd) {
+    //         bts.Log.Gen.info($"MongoDBStream constructed connection_string: {connection_string} " +
+    //                                 $"db_name: {db_name}");
+    //         _connection_string = connection_string;
+    //         _db_name = db_name;
+    //         _collection_name = collection_name;
+    //         _vsd = vsd;
+    //         _client = new MongoClient(_connection_string);
+    //         _database = _client.GetDatabase(_db_name);
+    //     }
 
-        protected override void myOnFirstSubscriber() {
-            MasterWorker.getInstance(false).addWorker(this, TimeSpan.FromSeconds(10));
-        }
+    //     protected override void myOnFirstSubscriber() {
+    //         MasterWorker.getInstance(false).addWorker(this, TimeSpan.FromSeconds(10));
+    //     }
 
-        protected override void myOnLastSubscriber() {
-            MasterWorker.getInstance(false).removeWorker(this);
-        }
+    //     protected override void myOnLastSubscriber() {
+    //         MasterWorker.getInstance(false).removeWorker(this);
+    //     }
 
-        public void doWork() {
-            _read_mogodb();
-        }
+    //     public void doWork() {
+    //         _read_mogodb();
+    //     }
 
-        private void _read_mogodb() {
-            var timer = Stopwatch.StartNew();
-            // Access the collection
-            IMongoCollection<BsonDocument> collection = _database.GetCollection<BsonDocument>(name: _collection_name);
+    //     private void _read_mogodb() {
+    //         var timer = Stopwatch.StartNew();
+    //         // Access the collection
+    //         IMongoCollection<BsonDocument> collection = _database.GetCollection<BsonDocument>(name: _collection_name);
 
-            // Fetch all documents from the collection
-            var documents = collection.Find(Builders<BsonDocument>.Filter.Empty).ToCursor();
-            _table = _table == null ? null : new BlueTable(_table.schema);
-            // Iterate through the documents and print values
-            foreach (var document in documents.ToEnumerable()) {
-                bts.Log.Gen.info($"document: {document} json: {document.ToJson()}");
-                if (_table == null) {
-                    _init_table(document);
-                }
-                _add_row(document);
-            }
-            timer.Stop();
-            Log.Gen.info($"MongoDBStream time to read {_collection_name} {timer.ElapsedMilliseconds}ms");
-            timer.Restart();
-            var dp = BlueDataUtility.build_dp_inc(_table, myLastValue == null ? null : myLastValue as DataPacket, true);
-            timer.Stop();
-            Log.Gen.info($"MongoDBStream time to compare tables :{timer.ElapsedMilliseconds}ms");
-            if (dp.addedRows.Count == 0 && dp.deletedRows.Count == 0)
-                return;
-            myFireEvent(dp);
-        }
+    //         // Fetch all documents from the collection
+    //         var documents = collection.Find(Builders<BsonDocument>.Filter.Empty).ToCursor();
+    //         _table = _table == null ? null : new BlueTable(_table.schema);
+    //         // Iterate through the documents and print values
+    //         foreach (var document in documents.ToEnumerable()) {
+    //             bts.Log.Gen.info($"document: {document} json: {document.ToJson()}");
+    //             if (_table == null) {
+    //                 _init_table(document);
+    //             }
+    //             _add_row(document);
+    //         }
+    //         timer.Stop();
+    //         Log.Gen.info($"MongoDBStream time to read {_collection_name} {timer.ElapsedMilliseconds}ms");
+    //         timer.Restart();
+    //         var dp = BlueDataUtility.build_dp_inc(_table, myLastValue == null ? null : myLastValue as DataPacket, true);
+    //         timer.Stop();
+    //         Log.Gen.info($"MongoDBStream time to compare tables :{timer.ElapsedMilliseconds}ms");
+    //         if (dp.addedRows.Count == 0 && dp.deletedRows.Count == 0)
+    //             return;
+    //         myFireEvent(dp);
+    //     }
 
-        private void _init_table(BsonDocument doc_row) {
-            bts.Log.assert(_table == null);
-            List<ColumnInfo> cis = new List<ColumnInfo>();
-            cis.Add((new ColumnInfo("id", 0, typeof(Int32))));
-            foreach (var n in doc_row.Names) {
-                cis.Add(new ColumnInfo(n, cis.Count, typeof(string)));
-            }
-            IBlueTableSchema schema = new BlueTableSchema(cis.ToArray(), _collection_name);
-            _table = new BlueTable(schema);
-        }
+    //     private void _init_table(BsonDocument doc_row) {
+    //         bts.Log.assert(_table == null);
+    //         List<ColumnInfo> cis = new List<ColumnInfo>();
+    //         cis.Add((new ColumnInfo("id", 0, typeof(Int32))));
+    //         foreach (var n in doc_row.Names) {
+    //             cis.Add(new ColumnInfo(n, cis.Count, typeof(string)));
+    //         }
+    //         IBlueTableSchema schema = new BlueTableSchema(cis.ToArray(), _collection_name);
+    //         _table = new BlueTable(schema);
+    //     }
 
-        private void _add_row(BsonDocument doc) {
-            var key_val = doc["_id"];
-            var key = key_val.ToString();
-            int id;
-            if (!_id_map.TryGetValue(key, out id)) {
-                id = _id_map.Count;
-                _id_map.Add(key, id);
-            }
-            MutableBlueRow nr = new MutableBlueRow(_table.schema);
-            nr.setValue("id", id);
-            foreach (var n in doc.Names) {
-                var val = doc[n].ToString();
-                nr.setValue(n, val);
-            }
-            _table.addRow(nr);
-        }
-    }
+    //     private void _add_row(BsonDocument doc) {
+    //         var key_val = doc["_id"];
+    //         var key = key_val.ToString();
+    //         int id;
+    //         if (!_id_map.TryGetValue(key, out id)) {
+    //             id = _id_map.Count;
+    //             _id_map.Add(key, id);
+    //         }
+    //         MutableBlueRow nr = new MutableBlueRow(_table.schema);
+    //         nr.setValue("id", id);
+    //         foreach (var n in doc.Names) {
+    //             var val = doc[n].ToString();
+    //             nr.setValue(n, val);
+    //         }
+    //         _table.addRow(nr);
+    //     }
+    // }
 
     [DataStreamUserRespresentation("mongodb")]
     public class MongoDBStream : ViewStream, IWorker {
         private ViewStreamDatabase _vsd;
         private readonly string _connection_string, _db_name, _collection_name;
+        private readonly TimeSpan _poll_period;
         private MongoClient _client;
         private BlueTable _table;
+        private SyncQueue<DataPacket> _table_queue = new SyncQueue<DataPacket>();
+        private BlockingQueue _work_queue = new BlockingQueue();
 
         // Access the database
         private IMongoDatabase _database;
         private IDictionary<string, int> _id_map = new Dictionary<string, int>();
         // mongodb://192.168.1.80:27017
 
-        public MongoDBStream(string connection_string, string db_name, string collection_name)
-        : this(connection_string, db_name, collection_name, ViewStreamDatabase.instance) {
+        public MongoDBStream(string connection_string, string db_name, string collection_name, string poll_period)
+        : this(connection_string, db_name, collection_name, TimeBasedFilter.parse_duration(poll_period), ViewStreamDatabase.instance) {
 
         }
-        public MongoDBStream(string connection_string, string db_name, string collection_name, ViewStreamDatabase vsd) {
+        public MongoDBStream(string connection_string, string db_name, string collection_name, TimeSpan poll_period, ViewStreamDatabase vsd) {
             bts.Log.Gen.info($"MongoDBStream constructed connection_string: {connection_string} " +
                                     $"db_name: {db_name}");
             _connection_string = connection_string;
@@ -130,19 +135,30 @@ namespace bts {
             _collection_name = collection_name;
             _vsd = vsd;
             _client = new MongoClient(_connection_string);
+            _poll_period = poll_period;
             _database = _client.GetDatabase(_db_name);
         }
 
         protected override void myOnFirstSubscriber() {
-            MasterWorker.getInstance(false).addWorker(this, TimeSpan.FromSeconds(10));
+            _vsd.addWorker(this);
+            //MasterWorker.getInstance(false).addWorker(this, _poll_period);
         }
 
         protected override void myOnLastSubscriber() {
-            MasterWorker.getInstance(false).removeWorker(this);
+            _vsd.removeWorker(this);
+            //MasterWorker.getInstance(false).removeWorker(this);
         }
 
+        private DateTime _last_poll_time = DateTime.MinValue;
         public void doWork() {
-            _read_mogodb();
+            var now = DateTime.Now;
+            if (now - _last_poll_time > _poll_period) {
+                ThreadPool.QueueUserWorkItem(_ => _read_mogodb());
+                _last_poll_time = DateTime.Now;
+            }
+            var dp = _table_queue.dequeue(0);
+            if (dp != null)
+                myFireEvent(dp);
         }
 
         private void _read_mogodb() {
@@ -166,10 +182,11 @@ namespace bts {
             timer.Restart();
             var dp = BlueDataUtility.build_dp_inc(_table, myLastValue == null ? null : myLastValue as DataPacket, true);
             timer.Stop();
-            Log.Gen.info($"MongoDBStream time to compare tables: {timer.ElapsedMilliseconds}ms");;
+            Log.Gen.info($"MongoDBStream time to compare tables: {timer.ElapsedMilliseconds}ms"); ;
             if (dp.addedRows.Count == 0 && dp.deletedRows.Count == 0)
                 return;
-            myFireEvent(dp);
+            _table_queue.enqueue(dp);
+            //myFireEvent(dp);
         }
 
         private void _init_table(BsonDocument doc_row) {
@@ -208,46 +225,46 @@ namespace bts {
         }
     }
 
-    [DataStreamUserRespresentation("mongodb_flat")]
-    public class MongoDBFlatStream : ViewStream {
-        public MongoDBFlatStream(string connection_string, string db_name, string collection_name, string timestamp_col, string ttl_duration)
-        :
-        this(ViewStreamDatabase.instance, connection_string, db_name, collection_name, timestamp_col, TimeBasedFilter.parse_duration(ttl_duration)) { }
+    // [DataStreamUserRespresentation("mongodb_flat")]
+    // public class MongoDBFlatStream : ViewStream {
+    //     public MongoDBFlatStream(string connection_string, string db_name, string collection_name, string timestamp_col, string ttl_duration)
+    //     :
+    //     this(ViewStreamDatabase.instance, connection_string, db_name, collection_name, timestamp_col, TimeBasedFilter.parse_duration(ttl_duration)) { }
 
-        public MongoDBFlatStream(ViewStreamDatabase vsd, string connection_string, string db_name, string collection_name, string timestamp_col, TimeSpan ttl_duration) {
-            ViewStream rs = new MongoDBStream(connection_string, db_name, collection_name, vsd);
-            rs = new KeyValueToRelationalStream(rs, "doc_id", "key", "val");
-            rs = new DateStandardizationStream(rs, timestamp_col, DateConversion.ISO8601);
-            //rs = new TimeBasedFilterStream(vsd, rs, timestamp_col, ttl_duration);
-            mySetRealSource(rs);
-        }
-    }
+    //     public MongoDBFlatStream(ViewStreamDatabase vsd, string connection_string, string db_name, string collection_name, string timestamp_col, TimeSpan ttl_duration) {
+    //         ViewStream rs = new MongoDBStream(connection_string, db_name, collection_name, vsd);
+    //         rs = new KeyValueToRelationalStream(rs, "doc_id", "key", "val");
+    //         rs = new DateStandardizationStream(rs, timestamp_col, DateConversion.ISO8601);
+    //         //rs = new TimeBasedFilterStream(vsd, rs, timestamp_col, ttl_duration);
+    //         mySetRealSource(rs);
+    //     }
+    // }
 
 
 
-    [DataStreamUserRespresentation("newsreplay")]
-    public class NewsReplayStream : ViewStream {
-        public NewsReplayStream(string connection_string, string db_name, string collection_name,
-                                int buffer_sz, string fire_period) :
-        this(ViewStreamDatabase.instance, connection_string, db_name, collection_name, buffer_sz, TimeBasedFilter.parse_duration(fire_period)) { }
+    // [DataStreamUserRespresentation("newsreplay")]
+    // public class NewsReplayStream : ViewStream {
+    //     public NewsReplayStream(string connection_string, string db_name, string collection_name,
+    //                             int buffer_sz, string fire_period) :
+    //     this(ViewStreamDatabase.instance, connection_string, db_name, collection_name, buffer_sz, TimeBasedFilter.parse_duration(fire_period)) { }
 
-        public NewsReplayStream(ViewStreamDatabase vsd, string connection_string, string db_name, string collection_name,
-                                int buffer_sz, TimeSpan fire_period) {
-            ViewStream r;
+    //     public NewsReplayStream(ViewStreamDatabase vsd, string connection_string, string db_name, string collection_name,
+    //                             int buffer_sz, TimeSpan fire_period) {
+    //         ViewStream r;
 
-            r = new MongoDBStream(connection_string, db_name, collection_name, vsd);
-            r = new KeyValueToRelationalStream(r,
-                                                "doc_id",
-                                                "key",
-                                                "value");
-            r = new DateStandardizationStream(r,
-                                               "created_at",
-                                               "ISO8601");
-            r = new SnapshotReplayStream(vsd, r, "created_at", buffer_sz, fire_period);
+    //         r = new MongoDBStream(connection_string, db_name, collection_name, vsd);
+    //         r = new KeyValueToRelationalStream(r,
+    //                                             "doc_id",
+    //                                             "key",
+    //                                             "value");
+    //         r = new DateStandardizationStream(r,
+    //                                            "created_at",
+    //                                            "ISO8601");
+    //         r = new SnapshotReplayStream(vsd, r, "created_at", buffer_sz, fire_period);
 
-            mySetRealSource(r);
-        }
-    }
+    //         mySetRealSource(r);
+    //     }
+    // }
 
 
 }

src/DataStreamSuite/DataStream/Primitives/Project/New/NewProjectStream.cs

@@ -6,6 +6,7 @@ namespace bts {
    using System;
    using System.Collections;
    using System.Collections.Generic;
+   using System.Linq;
 
    public class ProjectData {
       // schema from the base stream
@@ -147,12 +148,23 @@ namespace bts {
                mapped_to_orig_map.Add(mapped_col, orig_col);
                orig_to_mapped.Add(orig_col, mapped_col);
             } else {
-               Log.Gen.error($"NewProjectStream unexpecetd token: {p_tok}");
+               if (input_schema.columnNames.Contains(p_tok)) {
+                  string orig_col = p_tok;
+                  string mapped_col = orig_col;
+                  orig_to_mapped.Add(orig_col, mapped_col);
+               }
             }
          }
          // make the new schema
          List<ColumnInfo> new_cols = new List<ColumnInfo>();
          foreach (ColumnInfo ci in input_schema.columnInfo) {
+            if (ci.colName == "id") {
+               // if there's a hidden col, other columns will show by default, but if cols are being
+               // projected epxlicitely, than others wil be hidden
+               ColumnInfo nc = new ColumnInfo(ci.colName, new_cols.Count, ci.colType);
+               new_cols.Add(nc);
+               continue;
+            }
             if (removed_cols.Contains(ci.colName)) {
                continue;
             } else if (orig_to_mapped.ContainsKey(ci.colName)) {
@@ -160,8 +172,12 @@ namespace bts {
                ColumnInfo nc = new ColumnInfo(mapped_name, new_cols.Count, ci.colType);
                new_cols.Add(nc);
             } else {
-               ColumnInfo nc = new ColumnInfo(ci.colName, new_cols.Count, ci.colType);
-               new_cols.Add(nc);
+               if (ci.colName == "id" || removed_cols.Count > 0) {
+                  // if there's a hidden col, other columns will show by default, but if cols are being
+                  // projected epxlicitely, than others wil be hidden
+                  ColumnInfo nc = new ColumnInfo(ci.colName, new_cols.Count, ci.colType);
+                  new_cols.Add(nc);
+               }
             }
          }
          IBlueTableSchema new_schema = new BlueTableSchema(input_schema.tableName, new_cols.ToArray());

src/DataStreamSuite/DataStream/Primitives/SideEffectStreams/SchemaGuesserStream.cs

@@ -24,7 +24,6 @@ namespace bts {
             var dp_out = BlueDataUtility.build_dp_inc(_rows, myLastValue as DataPacket);
             myFireEvent(dp_out);
 
-
         }
 
 

src/DataStreamSuite/DataStream/Primitives/Time/MillisecondStream.cs

@@ -1,29 +1,71 @@
 using System;
 
-namespace BCG
-{
-   public class MSecondStream : TimeUnitStream 
-   {
-      public MSecondStream (int intervalInms, int size)
-         : base (DateTime.Now, new TimeSpan (0, 0, 0, 0, intervalInms), size, ViewStreamDatabase.instance)
-      {}
+using BCG.Utilities.General;
 
-      protected override DateTime myRoundTime (DateTime dt) {return dt;}
+using bts;
+
+namespace BCG {
+   [DataStreamUserRespresentation("milliseconds")]
+   public class MillisecondStream : ViewStream, IWorker {
+      private DateTime _last_fire_time = DateTime.MinValue;
+      private readonly TimeSpan _period;
+      private readonly int _size;
+      private readonly ViewStreamDatabase _vsd;
+      private IBlueTableSchema _schema;
+      private bts.CircularBufferTable _table;
+
+      public MillisecondStream(int interval_msecs, int size)
+      : this(ViewStreamDatabase.instance, interval_msecs, size) {
+      }
+
+      public MillisecondStream(ViewStreamDatabase vsd, int interval_msecs, int size) {
+         _vsd = vsd;
+         _period = TimeSpan.FromMilliseconds(interval_msecs);
+         _size = size;
+         _schema = BlueDataUtility.form_schema("Milliseconds",
+               Tuple.Create("id", TypeCode.Int32),
+               Tuple.Create("time", TypeCode.DateTime),
+               Tuple.Create("display_time", TypeCode.String));
+         _table = new CircularBufferTable(_schema, _size);
+      }
+
+      protected override void myOnFirstSubscriber() {
+         _vsd.addWorker(this);
+      }
+
+      protected override void myOnLastSubscriber() {
+         _vsd.removeWorker(this);
+      }
+
+      private int _id = 0;
+      public void doWork() {
+         if (DateTime.Now - _last_fire_time >= _period) {
+            _last_fire_time = DateTime.Now;
+            IBlueRow nr = new BlueRow(_schema, _id++, _last_fire_time,
+            _last_fire_time.ToString("yyyy-MM-dd HH:mm:ss.fff"));
+            _table.add_row(nr);
+            var dp = BlueDataUtility.build_dp_inc(_table, myLastValue as DataPacket);
+            myFireEvent(dp);
+         }
+      }
+   }
+   public class MSecondStream : TimeUnitStream {
+      public MSecondStream(int intervalInms, int size)
+         : base(DateTime.Now, new TimeSpan(0, 0, 0, 0, intervalInms), size, ViewStreamDatabase.instance) { }
+
+      protected override DateTime myRoundTime(DateTime dt) { return dt; }
    }
 
-   public class DisplayableMillisecondStream : ViewStream, IStreamMapper
-   {
-      private IBlueTableSchema mySchema = new BlueTableSchema ("ms",
-                                                               new ColumnInfo ("id", 0, typeof(int)),
-                                                               new ColumnInfo ("time", 1, typeof(string)));
-      public DisplayableMillisecondStream (int intervalInMs, int sz)
-      {
-         mySetRealSource (new MapStream (new MSecondStream (intervalInMs, sz), this));
+   public class DisplayableMillisecondStream : ViewStream, IStreamMapper {
+      private IBlueTableSchema mySchema = new BlueTableSchema("ms",
+                                                               new ColumnInfo("id", 0, typeof(int)),
+                                                               new ColumnInfo("time", 1, typeof(string)));
+      public DisplayableMillisecondStream(int intervalInMs, int sz) {
+         mySetRealSource(new MapStream(new MSecondStream(intervalInMs, sz), this));
       }
       #region IStreamMapper Members
 
-      public IBlueTableSchema getOutputSchema(IBlueTableSchema inputSchema)
-      {
+      public IBlueTableSchema getOutputSchema(IBlueTableSchema inputSchema) {
          return mySchema;
       }
 
@@ -31,15 +73,15 @@ namespace BCG
 
       #region IDynamicStreamMapper Members
 
-      public IBlueRow map (IBlueRow row)
-      {
-         return new BlueRow (mySchema, row["id"], myToString ((DateTime)row["time"]));
+      public IBlueRow map(IBlueRow row) {
+         DateTime time = row.get_datetime("time");
+         var time_txt = time.ToString("yyyy-MM-dd HH:mm:ss.fff");
+         return new BlueRow(mySchema, row["id"], time_txt);
       }
 
       #endregion
 
-      private string myToString (DateTime dt)
-      {
+      private string myToString(DateTime dt) {
          string sep = ":";
          return dt.Hour + sep + dt.Minute + sep + dt.Second + sep + dt.Millisecond;
       }

src/DataStreamSuite/DataStreamCore/BlueData/BlueDataUtility.cs

@@ -1628,6 +1628,13 @@ namespace BCG {
          return sb.ToString();
       }
 
+      public static string id_string(IBlueRowCollection table) {
+         string rv = String.Empty;
+         foreach (IBlueRow r in table)
+            rv += $"{r["id"]} ";
+         return rv;
+      }
+
       public static IDictionary<int, List<IBlueRow>> group_together(IBlueRowCollection rows, string[] group_by_col_names) {
          BlueRowHasher hasher = new BlueRowHasher(group_by_col_names.Length);
          return group_together(rows, BlueDataUtility.get_column_indices(rows.schema, group_by_col_names), ref hasher);
@@ -1776,6 +1783,18 @@ namespace BCG {
          #endregion
       }
 
+      public static IBlueRow find_max(IEnumerable<IBlueRow> rows, params string[] cols_to_compare) {
+         if (rows == null || rows.Count() == 0)
+            return null;
+         IBlueRow max_row = rows.ElementAt(0);
+         foreach (IBlueRow r in rows) {
+            int c = compare_rows(r, max_row, cols_to_compare);
+            if (c > 0)
+               max_row = r;
+         }
+         return max_row;
+      }
+
       public static int compare_rows(IBlueRow r1, IBlueRow r2, Dictionary<string, bool> sort_map) {
          foreach (KeyValuePair<string, bool> p in sort_map) {
             int comp = compare_send_dontcare_to_last(r1, p.Key, r2, p.Key, p.Value);

src/DataStreamSuite/DataStreamCore/BlueData/CircularBufferTable.cs

@@ -16,12 +16,6 @@ namespace bts {
          _buffer_sz = buffer_sz;
       }
 
-      // public OrderedBlueTable(List<IBlueRow> rows, int buffer_sz) {
-      //    _buffer_sz = buffer_sz;
-      //    foreach (IBlueRow r in rows)
-      //       add_row(r);
-      // }
-
       // returns the row that might need to be deleted to keep
       // the circular buffer intact
       public IBlueRow add_row(IBlueRow row) {

src/DataStreamSuite/DataStreamCore/Messaging/Publish/PubTableBroker.cs

@@ -51,13 +51,13 @@ namespace BCG {
       public virtual void make_batch_changes(IBlueRowCollection new_rows,
                                              IBlueRowCollection del_rows,
                                              IBlueRowCollection added_rows) {
-         string log_msg = $"PubTableBroker.make_batch_changes table: {new_rows.schema.tableName} del: {del_rows.Count} added: {added_rows.Count}";
-         foreach (IBlueRow r in del_rows)
-            log_msg += $"id: {r["id"]} ";
-         log_msg += "addedRows: ";
-         foreach (IBlueRow r in added_rows)
-            log_msg += $"id: {r["id"]}";
-         bts.Log.Gen.info(log_msg);
+         // string log_msg = $"PubTableBroker.make_batch_changes table: {new_rows.schema.tableName} del: {del_rows.Count} added: {added_rows.Count}";
+         // foreach (IBlueRow r in del_rows)
+         //    log_msg += $"id: {r["id"]} ";
+         // log_msg += "addedRows: ";
+         // foreach (IBlueRow r in added_rows)
+         //    log_msg += $"id: {r["id"]}";
+         // bts.Log.Gen.info(log_msg);
          SinglePubTableBroker table_publisher = null;
          var name_key = new_rows.schema.tableName.ToLower();
          if (!_publisher_map.TryGetValue(name_key, out table_publisher))

src/DataStreamSuite/DataStreamCore/Messaging/Subscription/TableAccumulator2.cs

@@ -5,18 +5,21 @@ using System.Linq;
 
 using bts;
 
-namespace BCG {
+namespace BCG
+{
    // This class is responsible for mantaining the state fo a table. It gets the
    // bytes from the pubsubserver via the handle_updates call (called by VSD), deserializes
    // updates the state of the table, and fires the TableUpdateDelegate.
-   public class TableAccumulator2 : ITableAccumulator {
+   public class TableAccumulator2 : ITableAccumulator
+   {
       private readonly string _tag;
       private readonly ISubscriberCore _subscriber_core;
       IStreamMsgDeserilizer _deser = new StreamIODeserializationController();
       private BlueTable _table;
       private TableUpdateDelegate _on_update_delegate;
 
-      public TableAccumulator2(string tag, ISubscriberCore sub_core) {
+      public TableAccumulator2(string tag, ISubscriberCore sub_core)
+      {
          _tag = tag;
          _subscriber_core = sub_core;
       }
@@ -25,10 +28,13 @@ namespace BCG {
 
       //public bool is_active() { return _on_update_delegate != null; }
 
-      public event TableUpdateDelegate on_update {
-         add {
+      public event TableUpdateDelegate on_update
+      {
+         add
+         {
             Log.Gen.info("TableAccumulator2.on_update.add {_tag} subscribed");
-            if (_on_update_delegate == null) {
+            if (_on_update_delegate == null)
+            {
                //ie first subscription
                Log.Gen.info("TableAccumulator2.on_update {_tag} subscribed for the first time");
                _subscriber_core.subscribe(_tag);
@@ -38,14 +44,17 @@ namespace BCG {
             if (_table != null)
                value(_table, _table, new BlueTable(_table.schema));
          }
-         remove {
+         remove
+         {
             Log.Gen.info("TableAccumulator2.on_update.remove {_tag} unsubscribed");
-            if (_on_update_delegate != null) {
+            if (_on_update_delegate != null)
+            {
                _on_update_delegate =
                   Delegate.Remove(_on_update_delegate, value) as TableUpdateDelegate;
                int n_subs = _on_update_delegate == null ? 0 : _on_update_delegate.GetInvocationList().Length;
                Log.Gen.info("TableAccumulator2.on_update.remove {_tag} unsubscribed n_subs: {n_subs}");
-               if (_on_update_delegate == null) {
+               if (_on_update_delegate == null)
+               {
                   _subscriber_core.unsubscribe(_tag);
                   // TODO
                   //_table = null;
@@ -54,17 +63,20 @@ namespace BCG {
 
          }
       }
-
-      public void handle_updates(IEnumerable<byte[]> msgs) {
+      private int _update_no = 0;
+      public void handle_updates(IEnumerable<byte[]> msgs)
+      {
          if (msgs == null || msgs.Count() == 0)
             return;
-         Log.Gen.info($"TableAccumulator2.handle_updates msgs: {msgs.Count()} msg_0: {msgs.ElementAt(0).Length}");
+         Log.Gen.info($"TableAccumulator2.handle_updates _update_no: {_update_no++} msgs: {msgs.Count()} msg_0: {msgs.ElementAt(0).Length}");
          IBlueRowCollection table_0 = _table == null ? null : new BlueTable(_table);
-         foreach (var msg in msgs) {
+         foreach (var msg in msgs)
+         {
             bts.StreamIOMessage str_msg = _deser.deserialize(msg);
-            var state_txt = _table == null ? "NULL" : _table.Count.ToString();
+            var state_txt = _table == null ? "NULL" : _table.Count.ToString();// BlueDataUtility.id_string(_table);
             Log.Gen.info($"TableAccumulator2.handle_update tag: {_tag} sz: {msg.Length} str_msg: {str_msg} _state: {state_txt}");
-            switch (str_msg.op) {
+            switch (str_msg.op)
+            {
                case bts.StreamIOMessage.StreamOp.CREATE:
                   _table = new BlueTable(str_msg.create_message.rows);
                   break;
@@ -77,11 +89,13 @@ namespace BCG {
          // this could be made faster if we just do this on the CREATE  message, and pass-through
          // what we get on the MULTIPLE_REMOVE_ADD
          IBlueRowCollection d_rows, a_rows;
-         BlueDataUtility.compute_differences_id_based(_table, table_0, out d_rows, out a_rows);
+         //BlueDataUtility.compute_differences_id_based(_table, table_0, out d_rows, out a_rows);
+         BlueDataUtility.compute_differences_fast(_table, table_0, out d_rows, out a_rows);
          _on_update_delegate(_table, a_rows, d_rows);
       }
 
-      public void update(Stream[] streams) {
+      public void update(Stream[] streams)
+      {
       }
 
       #endregion

src/DataStreamSuite/DataStreamCore/Messaging/v2/Publish/Serialization/StreamSerializationUtils.cs

@@ -95,8 +95,8 @@ namespace bts {
       }
 
       public static IBlueTableSchema deserialize_schema(ref BinaryReader br) {
-         var hex_txt = to_hex(br.BaseStream);
-         bts.Log.Gen.info($"StreamSerializationUtils.deserialize_schema bytes: {hex_txt}");
+         // var hex_txt = to_hex(br.BaseStream);
+         // bts.Log.Gen.info($"StreamSerializationUtils.deserialize_schema bytes: {hex_txt}");
          string table_name = br.read_string_cpp();
          int n_cols = br.ReadInt32(); // doing a br.Read was only advancing the data 1 byte!!
          List<ColumnInfo> cis = new List<ColumnInfo>();
@@ -137,8 +137,8 @@ namespace bts {
 
       public static ColumnInfo deserialize_column_info(ref BinaryReader br) {
          //col_no, name, type>
-         var hex_txt = to_hex(br.BaseStream);
-         bts.Log.Gen.info($"StreamSerializationUtils.deserialize_column_info bytes: {hex_txt}");
+         // var hex_txt = to_hex(br.BaseStream);
+         // bts.Log.Gen.info($"StreamSerializationUtils.deserialize_column_info bytes: {hex_txt}");
          int col_no = br.ReadInt32();
          string col_name = br.read_string_cpp();
          int tc_int = br.ReadInt32();
@@ -212,5 +212,18 @@ namespace bts {
          return rv;
       }
 
+      public static string to_hex(byte[] byte_arr, int start_pos, int end_pos) {
+         var len = end_pos - start_pos;
+         StringBuilder hex_builder = new StringBuilder(len * 2);
+         int i = 0;
+
+         foreach (byte b in byte_arr) {
+            if (i >= start_pos && i < end_pos)
+               hex_builder.Append($"{b:X2} ");
+            i++;
+         }
+         return hex_builder.ToString().Trim();
+      }
+
    }
 }

src/DataStreamSuite/DataStreamCore/Messaging/v2/Receive/PubSubServer/PubsubServerStreamTagHandler.cs

@@ -7,12 +7,23 @@ namespace bts {
 
    // for non stream tags
    public class PubsubServerGenericDataHandler : IPubSubServerTagHandler {
-      byte[] _state = null;
-      public void update(string tag, byte[] msg) {
-         _state = msg;
+      //byte[] _state = null;
+      private MemoryStream _str = new MemoryStream();
+      private BinaryWriter _bw;
+      public PubsubServerGenericDataHandler() {
+         _bw = new BinaryWriter(_str);
       }
+      public void update(string tag, byte[] msg) {
+         _str.Position = 0;
+         _bw.Write(msg);
+         Log.Gen.info($"PubsubServerGenericDataHandler.update tag: {tag} msg_len: {msg.Length}");
+      }
+
+
       public byte[] get_content(string tag) {
-         return _state;
+         var rv = _str.ToArray();
+         Log.Gen.info($"PubsubServerGenericDataHandler.get_content tag: {tag} rv: {rv.Length}");
+         return rv;
       }
       public string[] supportedTagTypes { get; }
       public void register(InternalPublishDelegate callback) { }
@@ -33,7 +44,7 @@ namespace bts {
       public void update(string tag, byte[] msg) {
          StreamIOMessage str_msg = _deser.deserialize(msg);
          var state_txt = _state == null ? "NULL" : _state.Count.ToString();
-         Log.Gen.info($"PubsubServerSingleStreamTagHandler.update (pre.1) tag: {tag} sz: {msg.Length} str_msg: {str_msg} _state: {state_txt}");
+         Log.Gen.info($"PubsubServerSingleStreamTagHandler.update (pre.1) tag: {tag} sz: {msg.Length} _state: {state_txt}");
          switch (str_msg.op) {
             case StreamIOMessage.StreamOp.CREATE:
                //Log.assert(_state == null);
@@ -45,7 +56,7 @@ namespace bts {
                _state.addRows(str_msg.multiple_remove_add_message.added_rows);
                break;
          }
-         Log.Gen.info($"PubsubServerSingleStreamTagHandler.update (post) tag: {tag} sz: {msg.Length} str_msg: {str_msg} state: {_state.Count}");
+         Log.Gen.info($"PubsubServerSingleStreamTagHandler.update (post) tag: {tag} sz: {msg.Length}  state: {_state.Count}");
       }
 
       public byte[] get_content(string tag) {
@@ -60,7 +71,7 @@ namespace bts {
          };
          var rv = get_content_();
          var rv_txt = rv == null ? "null" : rv.Length.ToString();
-         Log.Gen.info($"PubsubServerStreamTagHandler.get_content rv: {rv_txt}");
+         Log.Gen.info($"PubsubServerSingleStreamTagHandler.get_content rv: {rv_txt}");
          return rv;
       }
       public string[] supportedTagTypes {

src/DataStreamSuite/DataStreamPublisher/Configurations/MarketDataPublisher.json

@@ -1,6 +1,6 @@
 {
   "user": "bts",
-  "sparkd": "http://localhost:8000/config",
+  "sparkd": "http://192.168.1.80:8000/config",
   "pubsub": {
     "milliseconds_between_sub_info_updates": 1000,
     "milliseconds_between_time_slice_updates": 5000,
@@ -18,14 +18,14 @@
     "publisher_routes": [
       {
         "name": "default",
-        "server_address": "bluto://localhost:12345",
+        "server_address": "bts-cloud://127.0.0.1:12345",
         "tag": ".*"
       }
     ],
     "subscriber_routes": [
       {
         "name": "default",
-        "server_address": "bluto://localhost:12345",
+        "server_address": "bts-cloud://127.0.0.1:12345",
         "tag": ".*"
       }
     ],
@@ -89,8 +89,8 @@
       "name": "test_data",
       "streams": [
         {
-          "name": "seconds_1_10",
-          "expression": "second(1,10)",
+          "name": "milliseconds_1000_100",
+          "expression": "milliseconds(1000,100)",
           "is_on_websocket": true
         },
         {
@@ -105,5 +105,25 @@
         }
       ]
     }
+    // {
+    //   "name": "news_data",
+    //   "streams": [
+    //     {
+    //       "name": "benzinga_news",
+    //       "expression": "benzingaNews(\"mongodb://localhost:27017\",\"local\",\"news_headlines\", \"30s\", \"5d\")",
+    //       "is_on_websocket": true
+    //     },
+    //     {
+    //       "name": "benzinga_news_symbols",
+    //       "expression": "benzingaNewsSymbols(benzinga_news)",
+    //       "is_on_websocket": true
+    //     },
+    //     {
+    //       "name": "benzinga_news_symbol_state",
+    //       "expression": "benzingaNewsSymbolState(benzinga_news)",
+    //       "is_on_websocket": true
+    //     }
+    //   ]
+    // }
   ]
-}
+}

src/DataStreamSuite/DataStreamPublisher/Configurations/TestDataPublisher.json

@@ -6,7 +6,22 @@
       "streams": [
         {
           "name": "seconds_1_10",
-          "expression": "second(1,10)",
+          "expression": "milliseconds(1000,10)",
+          "is_on_websocket": true
+        },
+        {
+          "name": "benzinga_news",
+          "expression": "benzingaNews(\"mongodb://localhost:27017\",\"local\",\"news_headlines\", \"30s\", \"5d\")",
+          "is_on_websocket": true
+        },
+        {
+          "name": "benzinga_news_symbols",
+          "expression": "benzingaNewsSymbols(benzinga_news)",
+          "is_on_websocket": true
+        },
+        {
+          "name": "benzinga_news_symbol_state",
+          "expression": "benzingaNewsSymbolState(benzinga_news)",
           "is_on_websocket": true
         },
         {
@@ -22,4 +37,4 @@
       ]
     }
   ]
-}
+}

src/DataStreamSuite/DataStreamPublisher/DataStreamPublisherConfig.cs

@@ -10,7 +10,8 @@ using System.IO;
 using System.Text.Json;
 using System.Text.Json.Serialization;
 
-public class DataStreamPublisherConfig {
+public class DataStreamPublisherConfig
+{
    /// <summary>
    /// The login user for sparkd
    /// </summary>
@@ -38,9 +39,17 @@ public class DataStreamPublisherConfig {
    [JsonPropertyName("publishers")]
    public List<PublisherNode>? Publishers { get; set; }
 
-   [JsonPropertyName("simple_echo_client")]
-   [Option("simple-echo-client", HelpText = "Run a simple echo client (won't publish tables or connect to sparkd)")]
-   public bool? SimpleEchoClient { get; set; }
+   [JsonPropertyName("mongodb_url")]
+   public string MONGO_DB_URL { get; set; }
+
+   [JsonPropertyName("mongodb_db_name")]
+   public string MONGO_DB_NAME { get; set; }
+
+   [JsonPropertyName("mongodb_db_collection")]
+   public string MONGO_DB_COLLECTION { get; set; }
+
+   [JsonPropertyName("benzinga_news_ttl")]
+   public string BENZINGA_NEWS_TTL { get; set; }
 
    [JsonPropertyName("pubsub")]
    public PubSubConfig? PubSub { get; set; }
@@ -53,18 +62,18 @@ public class DataStreamPublisherConfig {
    public string? ConfigFile { get; set; }
 
    public static DataStreamPublisherConfig Default() =>
-      new() {
-         SimpleEchoClient = false,
+      new()
+      {
          PubSub = PubSubConfig.Default(),
       };
 
-   public void MergeWith(DataStreamPublisherConfig other) {
+   public void MergeWith(DataStreamPublisherConfig other)
+   {
       User ??= other.User;
       Password ??= other.Password;
       Sparkd ??= other.Sparkd;
       Publishers ??= other.Publishers;
       PubSub ??= other.PubSub;
-      SimpleEchoClient ??= other.SimpleEchoClient;
       ConfigFile ??= other.ConfigFile;
    }
 
@@ -75,8 +84,10 @@ public class DataStreamPublisherConfig {
    /// <exception cref="InvalidOperationException">
    /// JsonSerializer.Deserialize returns null
    /// </exception>
-   public void MergeWithConfigFile() {
-      if (ConfigFile is null) {
+   public void MergeWithConfigFile()
+   {
+      if (ConfigFile is null)
+      {
          return;
       }
       var cfgString = File.ReadAllText(ConfigFile);

src/DataStreamSuite/DataStreamPublisher/Program.cs

@@ -23,8 +23,10 @@ using System.Threading.Tasks;
 await Parser.Default.ParseArguments<DataStreamPublisherConfig>(args)
    .MapResult(RunAsync, errs => Task.FromResult(1));
 
-async Task<int> RunAsync(DataStreamPublisherConfig cfg) {
-   try {
+async Task<int> RunAsync(DataStreamPublisherConfig cfg)
+{
+   try
+   {
       cfg.MergeWithConfigFile();
       cfg.MergeWith(DataStreamPublisherConfig.Default());
       PubSubConfig.SetGlobalConfig(Log.notnull(cfg.PubSub));
@@ -32,12 +34,10 @@ async Task<int> RunAsync(DataStreamPublisherConfig cfg) {
       Log.Gen.info("Starting DataStreamPublisher");
       var service_ctx = ServiceContext.create(cfg.Sparkd);
       await service_ctx.login(cfg.User ?? "bts", cfg.Password ?? string.Empty);
-      if (cfg.SimpleEchoClient ?? false) {
-         start_echo_client();
-      } else {
-         _publish_streams(cfg.Publishers ?? new(0));
-      }
-   } catch (Exception ex) {
+      _publish_streams(cfg.Publishers ?? new(0));
+   }
+   catch (Exception ex)
+   {
       // Write the error to console first, in case the logging system is compromised.
       var error_msg = $"DataStreamPublisher threw exception:\n{ex}";
       Console.Error.WriteLine(error_msg);
@@ -48,41 +48,39 @@ async Task<int> RunAsync(DataStreamPublisherConfig cfg) {
    return 0;
 }
 
-static ViewStream? build_stream(string? stream_expression) {
-   if (stream_expression is null) {
+static ViewStream? build_stream(string? stream_expression)
+{
+   // Console.WriteLine("Hit key to continue");
+   // Console.ReadLine();
+   if (stream_expression is null)
+   {
       return null;
    }
-   try {
+   try
+   {
       bts.Log.Gen.info($"build_stream stream_expression: {stream_expression}");
       var exp = DataStreamExpressionsInterpreter.Evaluate(stream_expression);
       var rv = exp.getView(new Dictionary<string, StatementExpression>(0));
       return rv;
 
-   } catch (Exception ex) {
+   }
+   catch (Exception ex)
+   {
       bts.Log.assert(false, $"Could not build stream: {stream_expression} ex: {ex}");
       return null;
    }
 }
 
-static void start_echo_client() {
-   Console.WriteLine("Starting DSPublisher in TCP client echo mode, Hit key to continue");
-   Console.ReadLine();
-   SimpleTCPClient client = new SimpleTCPClient("127.0.0.1", 12345, 1000, 1000);
-   client.start();
-   while (true) {
-      System.Threading.Thread.Sleep(1000);
-   }
-   // await SimpleTCPClient.start("127.0.0.1", 12345);
-   // Task.Delay(1000000000).Wait();
-}
-
-void _publish_streams(IEnumerable<PublisherNode> publishers) {
+void _publish_streams(IEnumerable<PublisherNode> publishers)
+{
    PubTableBroker pb_broker = new PubTableBroker();
    RemoteQueryManager csm = new RemoteQueryManager();
    var csvPublishers = new List<CSVPublisher>();
 
-   foreach (var publisher in publishers) {
-      if (Directory.Exists(publisher.CsvDir)) {
+   foreach (var publisher in publishers)
+   {
+      if (Directory.Exists(publisher.CsvDir))
+      {
          csvPublishers.Add(new(publisher.CsvDir, pb_broker));
       }
    }
@@ -91,7 +89,8 @@ void _publish_streams(IEnumerable<PublisherNode> publishers) {
       .SelectMany(p => p.Streams.empty_if_null().Select(createViewPublisher))
       .ToList();
 
-   while (true) {
+   while (true)
+   {
       System.Threading.Thread.Sleep(TimeSpan.FromSeconds(1));
       ViewStreamDatabase.instance.doWork();
       csm.doWork();
@@ -99,16 +98,19 @@ void _publish_streams(IEnumerable<PublisherNode> publishers) {
       ManagedSubscriber.instance.doWork();
    }
 
-   ViewPublisher? createViewPublisher(PublishedStream ps) {
+   ViewPublisher? createViewPublisher(PublishedStream ps)
+   {
       Log.Gen.info(
          $"Instantiating ViewPublisher for: {JsonSerializer.Serialize(ps)}");
-      if (string.IsNullOrEmpty(ps.Name)) {
+      if (string.IsNullOrEmpty(ps.Name))
+      {
          Log.Gen.warn(
             $"Failed to instantiate ViewPublisher. Null/empty 'name': {JsonSerializer.Serialize(ps)}");
          return null;
       }
       var vs = build_stream(ps.Expression);
-      if (vs is null) {
+      if (vs is null)
+      {
          Log.Gen.warn(
             $"Failed to instantiate ViewPublisher. Invalid 'expression': {JsonSerializer.Serialize(ps)}");
          return null;
@@ -119,31 +121,39 @@ void _publish_streams(IEnumerable<PublisherNode> publishers) {
    }
 }
 
-void on_change(object obj) {
+void on_change(object obj)
+{
    bts.Log.Gen.info($"DSPublisher on_change dp: {obj}");
 }
 
-public class CSVPublisher : IWorker {
+public class CSVPublisher : IWorker
+{
    private IDictionary<string, ViewPublisher> _pubs = new Dictionary<string, ViewPublisher>();
    private readonly string _csv_directory;
    private PubTableBroker _pub_table_broker;
 
-   public CSVPublisher(string csv_directory, PubTableBroker pub_table_broker) {
+   public CSVPublisher(string csv_directory, PubTableBroker pub_table_broker)
+   {
       _csv_directory = csv_directory;
       _pub_table_broker = pub_table_broker;
       MasterWorker.getInstance(false).addWorker(this, TimeSpan.FromSeconds(1));
    }
-   public void doWork() {
+   public void doWork()
+   {
       var file_names = Directory.GetFiles(_csv_directory);
-      foreach (var fn in file_names) {
-         if (!_pubs.ContainsKey(fn)) {
+      foreach (var fn in file_names)
+      {
+         if (!_pubs.ContainsKey(fn))
+         {
             _handle_new_file(fn);
          }
       }
    }
 
-   private void _handle_new_file(string file_name) {
-      if (file_name.EndsWith(".csv")) {
+   private void _handle_new_file(string file_name)
+   {
+      if (file_name.EndsWith(".csv"))
+      {
          ViewStream vs = new CSVReaderStream(file_name);
          vs = new SchemaGuesserStream(vs);
          var toks = file_name.Split('\\');

src/DataStreamSuite/DataStreamQueryAnalyzer/Configurations/DataStreamQueryAnalyzer.json

@@ -1,61 +1,60 @@
-{
-  "user": "bts",
-  "pubsub": {
-    "milliseconds_between_sub_info_updates": 1000,
-    "milliseconds_between_time_slice_updates": 5000,
-    "milliseconds_between_no_buffer_space_retries": 100,
-    "retries_on_no_buffer_space": 1,
-    "do_it_all_seconds_between_logs": 0,
-    "heartbeat_interval": 15000,
-    "missing_heartbeats_before_gc": 50,
-    "use_managed_sockets": true,
-    "state_update_period": 5000,
-    "statistics_update_period": 5000,
-    "disposed_connection_check_period": 1000,
-    "message_processing_warning_threshold": 1000,
-    "server_port": 12345,
-    "publisher_routes": [
-      {
-        "name": "default",
-        "server_address": "bluto://localhost:12345",
-        "tag": ".*"
-      }
-    ],
-    "subscriber_routes": [
-      {
-        "name": "default",
-        "server_address": "bluto://localhost:12345",
-        "tag": ".*"
-      }
-    ],
-    "max_packet_size": 500000000,
-    "blue_comm_must_die": true,
-    "crushable_tags": [],
-    "tag_type_to_assembly_map": [
-      {
-        "handler_class": "BCG.PubSubServerTagHandler",
-        "assembly": "DataStreamCore.dll"
-      }
-    ],
-    "server_queue": "PubSubQueue",
-    "port_range": {
-      "first": 30400,
-      "last": 31700
-    },
-    "bind_to_random_port": false,
-    "debug_tags": "^NOTAGS$",
-    "latency_report_threshold_ms": 1500,
-    "use_substations": false,
-    "use_dynamic_clients": false,
-    "is_substation": false,
-    "internally_publish_sub_info": true,
-    "use_managed_clients_in_native_code": false,
-    "compressed_tags": "^NOTAGS$",
-    "tags_to_report_compression_ratio": "^NOTAGS$",
-    "disable_msmq": false,
-    "use_prepublished_server_state": false,
-    "ip_translation": null,
-    "gc_activation_threshold": 0,
-    "wait_period_after_gc": 1000
-  }
+"user": "bts",
+"pubsub": {
+  "milliseconds_between_sub_info_updates": 1000,
+  "milliseconds_between_time_slice_updates": 5000,
+  "milliseconds_between_no_buffer_space_retries": 100,
+  "retries_on_no_buffer_space": 1,
+  "do_it_all_seconds_between_logs": 0,
+  "heartbeat_interval": 15000,
+  "missing_heartbeats_before_gc": 50,
+  "use_managed_sockets": true,
+  "state_update_period": 5000,
+  "statistics_update_period": 5000,
+  "disposed_connection_check_period": 1000,
+  "message_processing_warning_threshold": 1000,
+  "server_port": 12345,
+  "publisher_routes": [
+    {
+      "name": "default",
+      "server_address": "bts-cloud://192.168.1.80:12345",
+      "tag": ".*"
+    }
+  ],
+  "subscriber_routes": [
+    {
+      "name": "default",
+      "server_address": "bts-cloud://192.168.1.80:12345",
+      "tag": ".*"
+    }
+  ],
+  "max_packet_size": 500000000,
+  "blue_comm_must_die": true,
+  "crushable_tags": [],
+  "tag_type_to_assembly_map": [
+    {
+      "handler_class": "BCG.PubSubServerTagHandler",
+      "assembly": "DataStreamCore.dll"
+    }
+  ],
+  "server_queue": "PubSubQueue",
+  "port_range": {
+    "first": 30400,
+    "last": 31700
+  },
+  "bind_to_random_port": false,
+  "debug_tags": "^NOTAGS$",
+  "latency_report_threshold_ms": 1500,
+  "use_substations": false,
+  "use_dynamic_clients": false,
+  "is_substation": false,
+  "internally_publish_sub_info": true,
+  "use_managed_clients_in_native_code": false,
+  "compressed_tags": "^NOTAGS$",
+  "tags_to_report_compression_ratio": "^NOTAGS$",
+  "disable_msmq": false,
+  "use_prepublished_server_state": false,
+  "ip_translation": null,
+  "gc_activation_threshold": 0,
+  "wait_period_after_gc": 1000
 }
+}

src/DataStreamSuite/DataStreamQueryAnalyzer/MainForm.cs

@@ -25,8 +25,10 @@ using System.Threading.Tasks;
 
 
 
-namespace BCG {
-   public partial class MainForm : System.Windows.Forms.Form {
+namespace BCG
+{
+   public partial class MainForm : System.Windows.Forms.Form
+   {
       // static async Task _run() {
       //    var login = "bts";
       //    var passwd = String.Empty;
@@ -48,7 +50,8 @@ namespace BCG {
       private string myKQ;
       private TextWriter? myConsoleOutput;
 
-      public MainForm() {
+      public MainForm()
+      {
          //Task.Run(async () => await _run());
          InitializeComponent();
          // myKQ = ConfigurationRequester.getApplicationQualifier();         
@@ -61,25 +64,40 @@ namespace BCG {
 
          this.Load += new EventHandler(myLoad);
          this.Closing += new CancelEventHandler(myClosing);
-         myQueryTextBox.TextChanged += (o, e) => {
+         myQueryTextBox.TextChanged += (o, e) =>
+         {
             myQueryTextBox.ForeColor = Color.White;
          };
       }
 
-      public string query_text {
+      public string query_text
+      {
          get { return myQueryTextBox.Text; }
          set { myQueryTextBox.Text = value; }
       }
-      private ViewStream myGetAllTagsStream() {
+      private ViewStream myGetAllTagsStream()
+      {
          // TODO: Port PublishedTagsStream from edge (then we can uncomment impl below)
          //ViewStream vs = new PubSubServerStateStream();
          ViewStream vs = new PubSubServerStateStream();
          return new FilterStream(vs, "tag~='DS_'");
-         //return new LiteralStream("tag/String", "DS_Comedians;DS_Baldies");
+         //return new LiteralStream("tag/String", "DS_table1;DS_table2");
       }
 
-      private void mySetText() {
-         this.Text = "Data Stream Query Analyzer | Site: " + mySite + (string.IsNullOrEmpty(myKQ) ? string.Empty : " | Key Qualifier: " + myKQ);
+      private void mySetText()
+      {
+         ViewStream routes = new PubsubRoutesStream();
+         routes.onChange += (o) => { };
+         var rows = routes.rows;
+         string route_txt = String.Empty;
+         foreach (IBlueRow r in rows)
+         {
+            var tag_regexp = r.get<string>("tag_regexp");
+            var server_url = r.get<string>("server_url");
+            route_txt += $"{tag_regexp} : {server_url},";
+         }
+
+         this.Text = $"Data Stream Query Analyzer  |  routes: {route_txt}";
          myToolStripStatusLabelTime.Text = "Ready";
          myToolStripStatusLabelRows.Text = string.Empty;
       }
@@ -88,10 +106,12 @@ namespace BCG {
       /// Gets the data in the form of a multilevel IDictionary
       /// </summary>
       /// <returns></returns>
-      private IDictionary myGetLibraryData() {
+      private IDictionary myGetLibraryData()
+      {
          SortedList retVal = new SortedList();
          string[] primitiveNames = PrimitiveManager.instance.getPrimitiveNames();
-         foreach (string pName in primitiveNames) {
+         foreach (string pName in primitiveNames)
+         {
             string aName = PrimitiveManager.instance.getAssemblyName(pName);
             ICollection cNames = PrimitiveManager.instance.getConstructorSigns(pName);
             if (!retVal.Contains(aName))
@@ -106,12 +126,15 @@ namespace BCG {
       }
 
 
-      private void myLoadAssemblies() {
+      private void myLoadAssemblies()
+      {
          myTreeView.Nodes.Clear();
          IDictionary libraryData = myGetLibraryData();
-         foreach (DictionaryEntry de in libraryData) {
+         foreach (DictionaryEntry de in libraryData)
+         {
             TreeNode assemblyNode = new TreeNode(de.Key as String);
-            foreach (DictionaryEntry innerDe in (de.Value as IDictionary)) {
+            foreach (DictionaryEntry innerDe in (de.Value as IDictionary))
+            {
                TreeNode primitiveNode = new TreeNode(innerDe.Key as string);
                foreach (string con in (innerDe.Value as ICollection))
                   primitiveNode.Nodes.Add(con);
@@ -131,23 +154,27 @@ namespace BCG {
          }*/
       }
 
-      private void myHandleTableListChange(object o) {
+      private void myHandleTableListChange(object o)
+      {
          IBlueRowCollection rows = (o as DataPacket).newRows;
          myUpdateTableList();
       }
 
 
       #region Persistence Keys
-      public int textBoxHeight {
+      public int textBoxHeight
+      {
          get { return myQueryTextBox.Height; }
          set { myQueryTextBox.Height = value; }
       }
       #endregion
 
-      private void myTryExpression(string expStr) {
+      private void myTryExpression(string expStr)
+      {
          myFreezeView(false); // force unfreeze the view
          DataStreamExpression oldExp = myExp;
-         try {
+         try
+         {
             var exp = DataStreamExpressionsInterpreter.Evaluate(expStr);
 
             if (exp == null)
@@ -173,7 +200,9 @@ namespace BCG {
                oldExp.getView(new Hashtable()).onChange -= new DataStream.ChangeDelegate(myCallback);
                */
             //myQueryTextBox.Text = expStr;
-         } catch (Exception exc) {
+         }
+         catch (Exception exc)
+         {
             Log.Ui.warn(exc.ToString());
 
             myDataGrid.DataSource = null;
@@ -187,7 +216,8 @@ namespace BCG {
          }
       }
 
-      private string myExtractFromClass(TextReader r) {
+      private string myExtractFromClass(TextReader r)
+      {
          string line = r.ReadLine();
          while (line != null && line.IndexOf("*Expression*") == -1)
             line = r.ReadLine();
@@ -200,7 +230,8 @@ namespace BCG {
          string result = "";
          line = r.ReadLine();
 
-         while (line != null && line.IndexOf("----------------------------------------------------------------------") == -1) {
+         while (line != null && line.IndexOf("----------------------------------------------------------------------") == -1)
+         {
             if (line != "")
                result += line.Substring("       ".Length) + "\n";
             line = r.ReadLine();
@@ -211,17 +242,24 @@ namespace BCG {
          return result;
       }
 
-      public void loadFile(string fileName) {
-         try {
+      public void loadFile(string fileName)
+      {
+         try
+         {
             TextReader r = new StreamReader(fileName);
-            try {
+            try
+            {
                myQueryTextBox.Text = myExtractFromClass(r);
-            } finally {
+            }
+            finally
+            {
                r.Close();
             }
             myLastFileName = fileName;
             this.Text = fileName;
-         } catch (Exception exc) {
+         }
+         catch (Exception exc)
+         {
             Log.Ui.warn(exc.ToString());
 
             myDataGrid.DataSource = null;
@@ -232,7 +270,8 @@ namespace BCG {
 
       }
 
-      private void myPrintCode(IBlueTableSchema schema, ICollection vars, string code) {
+      private void myPrintCode(IBlueTableSchema schema, ICollection vars, string code)
+      {
 
          myCSharpText.Text = "using System;\n\nnamespace BCG\n{";
 
@@ -252,7 +291,8 @@ namespace BCG {
 
          string lastVar = null;
          string variableDeclarations = "";
-         foreach (string var in vars) {
+         foreach (string var in vars)
+         {
             variableDeclarations += "         ViewStream " + var + ";\n";
             lastVar = var;
          }
@@ -282,13 +322,16 @@ namespace BCG {
          myCSharpText.Text += "}";
       }
 
-      static DataGridViewColumn _column_factory(ColumnInfo ci) {
-         return new DataGridViewTextBoxColumn {
+      static DataGridViewColumn _column_factory(ColumnInfo ci)
+      {
+         return new DataGridViewTextBoxColumn
+         {
             HeaderText = ci.colName,
             Width = get_col_width(ci)
          };
 
-         static int get_col_width(ColumnInfo ci) {
+         static int get_col_width(ColumnInfo ci)
+         {
             const int CHAR_WIDTH = 10;
             const int MIN_WIDTH = 50;
             const int MIN_DATE_WIDTH = 200;
@@ -299,12 +342,14 @@ namespace BCG {
       }
 
       // this is the stream callback
-      private void myCallback(object o) {
+      private void myCallback(object o)
+      {
          var timer = Stopwatch.StartNew();
          DataPacket dp = o as DataPacket;
          myCrono.Stop();
          Log.Ui.diag($"Received callback for {dp.schema.tableName}");
-         if (o == null) {
+         if (o == null)
+         {
             System.Windows.Forms.MessageBox.Show(this, "Received a Null Data packet",
                                                    "Invalid Data", MessageBoxButtons.OK, MessageBoxIcon.Error);
             return;
@@ -321,7 +366,8 @@ namespace BCG {
 
          myDataGrid.DataSource = new SimpleBindableCollection(dp.newRows.schema, dp.newRows);
          var force = false;
-         if (myChangeStyle && force) {
+         if (myChangeStyle && force)
+         {
             myChangeStyle = false;
             myDataGrid.Columns.Clear();
             myDataGrid.Columns.AddRange(Array.ConvertAll(dp.schema.columnInfo, _column_factory));
@@ -348,7 +394,8 @@ namespace BCG {
 
       private DataStreamExpression? myExp;
 
-      private void myTimer_Tick(object sender, System.EventArgs e) {
+      private void myTimer_Tick(object sender, System.EventArgs e)
+      {
          // try 
          // {
          ViewStreamDatabase.instance.doWork();
@@ -365,22 +412,27 @@ namespace BCG {
          // }
       }
 
-      class StringWrapper {
-         public StringWrapper(string s) {
+      class StringWrapper
+      {
+         public StringWrapper(string s)
+         {
             myTableName = s;
          }
-         public string tableName {
+         public string tableName
+         {
             get { return myTableName; }
          }
          private string myTableName;
       }
 
-      private void MainForm_Closed(object sender, System.EventArgs e) {
+      private void MainForm_Closed(object sender, System.EventArgs e)
+      {
          if (myExp != null)
             myExp.getView(new Dictionary<string, StatementExpression>(0)).onChange -= new DataStream.ChangeDelegate(myCallback);
       }
 
-      private void myMenuItemExecuteQueryClick(object sender, System.EventArgs e) {
+      private void myMenuItemExecuteQueryClick(object sender, System.EventArgs e)
+      {
          myQueryTextBox.SelectionStart = myQueryTextBox.Text.Length;
          myQueryTextBox.SelectionLength = myQueryTextBox.Text.Length;
          myQueryTextBox.SelectionColor = Color.White;
@@ -388,7 +440,8 @@ namespace BCG {
          myTryExpression(myQueryTextBox.Text);
       }
 
-      private void myToolStripMenuItemExecSel_Click(object sender, EventArgs e) {
+      private void myToolStripMenuItemExecSel_Click(object sender, EventArgs e)
+      {
          // int selstart = myQueryTextBox.SelectionStart;
          // int sellength = myQueryTextBox.SelectionLength;
          // myQueryTextBox.SelectionStart = 0;
@@ -403,7 +456,8 @@ namespace BCG {
          // myQueryTextBox.SelectionColor = Color.White;
       }
 
-      private void myMenuItemOpenClick(object sender, System.EventArgs e) {
+      private void myMenuItemOpenClick(object sender, System.EventArgs e)
+      {
          OpenFileDialog od = new OpenFileDialog();
          od.Filter = "C# class (*.cs)|*.cs";
 
@@ -411,21 +465,24 @@ namespace BCG {
             loadFile(od.FileName);
       }
 
-      private void mySave(string fileName) {
+      private void mySave(string fileName)
+      {
          using (TextWriter w = new StreamWriter(fileName))
             w.Write(myCSharpText.Text);
          myLastFileName = fileName;
          this.Text = fileName;
       }
 
-      private void myMenuItemSaveClick(object sender, System.EventArgs e) {
+      private void myMenuItemSaveClick(object sender, System.EventArgs e)
+      {
          if (myLastFileName == null)
             myMenuItemSaveAsClick(sender, e);
          else
             mySave(myLastFileName);
       }
 
-      private void myMenuItemSaveAsClick(object sender, System.EventArgs e) {
+      private void myMenuItemSaveAsClick(object sender, System.EventArgs e)
+      {
          SaveFileDialog sd = new SaveFileDialog();
          if (myLastFileName != null)
             sd.FileName = myLastFileName;
@@ -436,47 +493,56 @@ namespace BCG {
             mySave(sd.FileName);
       }
 
-      private void myTableListBox_DoubleClick(object sender, System.EventArgs e) {
-         if (myTableListBox.SelectedItem != null) {
+      private void myTableListBox_DoubleClick(object sender, System.EventArgs e)
+      {
+         if (myTableListBox.SelectedItem != null)
+         {
             string tableName = myTableListBox.SelectedItem.ToString();
             myQueryTextBox.Text = myQueryTextBox.Text == "" ? tableName : myQueryTextBox.Text + "\n" + tableName;
             myTryExpression(myQueryTextBox.Text);
          }
       }
 
-      private void myDataGrid_MouseEnter(object sender, System.EventArgs e) {
+      private void myDataGrid_MouseEnter(object sender, System.EventArgs e)
+      {
          //myDataGrid.TableStyles[0].GridLineColor = Color.LightSteelBlue;
          // if (myDataGrid.TableStyles.Count > 0)
          //    myDataGrid.TableStyles[0].HeaderBackColor = Color.LightSteelBlue;
       }
 
-      private void myDataGrid_MouseLeave(object sender, System.EventArgs e) {
+      private void myDataGrid_MouseLeave(object sender, System.EventArgs e)
+      {
          //myDataGrid.TableStyles[0].GridLineColor = Color.Gainsboro;
          // if (myDataGrid.TableStyles.Count > 0)
          //    myDataGrid.TableStyles[0].HeaderBackColor = Color.Gainsboro;
       }
 
-      private void myCSharpText_MouseEnter(object sender, System.EventArgs e) {
+      private void myCSharpText_MouseEnter(object sender, System.EventArgs e)
+      {
          //myCSharpText.BackColor = Color.Gainsboro;
          myCSharpText.Font = new Font(myCSharpText.Font.Name, myCSharpText.Font.Size, FontStyle.Bold);
       }
 
-      private void myCSharpText_MouseLeave(object sender, System.EventArgs e) {
+      private void myCSharpText_MouseLeave(object sender, System.EventArgs e)
+      {
          //myCSharpText.BackColor = Color.White;
          myCSharpText.Font = new Font(myCSharpText.Font.Name, myCSharpText.Font.Size, FontStyle.Regular);
       }
 
-      private void myTableListBox_MouseEnter(object sender, System.EventArgs e) {
+      private void myTableListBox_MouseEnter(object sender, System.EventArgs e)
+      {
          //myTableListBox.BackColor = Color.Gainsboro;
          myTableListBox.Font = new Font(myTableListBox.Font.Name, myTableListBox.Font.Size, FontStyle.Bold);
       }
 
-      private void myTableListBox_MouseLeave(object sender, System.EventArgs e) {
+      private void myTableListBox_MouseLeave(object sender, System.EventArgs e)
+      {
          //myTableListBox.BackColor = Color.White;
          myTableListBox.Font = new Font(myTableListBox.Font.Name, myTableListBox.Font.Size, FontStyle.Regular);
       }
 
-      private void myLoad(object sender, EventArgs e) {
+      private void myLoad(object sender, EventArgs e)
+      {
          myFreezeView(false);
 
          myToolStripMenuItemSite.DropDownItems.Clear();
@@ -500,7 +566,9 @@ namespace BCG {
          {
             var ex = new Win32Exception();  // same as new Win32Exception(Marshal.GetLastWin32Error())
             Console.WriteLine(ex.Message);
-         } else {
+         }
+         else
+         {
             Console.WriteLine("Set stdout worked.");
          }
 
@@ -508,7 +576,9 @@ namespace BCG {
          {
             var ex = new Win32Exception();  // same as new Win32Exception(Marshal.GetLastWin32Error())
             Console.WriteLine(ex.Message);
-         } else {
+         }
+         else
+         {
             Console.WriteLine("Set stderr worked.");
          }
          Console.WriteLine("Redirecting console ouput to this window...");
@@ -524,10 +594,12 @@ namespace BCG {
          //_library = new LibraryForm(this);
       }
 
-      private void myMenuItemSiteClick(object sender, System.EventArgs e) {
+      private void myMenuItemSiteClick(object sender, System.EventArgs e)
+      {
          ToolStripMenuItem mi = (ToolStripMenuItem)sender;
 
-         if (mi.Text != mySite) {
+         if (mi.Text != mySite)
+         {
             mySite = mi.Text;
             foreach (var m in mySiteItems)
                m.Checked = false;
@@ -548,63 +620,78 @@ namespace BCG {
          }
       }
 
-      private void myClosing(object sender, CancelEventArgs e) {
+      private void myClosing(object sender, CancelEventArgs e)
+      {
       }
 
-      private void myHandleTreeNodeSelect(object sender, TreeViewEventArgs e) {
-         if (myTreeView.SelectedNode != null) {
-            if (myTreeView.SelectedNode.LastNode == null) {
+      private void myHandleTreeNodeSelect(object sender, TreeViewEventArgs e)
+      {
+         if (myTreeView.SelectedNode != null)
+         {
+            if (myTreeView.SelectedNode.LastNode == null)
+            {
                string sn = myTreeView.SelectedNode.Parent.Text + "()";
                myQueryTextBox.Text = myQueryTextBox.Text == "" ? sn : myQueryTextBox.Text + "\n" + sn;
             }
          }
       }
 
-      private void myToolStripMenuItemExit_Click(object sender, EventArgs e) {
+      private void myToolStripMenuItemExit_Click(object sender, EventArgs e)
+      {
          Close();
       }
 
-      private void myToolStripMenuItemFreeze_Click(object sender, EventArgs e) {
+      private void myToolStripMenuItemFreeze_Click(object sender, EventArgs e)
+      {
          myFreezeView(myTimer.Enabled);
       }
 
-      private void myFreezeView(bool freeze) {
+      private void myFreezeView(bool freeze)
+      {
          myTimer.Enabled = !freeze;
          myToolStripMenuItemFreeze.Checked = freeze;
          myToolStripMenuItemStep.Enabled = freeze;
       }
 
-      private void myToolStripMenuItemStep_Click(object sender, EventArgs e) {
+      private void myToolStripMenuItemStep_Click(object sender, EventArgs e)
+      {
          ViewStreamDatabase.instance.doWork();
       }
 
-      private void myToolStripMenuItemLoadAss_Click(object sender, EventArgs e) {
+      private void myToolStripMenuItemLoadAss_Click(object sender, EventArgs e)
+      {
          OpenFileDialog ofd = new OpenFileDialog();
          ofd.ShowReadOnly = true;
          DialogResult dr = ofd.ShowDialog(this);
-         if (ofd.CheckFileExists && ofd.CheckPathExists && dr == DialogResult.OK) {
+         if (ofd.CheckFileExists && ofd.CheckPathExists && dr == DialogResult.OK)
+         {
             PrimitiveManager.instance.loadAssembly(ofd.FileName);
             myLoadAssemblies();
          }
       }
 
-      private void myToolStripMenuItemTest_Click(object sender, EventArgs e) {
-         if (myCurrentStream != null) {
+      private void myToolStripMenuItemTest_Click(object sender, EventArgs e)
+      {
+         if (myCurrentStream != null)
+         {
             myCurrentStream.onChange -= new DataStream.ChangeDelegate(myCallback);
             myCurrentStream.onChange += new DataStream.ChangeDelegate(myCallback);
          }
       }
 
-      private void myToolStripMenuItemClearConsole_Click(object sender, EventArgs e) {
+      private void myToolStripMenuItemClearConsole_Click(object sender, EventArgs e)
+      {
          myConsoleRichTextBox.Clear();
          Console.WriteLine("Redirecting console ouput to this window...");
       }
 
-      private void myTableFilterBox_TextChanged(object sender, EventArgs e) {
+      private void myTableFilterBox_TextChanged(object sender, EventArgs e)
+      {
          myUpdateTableList();
       }
 
-      private void myUpdateTableList() {
+      private void myUpdateTableList()
+      {
          IBlueRowCollection rows = myAllTagsStream.rows;
          if (rows == null)
             return;
@@ -619,12 +706,15 @@ namespace BCG {
          myTableListBox.Items.Clear();
          SortedList<string, bool> sl = new SortedList<string, bool>();
          if (string.IsNullOrEmpty(myTableFilterBox.Text))
-            foreach (IBlueRow r in rows) {
+            foreach (IBlueRow r in rows)
+            {
                sl[r["tag"].ToString().Substring(DSTagManager.get_stream_tag("").Length)] = true;
             }
-         else {
+         else
+         {
             Regex tableRegex = new Regex(myTableFilterBox.Text);
-            foreach (IBlueRow r in rows) {
+            foreach (IBlueRow r in rows)
+            {
                string thisTable = r["tag"].ToString().Substring(DSTagManager.get_stream_tag("").Length);
                if (tableRegex.IsMatch(thisTable))
                   sl[thisTable] = true;
@@ -642,7 +732,8 @@ namespace BCG {
          myTableListBox.EndUpdate();
       }
 
-      private void testGridToolStripMenuItem_Click(object sender, EventArgs e) {
+      private void testGridToolStripMenuItem_Click(object sender, EventArgs e)
+      {
          var timer = new Stopwatch();
          //while (true)
          //{
@@ -664,9 +755,11 @@ namespace BCG {
          foreach (IBlueRow r in s.rows)
             rows.Add(r);
 
-         for (int i = 0; i < 100; i++) {
+         for (int i = 0; i < 100; i++)
+         {
             timer.Restart();
-            rows.Sort(delegate (IBlueRow r1, IBlueRow r2) {
+            rows.Sort(delegate (IBlueRow r1, IBlueRow r2)
+            {
                return _sort_dicts_index(r1, r2, 5, true);
             });
             timer.Stop();
@@ -674,19 +767,22 @@ namespace BCG {
          }
       }
 
-      private static int mySortDictionaries(IBlueRow r1, IBlueRow r2, string column, bool low_to_high) {
+      private static int mySortDictionaries(IBlueRow r1, IBlueRow r2, string column, bool low_to_high)
+      {
          if (r1.Contains(column) && r2.Contains(column))
             return compare_send_dontcare_to_last(r1[column], r2[column], low_to_high);
          else
             return 0;
       }
 
-      private static int _sort_dicts_index(IBlueRow r1, IBlueRow r2, int col_no, bool low_to_high) {
+      private static int _sort_dicts_index(IBlueRow r1, IBlueRow r2, int col_no, bool low_to_high)
+      {
          //return compare_send_dontcare_to_last(r1[col_no], r2[col_no], low_to_high);
          return ((IComparable)r1[col_no]).CompareTo(r2[col_no]);
       }
 
-      internal static int compare_send_dontcare_to_last(object o1, object o2, bool low_to_high) {
+      internal static int compare_send_dontcare_to_last(object o1, object o2, bool low_to_high)
+      {
          if (myIsDontCare(o1))
             return myIsDontCare(o2) ? 0 : 1;
          else
@@ -694,14 +790,16 @@ namespace BCG {
                                                     : ((IComparable)o2).CompareTo(o1));
       }
 
-      private static bool myIsDontCare(object o) {
+      private static bool myIsDontCare(object o)
+      {
          return o == null || o == DBNull.Value ||
                   o is string && String.IsNullOrEmpty((string)o) ||
                   o is double && (Double.IsInfinity((double)o) || Double.IsNaN((double)o)) ||
                   o is float && (Single.IsInfinity((float)o) || Single.IsNaN((float)o));
       }
 
-      private void _new_grid_form(object args) {
+      private void _new_grid_form(object args)
+      {
          //System.Timers.Timer t = new System.Timers.Timer();
          ////t.SynchronizingObject
          //BCG.Utilities.BlueEventLog.logEvent(String.Format("New BTSTabularGridTestForm created on thread {0}", System.Threading.Thread.CurrentThread.ManagedThreadId),
@@ -713,11 +811,13 @@ namespace BCG {
 
       private void libraryToolStripMenuItem_Click(object sender, EventArgs e) { }
 
-      private void myTableFilterLabel_Click(object sender, EventArgs e) {
+      private void myTableFilterLabel_Click(object sender, EventArgs e)
+      {
 
       }
 
-      private void MainForm_Load(object sender, EventArgs e) {
+      private void MainForm_Load(object sender, EventArgs e)
+      {
 
       }
    }

src/DataStreamSuite/PubSub/Client/Communication/BlutoTCP/BlutoSubAddress.cs

@@ -1,77 +1,77 @@
-using System;
-using System.Collections.Generic;
-using System.Text;
-//using BCG.Utilities;
-using System.Net;
+// using System;
+// using System.Collections.Generic;
+// using System.Text;
+// //using BCG.Utilities;
+// using System.Net;
 
-namespace BCG
-{
-   public class BlutoSubAddress : SubAddress
-   {
-      private string myIp;
-      private int myPort;
+// namespace BCG
+// {
+//    public class BlutoSubAddress : SubAddress
+//    {
+//       private string myIp;
+//       private int myPort;
 
-      public static string prefix = "bluto";
+//       public static string prefix = "bluto";
 
-      public BlutoSubAddress(string ip, int port)
-      {
-         myIp = ip;
-         myPort = port;
-      }
+//       public BlutoSubAddress(string ip, int port)
+//       {
+//          myIp = ip;
+//          myPort = port;
+//       }
 
-      public BlutoSubAddress(string serverAddress)
-      {
-         string[] toks = serverAddress.Split(':');
-         // TODO
-         // BlueEventLog.logAssert(toks.Length == 3);
-         // BlueEventLog.logAssert(toks[0].ToLower().StartsWith(prefix));
+//       public BlutoSubAddress(string serverAddress)
+//       {
+//          string[] toks = serverAddress.Split(':');
+//          // TODO
+//          // BlueEventLog.logAssert(toks.Length == 3);
+//          // BlueEventLog.logAssert(toks[0].ToLower().StartsWith(prefix));
 
-         string machineNameOrIP = toks[1].Remove(0, 2);
+//          string machineNameOrIP = toks[1].Remove(0, 2);
 
-         if (machineNameOrIP.Split('.').Length == 4)
-            myIp = toks[1].Remove(0, 2);
-         else
-         {
-            //get the IP address from DNS
-            IPAddress[] theAddressList = Dns.GetHostEntry(machineNameOrIP).AddressList;
-            //BlueEventLog.logAssert(theAddressList.Length == 1, "Error: machine has multiple ip addresses.");
-            //if we get multiple addresses, we want the one that is of the corret dotted notation
-            bool oneWorked = false;
-            foreach (IPAddress addr in theAddressList)
-            {
-               if (addr.AddressFamily != System.Net.Sockets.AddressFamily.InterNetworkV6)
-               {
-                  myIp = addr.ToString();
-                  oneWorked = true;
-                      
-               }
-            }
+//          if (machineNameOrIP.Split('.').Length == 4)
+//             myIp = toks[1].Remove(0, 2);
+//          else
+//          {
+//             //get the IP address from DNS
+//             IPAddress[] theAddressList = Dns.GetHostEntry(machineNameOrIP).AddressList;
+//             //BlueEventLog.logAssert(theAddressList.Length == 1, "Error: machine has multiple ip addresses.");
+//             //if we get multiple addresses, we want the one that is of the corret dotted notation
+//             bool oneWorked = false;
+//             foreach (IPAddress addr in theAddressList)
+//             {
+//                if (addr.AddressFamily != System.Net.Sockets.AddressFamily.InterNetworkV6)
+//                {
+//                   myIp = addr.ToString();
+//                   oneWorked = true;
 
-            if (!oneWorked)
-            {
-               throw new ArgumentException("Couldn't form server address from string " + serverAddress);
-            }
-            
+//                }
+//             }
 
-         }
-         myPort = Convert.ToInt32(toks[2]);
-      }
+//             if (!oneWorked)
+//             {
+//                throw new ArgumentException("Couldn't form server address from string " + serverAddress);
+//             }
 
-      
 
-      public string ip
-      {
-         get { return myIp; }
-      }
+//          }
+//          myPort = Convert.ToInt32(toks[2]);
+//       }
 
-      public int port
-      {
-         get { return myPort; }
-      }
 
-      public override string ToString()
-      {
-         return prefix + "://" + ip + ":" + port;
-      }
-   }
-}
+
+//       public string ip
+//       {
+//          get { return myIp; }
+//       }
+
+//       public int port
+//       {
+//          get { return myPort; }
+//       }
+
+//       public override string ToString()
+//       {
+//          return prefix + "://" + ip + ":" + port;
+//       }
+//    }
+// }

src/DataStreamSuite/PubSub/Client/Communication/BlutoTCP/SmartConnectionManagedWrapper.cs

@@ -1,88 +1,89 @@
-using System;
-using System.Collections.Generic;
-using System.Text;
-using System.Threading;
-using System.Runtime.InteropServices;
+// using System;
+// using System.Collections.Generic;
+// using System.Text;
+// using System.Threading;
+// using System.Runtime.InteropServices;
 
-namespace BCG
-{
-   public class SmartConnectionManagedWrapper
-   {
-      public delegate void MessageDelegate(int msgType, byte[] msg);
-      private Dictionary<string, MessageDelegate> myServerToDel = new Dictionary<string, MessageDelegate>();
+// TODO: remove
+// namespace BCG
+// {
+//    public class SmartConnectionManagedWrapper
+//    {
+//       public delegate void MessageDelegate(int msgType, byte[] msg);
+//       private Dictionary<string, MessageDelegate> myServerToDel = new Dictionary<string, MessageDelegate>();
 
-      public SmartConnectionManagedWrapper ()
-      {
-         ClientToServerConnectionWrapper.init();
-         Thread readThr = new Thread(new ThreadStart(myReceiveData));
-         readThr.Start();
-      }
+//       public SmartConnectionManagedWrapper ()
+//       {
+//          ClientToServerConnectionWrapper.init();
+//          Thread readThr = new Thread(new ThreadStart(myReceiveData));
+//          readThr.Start();
+//       }
 
-      public void initConnection(BlutoSubAddress subAddress)
-      {
-         ClientToServerConnectionWrapper.initServerConnection(ClientToServerConnectionWrapper.stringToBytes(subAddress.ip),
-                                                               subAddress.port);
-      }
+//       public void initConnection(BlutoSubAddress subAddress)
+//       {
+//          ClientToServerConnectionWrapper.initServerConnection(ClientToServerConnectionWrapper.stringToBytes(subAddress.ip),
+//                                                                subAddress.port);
+//       }
 
-      public void send(BlutoSubAddress address, byte[] data)
-      {
-         byte[] ip = ClientToServerConnectionWrapper.stringToBytes (address.ip);
-         ClientToServerConnectionWrapper.sendDataToServer(ip, address.port, data, data.Length);
-      }
-      
+//       public void send(BlutoSubAddress address, byte[] data)
+//       {
+//          byte[] ip = ClientToServerConnectionWrapper.stringToBytes (address.ip);
+//          ClientToServerConnectionWrapper.sendDataToServer(ip, address.port, data, data.Length);
+//       }
 
-      public void resgisterDataCallback(BlutoSubAddress address, MessageDelegate del)
-      {
-         lock (this)
-         {
-            myServerToDel.Add(address.ToString(), del);
-         }
-      }
 
-      public void unregisterDataCallback(BlutoSubAddress address, MessageDelegate del)
-      {
-         lock (this)
-         {
-            myServerToDel.Remove(address.ip);
-         }
-      }
+//       public void resgisterDataCallback(BlutoSubAddress address, MessageDelegate del)
+//       {
+//          lock (this)
+//          {
+//             myServerToDel.Add(address.ToString(), del);
+//          }
+//       }
 
-      public string getClientAsSeenByServer(string serverURL)
-      {
-         return "";
-      }
+//       public void unregisterDataCallback(BlutoSubAddress address, MessageDelegate del)
+//       {
+//          lock (this)
+//          {
+//             myServerToDel.Remove(address.ip);
+//          }
+//       }
 
-      private unsafe void myReceiveData()
-      {
-         while (true)
-         {            
-            BlutoTCPClientMessage** psClientPackets = null;
-            int npts = 0;
-            ///returns as soon as it gets the first element
-            ClientToServerConnectionWrapper.getServerData(1000, 1, &psClientPackets, out npts);
-            for (int i = 0; i < npts; i++)
-            {
-               string ip = Marshal.PtrToStringAnsi(psClientPackets[i]->ip);
-               int port = psClientPackets[i]->port;
-               string serverURL = new BlutoSubAddress(ip, port).ToString();
+//       public string getClientAsSeenByServer(string serverURL)
+//       {
+//          return "";
+//       }
 
-               byte[] msg = null;
-               if (psClientPackets[i]->msgType == 1)
-               {
-                  msg = new byte[psClientPackets[i]->size];
-                  Marshal.Copy(psClientPackets[i]->msg, msg, 0, msg.Length);
-               }
+//       private unsafe void myReceiveData()
+//       {
+//          while (true)
+//          {            
+//             BlutoTCPClientMessage** psClientPackets = null;
+//             int npts = 0;
+//             ///returns as soon as it gets the first element
+//             ClientToServerConnectionWrapper.getServerData(1000, 1, &psClientPackets, out npts);
+//             for (int i = 0; i < npts; i++)
+//             {
+//                string ip = Marshal.PtrToStringAnsi(psClientPackets[i]->ip);
+//                int port = psClientPackets[i]->port;
+//                string serverURL = new BlutoSubAddress(ip, port).ToString();
 
-               MessageDelegate cb = null;
-               lock (this)
-               {
-                  myServerToDel.TryGetValue(serverURL, out cb);
-               }
-               if (cb != null)
-                  cb(psClientPackets[i]->msgType, msg);
-            }
-         }
-      }
+//                byte[] msg = null;
+//                if (psClientPackets[i]->msgType == 1)
+//                {
+//                   msg = new byte[psClientPackets[i]->size];
+//                   Marshal.Copy(psClientPackets[i]->msg, msg, 0, msg.Length);
+//                }
 
-   }
-}
+//                MessageDelegate cb = null;
+//                lock (this)
+//                {
+//                   myServerToDel.TryGetValue(serverURL, out cb);
+//                }
+//                if (cb != null)
+//                   cb(psClientPackets[i]->msgType, msg);
+//             }
+//          }
+//       }
+
+//    }
+// }

src/DataStreamSuite/PubSub/Client/Communication/IPubSubServerConnection.cs

@@ -72,6 +72,6 @@ namespace BCG {
       /// <param name="del"></param>
       void register_data_callback(PubSubServerDataDelegate del);
 
-      void register_send_confirmation_callback(SendConfirmationDelegate del);
+      //void register_send_confirmation_callback(SendConfirmationDelegate del);
    }
 }

src/DataStreamSuite/PubSub/Client/Communication/MSMQ/MSMQSubAddress.cs

@@ -1,52 +1,52 @@
-using System;
-//using System.Messaging;
+// using System;
+// //using System.Messaging;
 
-using BCG;
-//using BCG.Utilities;
+// using BCG;
+// //using BCG.Utilities;
 
-namespace BCG
-{
-   public class MSMQSubAddress : SubAddress
-   {
-      private string myIP, myQ;
-      public static readonly string prefix = "msmq";
+// namespace BCG
+// {
+//    public class MSMQSubAddress : SubAddress
+//    {
+//       private string myIP, myQ;
+//       public static readonly string prefix = "msmq";
 
-      public MSMQSubAddress (string adr)
-      {
-         try 
-         {
-            // throw away prefix
-            string s = adr;
-            s = s.Substring(prefix.Length + "://".Length);
-            // grab IP
-            myIP = s.Substring(0,s.IndexOf("/"));
-            // throw away ip
-            s = s.Substring(myIP.Length + 1);
-            myQ = s;
-      
-         }
-         catch (Exception)
-         {
-            throw (new InvalidSubAddress(adr + " is not a valid queue address"));
-         }
-      }
+//       public MSMQSubAddress (string adr)
+//       {
+//          try 
+//          {
+//             // throw away prefix
+//             string s = adr;
+//             s = s.Substring(prefix.Length + "://".Length);
+//             // grab IP
+//             myIP = s.Substring(0,s.IndexOf("/"));
+//             // throw away ip
+//             s = s.Substring(myIP.Length + 1);
+//             myQ = s;
 
-      public MSMQSubAddress (string ip, string simpleQName)
-      {
-         myIP = ip;
-         myQ = simpleQName;
-      }
+//          }
+//          catch (Exception)
+//          {
+//             throw (new InvalidSubAddress(adr + " is not a valid queue address"));
+//          }
+//       }
 
-      public string ip {get {return myIP;}}
+//       public MSMQSubAddress (string ip, string simpleQName)
+//       {
+//          myIP = ip;
+//          myQ = simpleQName;
+//       }
 
-      public string simpleQueueName {get {return myQ;}}
+//       public string ip {get {return myIP;}}
 
-      public override string ToString() { return prefix + "://" + myIP + "/" + myQ; }
+//       public string simpleQueueName {get {return myQ;}}
 
-      public string formatName
-      {
-         get{return "FormatName:DIRECT=TCP:" + myIP + "\\private$\\" + myQ;}
-      }
-      
-   }
-}
+//       public override string ToString() { return prefix + "://" + myIP + "/" + myQ; }
+
+//       public string formatName
+//       {
+//          get{return "FormatName:DIRECT=TCP:" + myIP + "\\private$\\" + myQ;}
+//       }
+
+//    }
+// }

src/DataStreamSuite/PubSub/Client/Communication/ManagedTCP/PubSubServerManagedTCPConnection.cs

@@ -1,296 +1,262 @@
-using System;
-using System.Collections.Generic;
-using System.Text;
-using System.Net;
-using System.Net.Sockets;
-using System.IO;
-//using BCG.Utilities;
-//using BCG.Utilities.General;
-using System.Threading;
-using mbts;
-using bts;
-using System.Linq;
-using System.Reflection.Metadata.Ecma335;
-using Bts.PubSub;
-
-namespace BCG {
-   /// <summary>
-   /// Keeps an ITCPConnection for talking to the server. It also monitors heartbeats
-   /// from the server, and in case of missing hbs, informs its clients, and tries to
-   /// restablish the connection.
-   /// Threads :
-   /// its assumed that IPubSubServerConnection functions (including doWork) will 
-   /// be called from the same thread. ITCPMessageHandler.onData and  
-   /// ITCPConnectionHandler.onConnection will be called on any arbitrary thread
-   /// 
-   /// </summary>
-   public class PubSubServerManagedTCPConnection : IPubSubServerConnection,// external facing interface to communicate msgs to Pubsubserver   
-                                                   ITCPMessageHandler,     // to get messages from ITCPCOnnection
-                                                   ITCPConnectionHandler { // to get connections from AsyncTCPConnector_APM
-      private ITCPConnection _connection, _new_connection;
-      private readonly BlutoSubAddress _server_address;
-      // if a message(with confirm) can not be sent , its id is queued here, 
-      // on reconnection, _on_send_onfirm_callback is fired with a negative on these ids, so the user
-      //can re-send it. The re-send mechanism is not in implemented in this class, but by the user of this class
-      private ThreadSafeQueue<string> _negative_unsent_confs = new ThreadSafeQueue<string>();
-      // in onData, data is queued here, for polling
-      private SyncQueue<byte[]> _in_data_queue = new SyncQueue<byte[]>();
-      private DateTime _last_hb = DateTime.Now;
-      private PubSubServerConnectionStatusChange _conn_callback;
-      private PubSubServerDataDelegate _data_callback;
-      private SendConfirmationDelegate _on_send_confirm_callback;
-      private readonly TimeSpan _connection_drop_threshold = PubSubConfig.Instance.ConnectionDropThreshold();
-
-      public PubSubServerManagedTCPConnection(string server_url)
-         : this(new BlutoSubAddress(server_url)) { }
-
-      public PubSubServerManagedTCPConnection(BlutoSubAddress server_address) {
-         _server_address = server_address;
-         _connect_to_ps_server();
-      }
-
-      private void _connect_to_ps_server() {
-         bts.Log.Gen.info($"PubSubServerManagedTCPConnection._connect_to_ps_server server: {_server_address}");
-         AsyncTCPConnector_APM.connect_to(_server_address, this);
-      }
-
-      #region IPubSubServerConnection Members
-
-      public void send_with_confirmation(byte[] data, ref string conf_id) {
-         bts.Log.Gen.info($"PubSubServerManagedTCPConnection.sendWithConfirmation server: {_server_address} data: {data.Length} _connection : {_connection != null}");
-         if (_connection != null) {
-            _connection.sendWithConfirmation(data, ref conf_id, (msg_id, confirmed) => {
-               if (_on_send_confirm_callback != null)
-                  _on_send_confirm_callback(msg_id, confirmed);
-            });
-         } else {
-            bts.Log.Gen.warn($"PubSubServerManagedTCPConnection.sendWithConfirmation server: {_server_address} no connection, buffering send");
-            conf_id = Guid.NewGuid().ToString();
-            _negative_unsent_confs.enqueue(conf_id);
-         }
-      }
-
-      private void _send(byte[] data) {
-         bts.Log.Gen.info($"PubSubServerManagedTCPConnection._send server: {_server_address} data: {data.Length} _connection : {_connection != null}");
-         if (_connection != null) {
-            bts.Log.Gen.info("PubSubServerManagedTCPConnection.send server: " + _server_address + "data: " + data.Length);
-            _connection.send(data);
-         }
-      }
-
-      public void send(bts.PubSubClientMessage ps_msg) {
-         bts.Log.Gen.info($"PubSubServerManagedTCPConnection.send server: {_server_address} ps_msg: {ps_msg}");
-         var msg_bytes = ps_msg.serialize_into_bytes();
-         PubSubClientMessage msg_retrv = new PubSubClientMessage();
-         msg_retrv.consruct_deserialize(msg_bytes);
-         bts.Log.Gen.info($"PubSubServerManagedTCPConnection.send server: {_server_address} msg_retrv: {msg_retrv}");
-         _send(msg_bytes);
-      }
-
-      // private void send(Stream str) {
-      //    BinaryReader br = new BinaryReader(str);
-      //    _send(br.ReadBytes(Convert.ToInt32(str.Length - str.Position)));
-      // }
-
-      // private void send(PubSubMessage msg) {
-      //    send(msg.to_stream());
-      // }
-
-      // public void send_with_confirmation(PubSubMessage msg, ref string conf_id) {
-      //    var bytes = msg.to_bytes();
-      //    send_with_confirmation(bytes, ref conf_id);
-      // }
-
-      public void send_with_confirmation(bts.PubSubClientMessage msg, ref string conf_id, SendConfirmationDelegate del) {
-         if (_connection == null) {
-            // quq
-         }
-         MemoryStream str = new MemoryStream();
-         msg.serialize_into_str(str);
-         var bytes = StreamUtilityMethods.getBytesFromStream(str);
-         _connection.sendWithConfirmation(bytes, ref conf_id, del);
-      }
-
-      public IEnumerable<bts.PubSubServerMessage> get_updates(TimeSpan timeout, int maxElements) {
-         List<byte[]> rv_raw = _in_data_queue.dequeueMultipleElements(Convert.ToInt32(timeout.TotalMilliseconds), maxElements);
-         List<bts.PubSubServerMessage> rv = new List<PubSubServerMessage>();
-         var i = 0;
-         foreach (var b in rv_raw) {
-            PubSubServerMessage ps_msg = new PubSubServerMessage();
-            ps_msg.consruct_deserialize(b);
-            bts.Log.Gen.info($"PubSubServerManagedTCPConnection.get_updates downloaded: {ps_msg} msg_no: {i} / {rv_raw.Count}");
-            rv.Add(ps_msg);
-            i++;
-         }
-         return rv;
-      }
-
-      public string server_url {
-         get { return _server_address.ToString(); }
-      }
-
-      public string client_url {
-         get {
-            string rv = "";
-            lock (this) {
-               if (_connection == null) {
-                  rv = "";
-               } else {
-                  if (PubSubConfig.Instance.IpTranslation.is_null_or_empty()) {
-                     string ip;
-                     int port;
-                     _connection.getSelfAddress(out ip, out port);
-                     rv = new BlutoSubAddress(ip, port).ToString();
-                  } else {
-                     rv = "";
-                  }
-               }
-            }
-            return rv;
-         }
-      }
-
-      public void register_connection_callback(PubSubServerConnectionStatusChange del) {
-         _conn_callback = del;
-      }
-
-      public void register_data_callback(PubSubServerDataDelegate del) {
-         _data_callback = del;
-      }
-
-      public void register_send_confirmation_callback(SendConfirmationDelegate del) {
-         _on_send_confirm_callback = del;
-      }
-
-      #endregion
-
-      #region IDisposable Members
-
-      private bool _is_disposed = false;
-      public void Dispose() {
-         bts.Log.Gen.info("PubSubServerManagedTCPConnection.Dispose");
-         //throw new Exception("The method or operation is not implemented.");
-         if (_connection != null) {
-            _connection.Dispose();
-
-         }
-         _is_disposed = true;
-      }
-      public bool isDisposed {
-         get {
-            return _is_disposed;
-         }
-
-      }
-
-      #endregion
-
-      #region ITCPMessageHandler Members
-
-      /// <summary>
-      /// This method is executed on an arbitrary CLR thread. To avoid blocking it
-      /// data is written to the queue and method returns. myLastHB being a primitive
-      /// (long) is safe for multi threaded operations
-      /// </summary>
-      /// <param name="message"></param>
-      public void onData(RemoteMessage message) {
-         bts.Log.Gen.info($"PubSubServerManagedTCPConnection.onData message: {message}");
-         // TODO check for disconenction
-         switch (message.messageType) {
-            case RemoteMessageType.Data:
-               if (_data_callback != null)
-                  _data_callback(message.remoteConnectionInfo.ToString(), message.message);
-               else
-                  _in_data_queue.enqueue(message.message);
-               break;
-            case RemoteMessageType.Heartbeat:
-               _last_hb = DateTime.Now;
-               break;
-            case RemoteMessageType.Connection:
-               _conn_callback(_server_address.ToString(), true);
-               break;
-            case RemoteMessageType.Disconnection:
-               if (_conn_callback != null)
-                  _conn_callback(_server_address.ToString(), false);
-               _connection.Dispose();
-               _connect_to_ps_server();
-               break;
-
-         }
-      }
-
-      #endregion
-
-      #region ITCPConnectionHandler Members
-
-      ///  will be called on on an rbitrary .Net thread
-      public void onConnection(ITCPConnection connection) {
-         string ip;
-         int port;
-         connection.getPeerAddress(out ip, out port);
-         BlutoSubAddress add = new BlutoSubAddress(ip, port);
-         bts.Log.Gen.info($"PubSubServerManagedTCPConnection.onConnection add: {add.ToString()} connection type: {connection.GetType()}");
-         if (add.ToString() == _server_address.ToString()) {
-            _last_hb = DateTime.Now;
-            _new_connection = connection;
-         }
-      }
-
-      #endregion
-
-      void _dispose_and_reconnect_if_missing_hb() {
-         if (!_connection.isDisposed) {
-            TimeSpan hb_interval = DateTime.Now - _last_hb;
-            bool hb_missed = hb_interval > _connection_drop_threshold;
-            if (hb_missed) {
-               if (_conn_callback != null)
-                  _conn_callback(_server_address.ToString(), false);
-               bts.Log.Gen.info("PubSubServerManagedTCPConnection._dispose_and_reconnect_if_missing_hb disposing connecion: {myServerAddress}");
-               _connection.Dispose();
-               _connect_to_ps_server();
-            }
-         } else {
-            // already disposed, nothing to do
-         }
-      }
-
-      void _check_for_new_connection() {
-         if (_new_connection != null) {
-            _connection = _new_connection;
-            _new_connection = null; // since we are using its presence as a marker of nw connections
-            bts.Log.Gen.info("PubSubServerManagedTCPConnection._check_for_new_connection assigning new connection");
-            _connection.register(this);
-            if (_conn_callback != null)
-               _conn_callback(_server_address.ToString(), true);
-            if (_on_send_confirm_callback != null) {
-               var msg_ids = _negative_unsent_confs.dequeueAll();
-               foreach (var id in msg_ids)
-                  _on_send_confirm_callback(id, false);
-            }
-         }
-      }
-
-      #region IWorker Members
-
-      /// <summary>
-      /// should be called on the same thread as the other user functions.
-      /// the design only allows for onConnection to be on a different
-      /// thread.
-      /// </summary>
-      public void doWork() {
-         //_dispose_and_reconnect_if_missing_hb();
-         //try {
-         _check_for_new_connection();
-         // } catch (Exception ex) {
-         //    bts.Log.Gen.warn($"PubsSubServerManagedTCPConnection.doWork ex: {ex}");
-         // }
-      }
-
-      #endregion
-
-      // private void myOnSendConfirm(string messageId, bool confirmed) {
-      //    _negative_confs.enqueue(new Pair<string, bool>(messageId, confirmed));
-      // }
-
-   }
-}
+using System;
+using System.Collections.Generic;
+using System.Text;
+using System.Net;
+using System.Net.Sockets;
+using System.IO;
+//using BCG.Utilities;
+//using BCG.Utilities.General;
+using System.Threading;
+using mbts;
+using bts;
+using System.Linq;
+using System.Reflection.Metadata.Ecma335;
+using Bts.PubSub;
+using Microsoft.Win32;
+using System.ComponentModel;
+
+namespace BCG {
+   /// <summary>
+   /// Keeps an ITCPConnection for talking to the server. It also monitors heartbeats
+   /// from the server, and in case of missing hbs, informs its clients, and tries to
+   /// restablish the connection.
+   /// Threads :
+   /// its assumed that IPubSubServerConnection functions (including doWork) will 
+   /// be called from the same thread. ITCPMessageHandler.onData and  
+   /// ITCPConnectionHandler.onConnection are called on an arbitrary thread.
+   /// _conn_callback is called from within doWork, and _data_callback is called grom 
+   // an arbitrary data thread
+   /// </summary>
+   public class PubSubServerManagedTCPConnection : IPubSubServerConnection,// external facing interface to communicate msgs to Pubsubserver   
+                                                   ITCPMessageHandler,     // to get messages from ITCPCOnnection
+                                                   ITCPConnectionHandler { // to get connections from AsyncTCPConnector_APM
+      private ITCPConnection _connection;
+      private readonly RemoteConnectionInfo _server_rc_info;
+      private readonly bool _is_sub;
+      // if _data_callback is not subscribed to , the user of this class can poll updates,
+      // in which case this queue will be used
+      private SyncQueue<byte[]> _in_data_queue = new SyncQueue<byte[]>();
+      private DateTime _last_hb = DateTime.Now;
+      private PubSubServerConnectionStatusChange _conn_callback;
+      private PubSubServerDataDelegate _data_callback;
+      private readonly TimeSpan _connection_drop_threshold = PubSubConfig.Instance.ConnectionDropThreshold();
+      private ThreadSafeQueue<ConnectionStatusChange> _conn_cb_queue = new ThreadSafeQueue<ConnectionStatusChange>();
+
+      // the callbacks for connection, comes from the ITCPConnection, when a connection is broken,
+      // and from AsyncTCPConnector whne a connection is established. ALl that info is assembled here and
+      // enqueued, deuqued oin the doWork and acted on
+      private class ConnectionStatusChange {
+         public enum Status {
+            CONNECTED,
+            DISCONNECTED
+         }
+         public readonly Status connection_satus;
+         public readonly ITCPConnection new_connection = null;
+         public ConnectionStatusChange(Status connection_satus_, ITCPConnection new_connection_) {
+            connection_satus = connection_satus_;
+            new_connection = new_connection_;
+         }
+         public override string ToString() {
+            return connection_satus.ToString();
+         }
+      }
+
+      public PubSubServerManagedTCPConnection(RemoteConnectionInfo server_address, bool is_sub) {
+         Log.Gen.info("PubSubServerManagedTCPConnection.cons server_address: {server_address} is_sub: {is_sub}");
+         _server_rc_info = server_address;
+         _is_sub = is_sub;
+         // this is where it tries to connect to the pubsub server, and gets a callback on onConnection
+         _connect_to_ps_server();
+      }
+
+      private void _connect_to_ps_server() {
+         bts.Log.Gen.info($"PubSubServerManagedTCPConnection._connect_to_ps_server server _server_rc_info: {_server_rc_info} is_sub: {_is_sub}");
+         // onConnection will be called on an arbitrary thread
+         AsyncTCPConnector_AsyncWrapper.connect_to(_server_rc_info, this);
+      }
+
+      #region IPubSubServerConnection Members
+
+      private void _send(byte[] data) {
+         bts.Log.Gen.info($"PubSubServerManagedTCPConnection._send client: {client_url} server: {_server_rc_info} data: {data.Length} is_sub: {_is_sub} _connection : {_connection != null}");
+         if (_connection != null) {
+            _connection.send(data);
+            bts.Log.Gen.info("PubSubServerManagedTCPConnection.send, data is sent, server: " + _server_rc_info + "data: " + data.Length);
+         } else {
+            bts.Log.Gen.warn($"PubSubServerManagedTCPConnection._send, could not send data, server: {_server_rc_info} data: {data.Length} is_sub: {_is_sub} _connection : {_connection != null} ");
+         }
+      }
+
+      public void send(bts.PubSubClientMessage ps_msg) {
+         bts.Log.Gen.info($"PubSubServerManagedTCPConnection.send server: {_server_rc_info} ps_msg: {ps_msg}");
+         // whys it constructed again?
+         var msg_bytes = ps_msg.serialize_into_bytes();
+         //bts.Log.Gen.info($"PubSubServerManagedTCPConnection.send server: {_server_rc_info} msg_retrv: {msg_retrv}");
+         _send(msg_bytes);
+      }
+
+      public IEnumerable<bts.PubSubServerMessage> get_updates(TimeSpan timeout, int maxElements) {
+         List<byte[]> rv_raw = _in_data_queue.dequeueMultipleElements(Convert.ToInt32(timeout.TotalMilliseconds), maxElements);
+         List<bts.PubSubServerMessage> rv = new List<PubSubServerMessage>();
+         var i = 0;
+         foreach (var b in rv_raw) {
+            PubSubServerMessage ps_msg = new PubSubServerMessage();
+            ps_msg.consruct_deserialize(b);
+            bts.Log.Gen.info($"PubSubServerManagedTCPConnection.get_updates downloaded: {ps_msg} msg_no: {i} / {rv_raw.Count}");
+            rv.Add(ps_msg);
+            i++;
+         }
+         return rv;
+      }
+
+      public string server_url {
+         get { return _server_rc_info.ToString(); }
+      }
+
+      public string client_url {
+         get {
+            if (_connection == null)
+               return RemoteConnectionInfo.null_instance.ToString();
+            return _connection.get_self_address().ToString();
+         }
+      }
+
+      public void register_connection_callback(PubSubServerConnectionStatusChange del) {
+         _conn_callback = del;
+      }
+
+      public void register_data_callback(PubSubServerDataDelegate del) {
+         _data_callback = del;
+      }
+
+      #endregion
+
+      #region IDisposable Members
+
+      private bool _is_disposed = false;
+      public void Dispose() {
+         if (isDisposed)
+            return;
+         bts.Log.Gen.info("PubSubServerManagedTCPConnection.Dispose");
+         if (_connection != null) {
+            _connection.Dispose();
+         }
+         _is_disposed = true;
+      }
+      public bool isDisposed {
+         get {
+            return _is_disposed;
+         }
+      }
+
+      #endregion
+
+      #region ITCPMessageHandler Members
+
+      /// <summary>
+      /// This method is executed on an arbitrary CLR thread.
+      /// </summary>
+      /// <param name="message"></param>
+      public void onData(RemoteMessage message) {
+         bts.Log.Gen.info($"PubSubServerManagedTCPConnection.onData connection: {_connection} message: {message}");
+         // TODO check for disconenction
+         switch (message.messageType) {
+            case RemoteMessageType.Data:
+               if (_data_callback != null)
+                  _data_callback(message.remoteConnectionInfo.ToString(), message.message);
+               else
+                  _in_data_queue.enqueue(message.message);
+               break;
+            case RemoteMessageType.Heartbeat:
+               _last_hb = DateTime.Now;
+               break;
+            case RemoteMessageType.Connection:
+               // since this is called from the connection, this condition will never hit, ssince
+               //new connections are detected only by AsyncTCPConnector
+               Log.assert(false);
+               break;
+            case RemoteMessageType.Disconnection:
+               bts.Log.Gen.info($"PubSubServerManagedTCPConnection.onData case RemoteMessageType.Disconnection: message: {message}");
+               _conn_cb_queue.enqueue(new ConnectionStatusChange(ConnectionStatusChange.Status.DISCONNECTED, null));
+               break;
+
+         }
+      }
+
+      #endregion
+
+      #region ITCPConnectionHandler Members
+
+      ///  will be called on on an arbitrary .Net thread
+      public void onConnection(ITCPConnection connection) {
+         var server_rci = connection.get_peer_address();
+         bts.Log.Gen.info($"PubSubServerManagedTCPConnection.onConnection connection: {_connection_txt(connection)}");
+         if (_server_rc_info.Equals(server_rci)) {
+            _conn_cb_queue.enqueue(new ConnectionStatusChange(ConnectionStatusChange.Status.CONNECTED, connection));
+            // _last_hb = DateTime.Now;
+            // _new_connection = connection;
+         }
+      }
+
+      #endregion
+
+      void _check_missing_hb() {
+         if (!_connection.isDisposed) {
+            TimeSpan hb_interval = DateTime.Now - _last_hb;
+            bool hb_missed = hb_interval > _connection_drop_threshold;
+            if (hb_missed) {
+               bts.Log.Gen.warn("PubSubServerManagedTCPConnection._dispose_and_reconnect_if_missing_hb disposing connecion: {myServerAddress}");
+               ConnectionStatusChange ci = new ConnectionStatusChange(ConnectionStatusChange.Status.DISCONNECTED, null);
+               _conn_cb_queue.enqueue(ci);
+            }
+         } else {
+            // already disposed, nothing to do
+         }
+      }
+
+      private static string _connection_txt(ITCPConnection conn) {
+         if (conn != null) {
+            return $"conn_type: {conn.GetType()} server: {conn.get_peer_address()} self: {conn.get_self_address()}";
+         }
+         return "NULL";
+      }
+
+      #region IWorker Members
+
+      /// <summary>
+      /// should be called on the same thread as the other user functions.
+      /// the design only allows for onConnection to be on a different
+      /// thread.
+      /// </summary>
+      public void doWork() {
+         //_dispose_and_reconnect_if_missing_hb();
+         if (_conn_cb_queue.getCount() > 0) {
+            var css = _conn_cb_queue.dequeueAll();
+            foreach (var c in css) {
+               Log.Gen.info($"PubSubServerManagedTCPConnection.doWork c: {c}");
+               switch (c.connection_satus) {
+                  case ConnectionStatusChange.Status.CONNECTED: {
+                        Log.assert(c.new_connection != null);
+                        c.new_connection.register(this);
+                        _connection = c.new_connection;
+                        _last_hb = DateTime.Now;
+                        if (_conn_callback != null)
+                           _conn_callback(_server_rc_info.ToString(), true);
+                        break;
+                     }
+                  case ConnectionStatusChange.Status.DISCONNECTED: {
+                        _connection.Dispose();
+                        _connect_to_ps_server();
+                        if (_conn_callback != null)
+                           _conn_callback(_server_rc_info.ToString(), false);
+                        break;
+                     }
+               }
+            }
+         }
+      }
+
+      #endregion
+
+   }
+}

src/DataStreamSuite/PubSub/Client/Communication/PubSubServerConnectionFactory.cs

@@ -3,55 +3,38 @@ using System.Collections.Generic;
 using System.Text;
 //using BCG.Utilities;
 
-namespace BCG
-{
+namespace BCG {
    /// <summary>
    /// Every instance of PublisherCore and SubscriberCore will have its
    /// factory. Rather than have once singelton factory, this approach
    /// is preffered since multiple pub/sub core can connect to the same
    /// connection and make it difficultt o manage connections.
    /// </summary>
-   public class PubSubServerConnectionFactory : IDisposable
-   {
-      private bool myIsSub;
-      private Dictionary<string, IPubSubServerConnection> myConnections = new Dictionary<string, IPubSubServerConnection>();
+   public class PubSubServerConnectionFactory : IDisposable {
+      private Dictionary<RemoteConnectionInfo, IPubSubServerConnection> _conn_map =
+         new Dictionary<RemoteConnectionInfo, IPubSubServerConnection>();
+      private readonly bool _is_sub;
 
-      internal PubSubServerConnectionFactory (bool isSub)
-      {
-         myIsSub = isSub;
+      internal PubSubServerConnectionFactory(bool is_sub) {
+         _is_sub = is_sub;
       }
 
-      public IPubSubServerConnection getConnection(string url)
-      {
-         if (!myConnections.ContainsKey(url))
-            myConnections.Add(url, myBuildConnection(url));
-         return myConnections[url];
+      public IPubSubServerConnection get_connection(RemoteConnectionInfo server_rci) {
+         IPubSubServerConnection rv = null;
+         if (!_conn_map.TryGetValue(server_rci, out rv)) {
+            rv = _build_connection(server_rci);
+            _conn_map.Add(server_rci, rv);
+         }
+         return rv;
       }
 
-      private IPubSubServerConnection myBuildConnection(string url)
-      {
-         if (url.StartsWith(MSMQSubAddress.prefix))
-         {
-              // TODO, no more MSMQ 
-             //Diagnostics.assert(false);
-             return null;
-            // MSMQSubAddress serverAdd = new MSMQSubAddress(url);
-            // return new PubSubServerMSMQConnection(serverAdd.ToString(), myIsSub);
-         }
-         else if (url.StartsWith(BlutoSubAddress.prefix))
-         {
-            return new PubSubServerManagedTCPConnection(url);
-         }
-         else
-         {
-            // BlueEventLog.logAssert(false, "Unknown transport, only msmq and blutoTcp allowed");
-            return null;
-         }
+      private IPubSubServerConnection _build_connection(RemoteConnectionInfo server_rci) {
+         return new PubSubServerManagedTCPConnection(server_rci, _is_sub);
       }
 
       #region IDisposable Members
 
-      public void Dispose()   {}
+      public void Dispose() { }
 
       #endregion
    }

src/DataStreamSuite/PubSub/Client/MetaData/PubSubClientMessage.cs

@@ -124,7 +124,14 @@ namespace bts {
         public long publish_time_epoch_ms = DateTimeOffset.UtcNow.ToUnixTimeMilliseconds();
         public byte[] data_to_publish;
 
+        public void sanity_check() {
+            Log.assert(!String.IsNullOrEmpty(tag_type));
+            Log.assert(!String.IsNullOrEmpty(tag));
+            Log.assert(!String.IsNullOrEmpty(connection));
+        }
+
         public void serialize_into_str(Stream str) {
+            sanity_check();
             BinaryWriter bw = new BinaryWriter(str);
             bw.write_string_cpp(connection);
             bw.write_string_cpp(publishing_app);

src/DataStreamSuite/PubSub/Client/PubsubTagHandlerFactory.cs

@@ -24,6 +24,7 @@ namespace bts {
       }
       private static Dictionary<string, TagHandlerEntry> _config_map = new Dictionary<string, TagHandlerEntry>()
       {
+         {String.Empty, new TagHandlerEntry(DSTagManager.DataStreamTagType , "DataStreamCore", "bts.PubsubServerStreamTagHandler")},
       {DSTagManager.DataStreamTagType, new TagHandlerEntry(DSTagManager.DataStreamTagType , "DataStreamCore", "bts.PubsubServerStreamTagHandler")},
       {"System_Sub", new TagHandlerEntry("System_Sub", "PubSub", "BCG.SystemSubscriptionMapPublisher")},
       {"System_Stats", new TagHandlerEntry("System_Stats", "PubSub", "BCG.SystemStatsPublisher")}

src/DataStreamSuite/PubSub/Client/Routing/IPubSubRouter.cs

@@ -8,11 +8,11 @@ namespace BCG {
 
    public interface IPubSubRouter {
 
-      event PubSubTagURLChangeDelegate onSubURLTagRouteChange;
-      event PubSubTagURLChangeDelegate onPubURLTagRouteChange;
-      List<string> get_all_pubsub_sub_urls();
-      List<string> get_all_pubsub_pub_urlss();
-      string get_sub_url_for_tag(string tag);
-      string get_pub_url_for_tag(string tag);
+      // event PubSubTagURLChangeDelegate onSubURLTagRouteChange;
+      // event PubSubTagURLChangeDelegate onPubURLTagRouteChange;
+      IEnumerable<PubSubRoute> sub_routes();
+
+      RemoteConnectionInfo get_sub_route_for_tag(string tag);
+      RemoteConnectionInfo get_pub_route_for_tag(string tag);
    }
 }

src/DataStreamSuite/PubSub/Client/Routing/PubSubRoute.cs

@@ -5,43 +5,38 @@ using System.Text.RegularExpressions;
 using System.Text.Json.Serialization;
 using System.Text.Json;
 using System.Text.Json.Nodes;
+using System.Runtime.InteropServices;
 
 namespace BCG {
    [JsonConverter(typeof(PubSubRouteJsonConverter))]
    public class PubSubRoute : IComparable<PubSubRoute> {
-      public string serverAddress;
-      public string tagRegExp;
+      public string server_address;
+      public string tag_reg_exp;
       //route names are used in intermediary code (i.e. while substations are being deployed)
       //so that MC Hammer can advertise them. 
       public string name;
-      private Regex myRegex = null;
+      private Regex _regex = null;
 
       public PubSubRoute() { }
 
-      public PubSubRoute(string theServerAddress, string theTagRegExp) {
-         serverAddress = theServerAddress;
-         tagRegExp = theTagRegExp;
+      public PubSubRoute(string the_server_address, string the_tag_reg_exp) {
+         server_address = the_server_address;
+         tag_reg_exp = the_tag_reg_exp;
       }
 
-      // public PubSubRoute()
-      // {
-      //    // serverAddress = string.Format("bluto://{0}:{1}", cfg.serverAddress, cfg.serverPort);
-      //    // tagRegExp = cfg.tagsRegex;
-      // }
-
       public string[] getAllPubSubs() {
-         return serverAddress.Split(';');
+         return server_address.Split(';');
       }
 
-      public Regex getRegex() {
-         return myRegex ?? new Regex(tagRegExp);
+      public Regex get_regex() {
+         return _regex ?? new Regex(tag_reg_exp);
       }
 
       #region IComparable<PubSubRoute> Members
 
       public int CompareTo(PubSubRoute other) {
-         return serverAddress == other.serverAddress ? tagRegExp.CompareTo(other.tagRegExp)
-                                                      : serverAddress.CompareTo(other.serverAddress);
+         return server_address == other.server_address ? tag_reg_exp.CompareTo(other.tag_reg_exp)
+                                                      : server_address.CompareTo(other.server_address);
       }
 
       #endregion
@@ -51,8 +46,8 @@ namespace BCG {
             return false;
          else {
             PubSubRoute other = obj as PubSubRoute;
-            return serverAddress.Equals(other.serverAddress) &&
-                     tagRegExp.Equals(other.tagRegExp);
+            return server_address.Equals(other.server_address) &&
+                     tag_reg_exp.Equals(other.tag_reg_exp);
          }
       }
 
@@ -61,29 +56,33 @@ namespace BCG {
       }
 
       public static PubSubRoute get_default_route() {
-         return new PubSubRoute("bluto://127.0.0.0.1:12345", ".*");
+         return new PubSubRoute("bluto://127.0.0.1:12345", ".*");
+      }
+
+      public override string ToString() {
+         return $"tag_reg_exp: {tag_reg_exp} server_address: {server_address}";
       }
    }
 
    public static class PubSubRouteExtensions {
       public static bool IsMatch(this PubSubRoute route, string tag) =>
-         route.getRegex().IsMatch(tag);
+         route.get_regex().IsMatch(tag);
    }
 
    public class PubSubRouteJsonConverter : JsonConverter<PubSubRoute> {
       public override PubSubRoute? Read(ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options) {
-         var jsonObj = JsonSerializer.Deserialize<JsonObject>(ref reader);
-         if (jsonObj is null) {
+         var json_obj = JsonSerializer.Deserialize<JsonObject>(ref reader);
+         if (json_obj is null) {
             return null;
          }
          var rv = new PubSubRoute();
          ReadString("name", ref rv.name);
-         ReadString("server_address", ref rv.serverAddress);
-         ReadString("tag", ref rv.tagRegExp);
+         ReadString("server_address", ref rv.server_address);
+         ReadString("tag", ref rv.tag_reg_exp);
          return rv;
 
          void ReadString(string key, ref string valueOut) {
-            if (jsonObj.TryGetPropertyValue(key, out var valueJson)) {
+            if (json_obj.TryGetPropertyValue(key, out var valueJson)) {
                var valueStr = valueJson?.GetValue<string>();
                // Only (over)write valueOut if non-null
                if (valueStr is not null) {
@@ -96,8 +95,8 @@ namespace BCG {
       public override void Write(Utf8JsonWriter writer, PubSubRoute value, JsonSerializerOptions options) {
          var jsonObj = new JsonObject {
             ["name"] = value.name,
-            ["server_address"] = value.serverAddress,
-            ["tag"] = value.tagRegExp,
+            ["server_address"] = value.server_address,
+            ["tag"] = value.tag_reg_exp,
          };
          JsonSerializer.Serialize(writer, jsonObj);
       }

src/DataStreamSuite/PubSub/Client/Routing/PubSubRouter.cs

@@ -1,97 +1,83 @@
-using System;
-using System.Text.RegularExpressions;
-using System.Collections;
-using System.Collections.Generic;
-
-using BCG;
-using Bts.PubSub;
-using bts;
-using System.Linq;
-//using BCG.Utilities;
-//using bts;
-
-namespace BCG {
-   public class PubSubRouter : IPubSubRouter {
-      private Regex[] myPubRouteRegExps;
-      private Regex[] mySubRouteRegExps;
-      private PubSubRoute[] myPubRoutes;
-      private PubSubRoute[] mySubRoutes;
-
-      public PubSubRouter() : this(
-         PubSubConfig.Instance.SubscriberRoutes?.ToArray()
-            ?? throw new InvalidOperationException("Subscriber routes are not configured"),
-         PubSubConfig.Instance.PublisherRoutes?.ToArray()
-            ?? throw new InvalidOperationException("Publisher routes are not configured")) { }
-
-      public PubSubRouter(params PubSubRoute[] routes) : this(routes, routes) { }
-
-      public PubSubRouter(PubSubRoute[] subRoutes, PubSubRoute[] pubRoutes) {
-         mySubRouteRegExps = new Regex[subRoutes.Length];
-         myPubRouteRegExps = new Regex[pubRoutes.Length];
-
-         mySubRoutes = subRoutes;
-         myPubRoutes = pubRoutes;
-
-         for (int i = 0; i < subRoutes.Length; ++i)
-            mySubRouteRegExps[i] = new Regex(subRoutes[i].tagRegExp);
-         for (int i = 0; i < pubRoutes.Length; ++i)
-            myPubRouteRegExps[i] = new Regex(pubRoutes[i].tagRegExp);
-      }
-
-      private int myGetRouteIndex(string forTag, PubSubRoute[] routes, Regex[] regexes) {
-         for (int i = 0; i < routes.Length; i++)
-            if (regexes[i].IsMatch(forTag))
-               return i;
-
-         throw new PubSubRouteNotFound(String.Format("No route found for {0}", forTag));
-      }
-
-      #region IPubSubRouter methods
-
-      private string myPickAServer(string tag, string[] urls) {
-         if (urls.Length == 1)
-            return urls[0];
-         else {
-            uint hc = (uint)tag.GetHashCode();
-            return urls[hc % urls.Length];
-         }
-      }
-
-      public string get_pub_url_for_tag(string tag) {
-         if (!myPubRoutes.TryGetFirst(r => r?.IsMatch(tag) ?? false, out var route)) {
-            throw new ArgumentException($"Failed to find route for tag '{tag}'", nameof(tag));
-         }
-         Log.Gen.info($"PubSubRouter.get_sub_url_for_tag. tag: {tag} route: {route}");
-         return route.serverAddress;
-      }
-
-      public string get_sub_url_for_tag(string tag) {
-         if (!mySubRoutes.TryGetFirst(r => r?.IsMatch(tag) ?? false, out var route)) {
-            throw new ArgumentException($"Failed to find route for tag '{tag}'", nameof(tag));
-         }
-         Log.Gen.info($"PubSubRouter.get_sub_url_for_tag. tag: {tag} route: {route}");
-         return route.serverAddress;
-      }
-
-      public List<string> get_all_pubsub_sub_urls() =>
-         mySubRoutes.Select(r => r.serverAddress).ToList();
-
-      public List<string> get_all_pubsub_pub_urlss() =>
-         myPubRoutes.Select(r => r.serverAddress).ToList();
-
-      public event PubSubTagURLChangeDelegate onSubURLTagRouteChange;
-      public event PubSubTagURLChangeDelegate onPubURLTagRouteChange;
-
-      #endregion
-
-      //this function is put here so that the events are used, in order to prevent build errors. 
-      private void myNeverCallThisFunction() {
-         if (onPubURLTagRouteChange == null || onSubURLTagRouteChange == null) {
-            int aVar = 2;
-            aVar = aVar + 2;
-         }
-      }
-
-
-   }
-}
+using System;
+using System.Text.RegularExpressions;
+using System.Collections;
+using System.Collections.Generic;
+
+using BCG;
+using Bts.PubSub;
+using bts;
+using System.Linq;
+//using BCG.Utilities;
+//using bts;
+
+namespace BCG {
+   public class PubSubRouter : IPubSubRouter {
+      // private Regex[] mySubRouteRegExps;
+      private PubSubRoute[] _pub_routes;
+      private PubSubRoute[] _sub_routes;
+
+      public PubSubRouter() : this(
+         PubSubConfig.Instance.SubscriberRoutes?.ToArray()
+            ?? throw new InvalidOperationException("Subscriber routes are not configured"),
+         PubSubConfig.Instance.PublisherRoutes?.ToArray()
+            ?? throw new InvalidOperationException("Publisher routes are not configured")) { }
+
+      public PubSubRouter(params PubSubRoute[] routes) : this(routes, routes) { }
+
+      public PubSubRouter(PubSubRoute[] sub_routes, PubSubRoute[] pub_routes) {
+         string log_msg = String.Empty;
+         foreach (var route in sub_routes) {
+            log_msg += $" sub_route: {route}";
+         }
+         foreach (var route in pub_routes) {
+            log_msg += $" pub_route: {route}";
+         }
+         Log.Gen.info($"PubSubRouter routes: {log_msg}");
+         _sub_routes = sub_routes;
+         _pub_routes = pub_routes;
+      }
+
+      #region IPubSubRouter methods
+
+      public IEnumerable<PubSubRoute> sub_routes() {
+         return _sub_routes;
+      }
+
+      public RemoteConnectionInfo get_pub_route_for_tag(string tag) {
+         if (!_pub_routes.TryGetFirst(r => r?.IsMatch(tag) ?? false, out var route)) {
+            throw new ArgumentException($"Failed to find route for tag '{tag}'", nameof(tag));
+         }
+         Log.Gen.info($"PubSubRouter.get_sub_url_for_tag tag: {tag} route: {route}");
+         return RemoteConnectionInfo.parse_url(route.server_address);
+      }
+
+      public RemoteConnectionInfo get_sub_route_for_tag(string tag) {
+         if (!_sub_routes.TryGetFirst(r => r?.IsMatch(tag) ?? false, out var route)) {
+            throw new ArgumentException($"Failed to find route for tag '{tag}'", nameof(tag));
+         }
+         Log.Gen.info($"PubSubRouter.get_sub_url_for_tag tag: {tag} route: {route}");
+         return RemoteConnectionInfo.parse_url(route.server_address);
+      }
+
+      public List<string> get_all_pubsub_sub_urls() =>
+         _sub_routes.Select(r => r.server_address).ToList();
+
+      public List<string> get_all_pubsub_pub_urlss() =>
+         _pub_routes.Select(r => r.server_address).ToList();
+
+      public event PubSubTagURLChangeDelegate onSubURLTagRouteChange;
+      public event PubSubTagURLChangeDelegate onPubURLTagRouteChange;
+
+      #endregion
+
+      //this function is put here so that the events are used, in order to prevent build errors. 
+      private void myNeverCallThisFunction() {
+         if (onPubURLTagRouteChange == null || onSubURLTagRouteChange == null) {
+            int aVar = 2;
+            aVar = aVar + 2;
+         }
+      }
+
+
+   }
+}

src/DataStreamSuite/PubSub/Client/SubAddress.cs

@@ -1,31 +1,31 @@
-using System;
-using BCG;
-//using BCG.Utilities;
+// using System;
+// using BCG;
+// //using BCG.Utilities;
 
-namespace BCG
-{
-   public class InvalidSubAddress : Exception
-   {
-      public InvalidSubAddress(string msg) : base (msg){}
-   }
+// namespace BCG
+// {
+//    public class InvalidSubAddress : Exception
+//    {
+//       public InvalidSubAddress(string msg) : base (msg){}
+//    }
 
-   public abstract class SubAddress
-   {
-      public static SubAddress decode (string s)
-      {
-         //if (s.StartsWith(TcpSubAddress.prefix))
-         //   return new TcpSubAddress(s);
-         //else if (s.StartsWith(SuperSocketAddress.prefix))
-         //   return new SuperSocketAddress(s);
-         if (s.StartsWith(MSMQSubAddress.prefix))
-            return new MSMQSubAddress(s);
-         else if (s.StartsWith(BlutoSubAddress.prefix))
-            return new BlutoSubAddress(s);
-         else
-         {
-            //BlueEventLog.logAssert(false, "Unknown pubsub transport for : " + s);
-            return null;
-         }
-      }
-   }
-}
+//    public abstract class SubAddress
+//    {
+//       public static SubAddress decode (string s)
+//       {
+//          //if (s.StartsWith(TcpSubAddress.prefix))
+//          //   return new TcpSubAddress(s);
+//          //else if (s.StartsWith(SuperSocketAddress.prefix))
+//          //   return new SuperSocketAddress(s);
+//          if (s.StartsWith(MSMQSubAddress.prefix))
+//             return new MSMQSubAddress(s);
+//          else if (s.StartsWith(BlutoSubAddress.prefix))
+//             return new BlutoSubAddress(s);
+//          else
+//          {
+//             //BlueEventLog.logAssert(false, "Unknown pubsub transport for : " + s);
+//             return null;
+//          }
+//       }
+//    }
+// }

src/DataStreamSuite/PubSub/Config/PubSubConfig.cs

@@ -27,35 +27,39 @@ public class PubSubConfig {
    [JsonPropertyName("simple_echo_server")]
    public bool? SimpleEchoServer { get; set; }
 
+   // unused
    [JsonPropertyName("milliseconds_between_sub_info_updates")]
    public int? MillisecondsBetweenSubInfoUpdates { get; set; } = 1000;
 
+   // unused
    [JsonPropertyName("milliseconds_between_time_slice_updates")]
    public int? MillisecondsBetweenTimeSliceUpdates { get; set; } = 5000;
 
+   // unused
    [JsonPropertyName("milliseconds_between_no_buffer_space_retries")]
    public int? MillisecondsBetweenNoBufferRetries { get; set; } = 100;
 
+   // unused
    [JsonPropertyName("retries_on_no_buffer_space")]
    public int? RetriesOnNoBufferSpace { get; set; } = 1;
 
+   // unused
    [JsonPropertyName("do_it_all_seconds_between_log")]
    public int? DoItAllSecondsBetweenLog { get; set; } = 0;
 
-   // Milliseconds
+   // Used by the PubSubMessageReactor to pass to the ManagedTCPMessageHb (but unsed within it)
    [JsonPropertyName("heartbeat_interval")]
    public int? HeartbeatInterval { get; set; } = 15000;
 
    [JsonPropertyName("missing_heartbeats_before_gc")]
    public int? MissingHeartbeatsBeforeGc { get; set; } = 50;
 
-   [JsonPropertyName("use_managed_sockets")]
-   public bool? UseManagedSockets { get; set; } = true;
-
+   // unused
    // Milliseconds
    [JsonPropertyName("state_update_period")]
    public int? StateUpdatePeriod { get; set; } = 5000;
 
+   // unused
    // Milliseconds
    [JsonPropertyName("statistics_update_period")]
    public int? StatisticsUpdatePeriod { get; set; } = 5000;
@@ -64,67 +68,20 @@ public class PubSubConfig {
    [JsonPropertyName("disposed_connection_check_period")]
    public int? DisposedConnectionCheckPeriod { get; set; } = 1000;
 
-   // Milliseconds
-   [JsonPropertyName("message_processing_warning_threshold")]
-   public int? MessageProcessingWarningThreshold { get; set; } = 1000;
-
-   [JsonPropertyName("max_packet_size")]
-   public int? MaxPacketSize { get; set; } = 500000000;
-
-   [JsonPropertyName("blue_comm_must_die")]
-   public bool? BlueCommMustDie { get; set; } = true;
-
-   [JsonPropertyName("crushable_tags")]
-   public List<string>? CrushableTags { get; set; }
-
    [JsonPropertyName("tag_type_to_assembly_map")]
    public TagTypeToAssembly? TagTypeToAssemblyMap { get; set; }
 
-   [JsonPropertyName("server_queue")]
-   public string? ServerQueue { get; set; } = "PubSubQueue";
-
-   [JsonPropertyName("port_range")]
-   public PubSubPortRange? PortRange { get; set; }
-
-   [JsonPropertyName("bind_to_random_port")]
-   public bool? BindToRandomPort { get; set; } = false;
-
-   [JsonPropertyName("debug_tags")]
-   public string? DebugTags { get; set; }
-
+   // unused
    [JsonPropertyName("latency_report_threshold_ms")]
    public int? LatencyReportThresholdMs { get; set; }
 
-   [JsonPropertyName("use_substations")]
-   public bool? UseSubstations { get; set; }
-
-   [JsonPropertyName("use_dynamic_clients")]
-   public bool? UseDynamicClients { get; set; }
-
-   [JsonPropertyName("is_substation")]
-   public bool? IsSubstation { get; set; }
-
+   // unused
    [JsonPropertyName("internally_publish_sub_info")]
    public bool? InternallyPublishSubInfo { get; set; }
 
-   [JsonPropertyName("use_managed_clients_in_native_code")]
-   public bool? UseManagedClientsInNativeCode { get; set; }
-
    [JsonPropertyName("compressed_tags")]
    public string? CompressedTags { get; set; }
 
-   [JsonPropertyName("tags_to_report_compression_ratio")]
-   public string? TagsToReportCompressionRatio { get; set; }
-
-   [JsonPropertyName("disable_msmq")]
-   public bool? DisableMsmq { get; set; }
-
-   [JsonPropertyName("use_prepublished_server_state")]
-   public bool? UsePrepublishedServerState { get; set; }
-
-   [JsonPropertyName("ip_translation")]
-   public string? IpTranslation { get; set; }
-
    [JsonPropertyName("gc_activation_threshold")]
    public int? GcActivationThreshold { get; set; }
 
@@ -148,38 +105,22 @@ public class PubSubConfig {
          DoItAllSecondsBetweenLog = 0,
          HeartbeatInterval = 15000,
          MissingHeartbeatsBeforeGc = 50,
-         UseManagedSockets = true,
          StateUpdatePeriod = 5000,
          StatisticsUpdatePeriod = 5000,
          DisposedConnectionCheckPeriod = 1000,
-         MessageProcessingWarningThreshold = 1000,
          ServerPort = 12345,
-         MaxPacketSize = 500000000,
-         BlueCommMustDie = true,
-         CrushableTags = new(0),
          TagTypeToAssemblyMap = new(),
-         ServerQueue = "PubSubQueue",
-         PortRange = new(30400, 31700),
-         BindToRandomPort = false,
-         DebugTags = NOTAGS,
          LatencyReportThresholdMs = 1500,
-         UseSubstations = false,
-         IsSubstation = false,
          InternallyPublishSubInfo = true,
-         UseManagedClientsInNativeCode = false,
          CompressedTags = NOTAGS,
-         TagsToReportCompressionRatio = NOTAGS,
-         DisableMsmq = false,
-         UsePrepublishedServerState = false,
-         IpTranslation = null,
          GcActivationThreshold = 0,
          WaitPeriodAfterGc = 1000,
          SimpleEchoServer = false,
          PublisherRoutes = new() {
-            new("bluto://127.0.0.0.1:12345", ".*")
+            new("bts-cloud://192.168.1.80:12345", ".*")
          },
          SubscriberRoutes = new() {
-            new("bluto://127.0.0.0.1:12345", ".*")
+            new("bts-cloud://192.168.1.80:12345", ".*")
          }
       };
       return rv;
@@ -193,30 +134,14 @@ public class PubSubConfig {
       DoItAllSecondsBetweenLog ??= other.DoItAllSecondsBetweenLog;
       HeartbeatInterval ??= other.HeartbeatInterval;
       MissingHeartbeatsBeforeGc ??= other.MissingHeartbeatsBeforeGc;
-      UseManagedSockets ??= other.UseManagedSockets;
       StateUpdatePeriod ??= other.StateUpdatePeriod;
       StatisticsUpdatePeriod ??= other.StatisticsUpdatePeriod;
       DisposedConnectionCheckPeriod ??= other.DisposedConnectionCheckPeriod;
-      MessageProcessingWarningThreshold ??= other.MessageProcessingWarningThreshold;
       ServerPort ??= other.ServerPort;
-      MaxPacketSize ??= other.MaxPacketSize;
-      BlueCommMustDie ??= other.BlueCommMustDie;
-      CrushableTags ??= other.CrushableTags;
       TagTypeToAssemblyMap ??= other.TagTypeToAssemblyMap;
-      ServerQueue ??= other.ServerQueue;
-      PortRange ??= other.PortRange;
-      BindToRandomPort ??= other.BindToRandomPort;
-      DebugTags ??= other.DebugTags;
       LatencyReportThresholdMs ??= other.LatencyReportThresholdMs;
-      UseSubstations ??= other.UseSubstations;
-      IsSubstation ??= other.IsSubstation;
       InternallyPublishSubInfo ??= other.InternallyPublishSubInfo;
-      UseManagedClientsInNativeCode ??= other.UseManagedClientsInNativeCode;
       CompressedTags ??= other.CompressedTags;
-      TagsToReportCompressionRatio ??= other.TagsToReportCompressionRatio;
-      DisableMsmq ??= other.DisableMsmq;
-      UsePrepublishedServerState ??= other.UsePrepublishedServerState;
-      IpTranslation ??= other.IpTranslation;
       GcActivationThreshold ??= other.GcActivationThreshold;
       WaitPeriodAfterGc ??= other.WaitPeriodAfterGc;
       SimpleEchoServer ??= other.SimpleEchoServer;

src/DataStreamSuite/PubSub/ManagedTCP/AsyncTCPConnection.cs

@@ -0,0 +1,542 @@
+// using System;
+// using System.Collections.Generic;
+// using System.Text;
+// using System.Net;
+// using System.Net.Sockets;
+// using System.IO;
+// //using BCG.Utilities;
+// using BCG.Utilities.General;
+// using System.Runtime.InteropServices;
+
+// namespace BCG
+// {
+//    /// <summary>
+//    /// This class, mantains a connection to a peer socket, using
+//    /// the .Net async TCP library. It sends heartbeats at regular
+//    /// intervals (config). It is in active states when messages
+//    /// are sent and received without an error state. It assumed 
+//    /// dead state when either of the following happens
+//    /// BeginReceive returns a SocketError
+//    /// BeginSend returns a SocketError
+//    /// EndReceive returns a SocketError
+//    /// EndSend returns a SocketError
+//    /// As soon as it assumes dead state it stops sending or
+//    /// receiving any messages (including heartbeats).
+//    /// The user of this class can either detect that by
+//    /// missing heartbeats or checking the isConnected property.
+//    /// Missing should be the more general way, since if the
+//    /// peer dies in a 'silent' way, where none of the above 
+//    /// methods indicate an error, missing heartbeats is guaranteed
+//    /// to detect it.
+//    /// </summary>
+//    public class AsyncTCPConnection : ITCPConnection, IWorker, IDisposable
+//    {
+//       /// <summary>
+//       /// This is used as a state for the Async send. Reason being 
+//       /// that if onSend indicates that it could not send the full
+//       /// data (ie EndSend does not return the full size), the offset
+//       /// is used to resend the data
+//       /// </summary>
+//       public class AsyncMessage
+//       {
+//          public byte[] buffer;
+//          public int offset;
+//          public string messageId;
+//          public SendConfirmationDelegate onSendDel;
+
+//          public AsyncMessage(byte[] theBuffer, int theOffset)
+//             : this(theBuffer, theOffset, String.Empty, null) { }
+
+//          public AsyncMessage(byte[] theBuffer, int theOffset, string theMessageId, SendConfirmationDelegate theOnSendDel)
+//          {
+//             buffer = theBuffer;
+//             offset = theOffset;
+//             messageId = theMessageId;
+//             onSendDel = theOnSendDel;
+//          }
+
+
+//          public int length
+//          {
+//             get
+//             {
+//                return buffer.Length - offset;
+//             }
+//          }
+//       }
+
+//       enum ReadState
+//       {
+//          readingSize,
+//          readingData
+//       };
+
+//       #region work around for .Net socket's LocalEndPoint bug
+
+//       [StructLayout(LayoutKind.Sequential)]
+//       private struct in_addr
+//       {
+//          [MarshalAs(UnmanagedType.ByValArray, SizeConst = 4)]
+//          public byte[] sin_addr;
+//       }
+
+//       [StructLayout(LayoutKind.Sequential)]
+//       private struct sockaddr
+//       {
+//          public short sin_family;
+//          public ushort sin_port;
+//          public in_addr sin_addr;
+//          [MarshalAs(UnmanagedType.ByValArray, SizeConst = 8)]
+//          public byte[] sin_zero;
+//       }
+
+//       [DllImport("wsock32.dll")]
+//       private static extern int getsockname(IntPtr s, ref sockaddr name, ref int namelen);
+
+//       [DllImport("wsock32.dll")]
+//       private static extern int getpeername(IntPtr s, ref sockaddr name, ref int namelen);
+
+//       #endregion
+
+//       private readonly int myMaxMessageSize = PubSub3Config.instance.PS3_MaxPacketSize;
+//       private RemoteConnectionInfo myPeer, mySelf;
+//       private Socket mySock;
+//       private ITCPMessageHandler myMessageHandler;
+
+//       private byte[] myCurrentReadBuffer;
+//       private int myCurrentReadBufferOffset;
+//       private ReadState myCurrentReadState;
+
+//       private bool myAlive = true;
+
+//       private byte[] myHBMessage;
+//       private TimeSpan myHBTimeSpan;
+//       private MasterWorker myMasterWorker = MasterWorker.getInstance(true);
+//       private bool myHBReceived = false, myDataReceived = false, myHBSent = false;
+//       private DateTime myLastHBReceivedTime = new DateTime(1900, 1, 1);
+
+//       public AsyncTCPConnection(Socket sock, TimeSpan heartBeatTimeSpan)
+//       {
+//          myHBTimeSpan = heartBeatTimeSpan;
+//          mySock = sock;
+//          mySock.SetSocketOption(SocketOptionLevel.Socket, SocketOptionName.DontRoute, true);
+//          string ip;
+//          int port;
+//          getPeerAddress(out ip, out port);
+//          myPeer = new RemoteConnectionInfo(ip, port);
+//          getSelfAddress(out ip, out port);
+//          mySelf = new RemoteConnectionInfo(ip, port);
+//          // BlueEventLog.logEvent(String.Format("AsyncTCPConnection created : self = {0} , peer = {1}",
+//          //                                        mySelf.ToString(), myPeer.ToString()), PS3LogLevels.connection);
+//       }
+
+//       public override string ToString()
+//       {
+//          return String.Format("peer = {0}, self = {1}", myPeer.ToString(), mySelf.ToString());
+//       }
+
+//       #region ITCPConnection Members
+
+//       public void sendWithConfirmation(byte[] data, ref string confId, SendConfirmationDelegate del)
+//       {
+//          confId = Guid.NewGuid().ToString();
+//          byte[] msg = myBuildDataMessage(data);
+//          myBeginOrResumeSending(new AsyncMessage(msg, 0, confId, del));
+//       }
+
+//       public void send(byte[] data)
+//       {
+//          byte[] msg = myBuildDataMessage(data);
+//          myBeginOrResumeSending(new AsyncMessage(msg, 0));
+//       }
+
+//       public void register(ITCPMessageHandler messageHandler)
+//       {
+//          // BlueEventLog.logAssert(myMessageHandler == null, "Message Handler is already assigned");
+//          // BlueEventLog.logEvent(String.Format("AsyncTCPConnection for peer = {0} being registered", myPeer.ToString()),
+//          //                         PS3LogLevels.connection);
+//          myMessageHandler = messageHandler;
+//          myMasterWorker.addWorker(this, myHBTimeSpan);
+//          myReadSize();
+//       }
+
+//       public void getPeerAddress(out string ip, out int port)
+//       {
+//          ip = "";
+//          port = 0;
+//          if (myAlive)
+//          {
+//             IPEndPoint ipEp = mySock.RemoteEndPoint as IPEndPoint;
+//             ip = ipEp.Address.ToString();
+//             port = ipEp.Port;
+//          }
+//          ///.Net has some problems getting the port, so using
+//          ///native code for that
+//          //sockaddr addr = new sockaddr();
+//          //int namelen = 16;
+//          //getsockname(mySock.Handle, ref addr, ref namelen);
+//          //port = addr.sin_port;
+//       }
+
+//       public void getSelfAddress(out string ip, out int port)
+//       {
+//          ip = "";
+//          port = 0;
+//          if (myAlive)
+//          {
+//             IPEndPoint ipEp = mySock.LocalEndPoint as IPEndPoint;
+//             ip = ipEp.Address.ToString();
+//             port = ipEp.Port;
+//          }
+//          ///due to this bug in .Net socket, we have to resort to
+//          ///native code
+//          //sockaddr addr = new sockaddr();
+//          //int namelen = 16;
+//          //getsockname(mySock.Handle, ref addr, ref namelen);
+//          //port = addr.sin_port;
+//       }
+
+//       public bool isDisposed
+//       {
+//          get { return !myAlive; }
+//       }
+
+//       #endregion
+
+//       #region IWorker Members
+
+//       bool _last_connected = false;
+//       public void doWork()
+//       {
+//          if (mySock != null)
+//          //if (mySock.Connected != _last_connected)
+//          {
+//             Console.WriteLine($"AsyncTCPConnection.doWork connected: {mySock.Connected}");
+//             _last_connected = mySock.Connected;
+//          }
+//          if (!myHBSent)
+//          {
+//             // BlueEventLog.logEvent(String.Format("Sending heartbeat to peer {0} for the first time", myPeer.ToString()),
+//             //                         PS3LogLevels.connection);
+//             myHBSent = true;
+//          }
+//          myBeginOrResumeSending(new AsyncMessage(myBuildHeartbeatMessage(), 0));
+//       }
+
+//       #endregion
+
+//       #region sending data
+
+//       private void myBeginOrResumeSending(AsyncMessage message)
+//       {
+//          if (myAlive)
+//          {
+//             try
+//             {
+//                //Console.WriteLine($"AsyncTCPConnection.myBeginOrResumeSending len: {message.length} connected: {mySock.Connected}");
+//                SocketError beginSendErr;
+//                mySock.BeginSend(message.buffer, message.offset, message.length, SocketFlags.None,
+//                                  out beginSendErr, new AsyncCallback(myOnSend), message);
+//                Console.WriteLine($"AsyncTCPConnection.myBeginOrResumeSending len: {message.length} socker_error: {beginSendErr} connected: {mySock.Connected}");
+//                if (beginSendErr != SocketError.Success)
+//                   myHandleError(beginSendErr);
+//             }
+//             catch (Exception ex)
+//             {
+//                myHandleError(ex);
+//             }
+//          }
+//       }
+
+//       private void myOnSend(IAsyncResult res)
+//       {
+//          if (myAlive)
+//          {
+//             try
+//             {
+//                SocketError writeErr;
+//                int bytesSent = mySock.EndSend(res, out writeErr);
+//                Console.WriteLine($"AsyncTCPConnection.myOnSend bytes: {bytesSent} writeErr: {writeErr} connected: {mySock.Connected}");
+//                AsyncMessage message = res.AsyncState as AsyncMessage;
+//                message.offset += bytesSent;
+//                if (writeErr == SocketError.Success)
+//                {
+//                   if (message.offset == message.buffer.Length)
+//                   {
+//                      // if confirmation needs to be sent
+//                      if (String.Empty != message.messageId)
+//                      {
+//                         message.onSendDel(message.messageId, true);
+//                      }
+//                   }
+//                   else
+//                   {
+//                      ///resume writing rest of the message
+//                      // BlueEventLog.logEvent("could not send the whole message all at once , messageSize = " +
+//                      //    message.buffer.Length + " offset = " + message.offset, PS3LogLevels.info);
+//                      myBeginOrResumeSending(message);
+//                   }
+//                }
+//                else
+//                {
+//                   myHandleError(writeErr);
+//                }
+//             }
+//             catch (Exception ex)
+//             {
+//                myHandleError(ex);
+//             }
+//          }
+//       }
+
+//       /// <summary>
+//       /// Assembles the heartbeat message, and uses it for 
+//       /// future references
+//       /// </summary>
+//       /// <returns></returns>
+//       private byte[] myBuildHeartbeatMessage()
+//       {
+//          if (myHBMessage == null)
+//          {
+//             BinaryWriter bw = new BinaryWriter(new MemoryStream());
+//             int sz = 4;
+//             int msgType = 1;
+//             bw.Write(sz);
+//             bw.Write(msgType);
+//             myHBMessage = (bw.BaseStream as MemoryStream).ToArray();
+//          }
+//          return myHBMessage;
+//       }
+
+//       /// <summary>
+//       /// Assembles the data message. This is necessary
+//       /// for the other side to understand the message. This can also
+//       /// be achieved by not forming the message and use the 
+//       /// BeginSend (List<>) function rather than BeginSend (byte[])
+//       /// </summary>
+//       /// <param name="data"></param>
+//       /// <returns></returns>
+//       private byte[] myBuildDataMessage(byte[] data)
+//       {
+//          BinaryWriter bw = new BinaryWriter(new MemoryStream());
+//          bw.Write(data.Length + 8);
+//          bw.Write(0);
+//          bw.Write(0);   ///mid for backward compatibility
+//          bw.Write(data);
+//          return (bw.BaseStream as MemoryStream).ToArray();
+//       }
+
+//       #endregion
+
+//       #region receiving data
+
+//       private void myReadSize()
+//       {
+//          myCurrentReadBuffer = new byte[4];
+//          myCurrentReadBufferOffset = 0;
+//          myCurrentReadState = ReadState.readingSize;
+//          myBeginOrResumeReading();
+//       }
+
+//       private void myReadData(int dataSize)
+//       {
+//          myCurrentReadBuffer = new byte[dataSize];
+//          myCurrentReadBufferOffset = 0;
+//          myCurrentReadState = ReadState.readingData;
+//          myBeginOrResumeReading();
+//       }
+
+//       private void myBeginOrResumeReading()
+//       {
+//          if (myAlive)
+//          {
+//             try
+//             {
+//                int beginReceiveAttempts = 0;
+//                while (true)
+//                {
+//                   beginReceiveAttempts++;
+//                   SocketError readSockErr;
+//                   mySock.BeginReceive(myCurrentReadBuffer, myCurrentReadBufferOffset,
+//                                        myCurrentReadBuffer.Length - myCurrentReadBufferOffset,
+//                                        SocketFlags.None, out readSockErr,
+//                                        new AsyncCallback(myOnRead),
+//                                        myCurrentReadBuffer);
+//                   Console.WriteLine($"AsyncTCPConnection.myBeginOrResumeReading socket_error: {readSockErr} connected: {mySock.Connected}");
+//                   if (readSockErr == SocketError.NoBufferSpaceAvailable)
+//                   {
+//                      string msg = String.Format("Not enough buffer space to receive {0} bytes. Attempt: {1} of {2}.",
+//                         myCurrentReadBuffer.Length - myCurrentReadBufferOffset,
+//                         beginReceiveAttempts, PubSub3Config.instance.PS3_retriesOnNoBufferSpace + 1);
+//                      //BlueEventLog.logEvent(msg, PS3LogLevels.disconnection);
+
+//                      if (beginReceiveAttempts < (PubSub3Config.instance.PS3_retriesOnNoBufferSpace + 1))
+//                         System.Threading.Thread.Sleep(PubSub3Config.instance.PS3_millisecondsBetweenNoBufferSpaceRetries);
+//                      else
+//                      {
+//                         //BlueEventLog.logEvent("Could not recover from NoBufferSpace condition. Breaking connection.", PS3LogLevels.disconnection);
+//                         myHandleError(readSockErr);
+//                      }
+//                   }
+//                   else if (readSockErr == SocketError.Success)
+//                   {
+//                      if (beginReceiveAttempts > 1)
+//                         // BlueEventLog.logEvent(String.Format("Recovered from NoBufferSpace after {0} tries",beginReceiveAttempts-1) , 
+//                         //    PS3LogLevels.disconnection);
+//                         break;
+//                   }
+//                   else
+//                      myHandleError(readSockErr);
+//                }
+//             }
+//             catch (Exception ex)
+//             {
+//                myHandleError(ex);
+//             }
+//          }
+//       }
+
+//       private void myOnRead(IAsyncResult res)
+//       {
+//          if (myAlive)
+//          {
+//             try
+//             {
+//                SocketError readErr;
+//                Console.WriteLine($"AsyncTCPConnection.myOnRead pre connected: {mySock.Connected}");
+//                int bytesRead = mySock.EndReceive(res, out readErr);
+//                Console.WriteLine($"AsyncTCPConnection.myOnRead bytes: {bytesRead} readErr: {readErr} connected: {mySock.Connected}");
+//                myCurrentReadBufferOffset += bytesRead;
+//                if (readErr == SocketError.Success)
+//                {
+//                   if (myCurrentReadBufferOffset == myCurrentReadBuffer.Length)
+//                   {
+//                      ///read the whole of the buffer
+//                      if (myCurrentReadState == ReadState.readingSize)
+//                      {
+//                         ///the size has already been read, now convert
+//                         ///it into an int, and start reading the data
+//                         int dataSize = new BinaryReader(new MemoryStream(myCurrentReadBuffer)).ReadInt32();
+//                         if (dataSize > myMaxMessageSize)
+//                         {
+//                            string errMessage = "Encountered malformed message, size indicated  = " + dataSize;
+//                            //BlueEventLog.logEvent(errMessage, "Error");
+//                            myHandleError(new ApplicationException(errMessage));
+//                         }
+//                         else
+//                            myReadData(dataSize);
+//                      }
+//                      else
+//                      {
+//                         ///the whole data has been finally read, time to 
+//                         ///tell the subscriber about it
+//                         BinaryReader br = new BinaryReader(new MemoryStream(myCurrentReadBuffer));
+//                         int msgType = br.ReadInt32();
+//                         if (msgType == 0)
+//                         {
+//                            ///skips the message ID
+//                            br.ReadInt32();
+//                            byte[] data = br.ReadBytes(myCurrentReadBuffer.Length - 8);
+//                            ///callbacks can be made through thread pool to avoid this
+//                            ///thread from being blocked, by this callback
+//                            if (!myDataReceived)
+//                            {
+//                               myDataReceived = true;
+//                               Console.WriteLine($"AsyncTCPConnection Data received for the first time from peer {myPeer.ToString()}");
+//                            }
+//                            myMessageHandler.onData(new RemoteMessage(data, RemoteMessageType.Data, myPeer));
+
+//                         }
+//                         else if (msgType == 1)
+//                         {
+//                            if (!myHBReceived)
+//                            {
+//                               myHBReceived = true;
+//                               Console.WriteLine($"AsyncTCPConnection HB received for the first time from peer {myPeer.ToString()}");
+//                            }
+//                            myLastHBReceivedTime = DateTime.Now;
+//                            myMessageHandler.onData(new RemoteMessage(null, RemoteMessageType.Heartbeat, myPeer));
+//                         }
+//                         myReadSize();
+//                      }
+//                   }
+//                   else
+//                   {
+//                      ///still part of the buffer left to read
+//                      ///continue reading
+//                      myBeginOrResumeReading();
+//                   }
+//                }
+//                else
+//                {
+//                   myHandleError(readErr);
+//                }
+//             }
+//             catch (Exception ex)
+//             {
+//                myHandleError(ex);
+//             }
+//          }
+//       }
+
+//       #endregion
+
+//       #region IDisposable Members
+
+//       public void Dispose()
+//       {
+//          Console.WriteLine("AsyncTCPConnection.Dispose()");
+//          lock (this)
+//          {
+//             if (myAlive)
+//             {
+//                myMasterWorker.removeWorker(this);
+//                myAlive = false;
+//                // BlueEventLog.logEvent(String.Format("AsycnTCPConnection with peer {0} being disposed, last HB was received at {1}" , 
+//                //                                     myPeer.ToString(), myLastHBReceivedTime.ToLongTimeString()),
+//                //                         PS3LogLevels.disconnection);
+//                try
+//                {
+//                   mySock.Close();
+//                }
+//                catch (Exception ex)
+//                {
+//                   String.Format("Exception of type {0} thrown while trying to close socket connected to {1} , with" +
+//                      " the following exception msg {2}", ex.GetType().ToString(), myPeer.ToString(), ex.Message);
+//                   //BlueEventLog.logEvent(ex);
+//                   if (ex is OutOfMemoryException)
+//                      throw ex;
+//                }
+//             }
+//          }
+//       }
+
+//       #endregion
+
+//       private void myHandleError(object err)
+//       {
+//          lock (this)
+//          {
+//             string errMsg = "";
+//             if (err is Exception)
+//             {
+//                errMsg = String.Format("AsyncTCPConnection.Dispose Encountered an exception of type '{0}' on the socket connected to ip = {1} , port = {2} ; " +
+//                   "Exception message = {3} , if the socket has not been disposed, it will be disposed now", err.GetType(), myPeer.ip, myPeer.port, (err as Exception).Message);
+//                if (err is OutOfMemoryException)
+//                   throw (err as OutOfMemoryException);
+//             }
+//             else if (err is SocketError)
+//             {
+//                errMsg = String.Format("AsyncTCPConnection.Dispose Encountered a SocketError of type '{0}' on the socket connected to ip = {1} , port = {2} ; " +
+//                   "if the socket has not been disposed, it will be disposed now", err.ToString(), myPeer.ip, myPeer.port);
+//                Console.WriteLine($"AsyncTCPConnection.myHandleError errMsg: {errMsg}");
+//             }
+//             //BlueEventLog.logEvent(errMsg, PS3LogLevels.disconnection);
+//             if (myAlive)
+//             {
+//                Dispose();
+//             }
+//          }
+//       }
+//    }
+// }

src/DataStreamSuite/PubSub/ManagedTCP/AsyncTCPConnection2.cs

@@ -0,0 +1,248 @@
+// using BCG;
+
+// using Google.Protobuf;
+
+// using System;
+// using System.IO;
+// using System.Linq;
+// using System.Net;
+// using System.Net.Sockets;
+// using System.Runtime.CompilerServices;
+// using System.Text;
+// using System.Threading.Tasks;
+// using BCG;
+// using System.Xml.Serialization;
+// using System.Reflection;
+// using BCG.Utilities.General;
+// using System.Net.WebSockets;
+
+// namespace BCG {
+
+//    public class AsyncTCPConnection2 : ITCPConnection, IWorker, IDisposable {
+//       private Socket _socket;
+//       // tied to the socket above
+//       private NetworkStream _nw_stream;
+//       private readonly ConnectionSide _side;
+//       ITCPMessageHandler _message_handler;
+//       private int _msg_id = 0;
+//       RemoteConnectionInfo _peer_address;
+//       RemoteConnectionInfo _self_address;
+//       private TimeSpan _hb_period = TimeSpan.FromSeconds(1);
+//       private DateTime _last_hb_sent = DateTime.MinValue;
+//       private DateTime _last_hb_received = DateTime.MaxValue;
+//       private static byte _msg_type_data = 1;
+//       private static byte _msg_type_hb = 0;
+//       private bool _is_disposed = false;
+//       SyncQueue<QueuedMessage> _data_send_queue = new SyncQueue<QueuedMessage>();
+
+//       public class QueuedMessage {
+//          public string id;
+//          public SendConfirmationDelegate del;
+//          public byte[] data;
+
+//          public QueuedMessage(string id, SendConfirmationDelegate del, byte[] data) {
+//             id = id;
+//             del = del;
+//             data = data;
+//          }
+
+//          public QueuedMessage(byte[] data)
+//          : this(String.Empty, null, data) { }
+
+//       }
+
+//       public enum ConnectionSide {
+//          Client,
+//          Server
+//       }
+
+//       public AsyncTCPConnection2(Socket socket, ConnectionSide side) {
+//          _socket = socket;
+//          _nw_stream = new NetworkStream(socket);
+//          _side = side;
+//          IPEndPoint r_ep = socket.RemoteEndPoint as IPEndPoint;
+//          _peer_address = new RemoteConnectionInfo(r_ep.Address.ToString(), r_ep.Port);
+
+//          var loc_ep = socket.LocalEndPoint as IPEndPoint;
+//          _self_address = new RemoteConnectionInfo(loc_ep.Address.ToString(), loc_ep.Port);
+//       }
+
+//       public void register(ITCPMessageHandler message_handler) {
+//          _message_handler = message_handler;
+//          Task.Run(async () => _keep_reading());
+//          Task.Run(async () => await _keep_sending());
+//          //_keep_reading();
+//          // if add a _keep_sending threadis added then we can maek send a non async
+//          // methid making it much easier to integrate with the rest
+//       }
+
+//       // this thread pulls 
+//       private async Task _keep_sending() {
+//          bts.Log.Gen.info($"TCPConnection._keep_sending starting");
+//          while (!_is_disposed) {
+//             try {
+//                QueuedMessage m = _data_send_queue.dequeue(100);
+//                if (m == null) {
+//                   continue;
+//                }
+//                await send_async(m.data);
+//                if (String.IsNullOrEmpty(m.id)) {
+//                   m.del(m.id, true);
+//                }
+//             } catch (Exception ex) {
+//                bts.Log.Gen.error($"TCPConnection._keep_sending ex: {ex.Message}");
+//                await Task.Delay(100);
+//                _handle_error(ex);
+//             }
+//          }
+//       }
+
+//       private async Task _keep_reading() {
+//          bts.Log.Gen.info($"TCPConnection._keep_reading starting");
+//          while (!_is_disposed) {
+//             try {
+//                byte[] msg = await TCPSocketUtils.read_bytes_async(_nw_stream);
+//                bts.Log.Gen.info($"AsyncTCPConnection2._keep_reading side: {_side} self: {get_self_address()} peer: {get_peer_address()} msg len: {msg.Length}");
+//                await _handle_msg(msg);
+//             } catch (Exception ex) {
+//                bts.Log.Gen.error($"TCPConnection._keep_reading ex: {ex.Message}");
+//                await Task.Delay(100);
+//                _handle_error(ex);
+//             }
+//          }
+//       }
+
+//       private async Task _handle_msg(byte[] msg) {
+//          // read the first byte to read the msg type and deal with it accordingly
+//          BinaryReader br = new BinaryReader(new MemoryStream(msg));
+//          byte msg_type = br.ReadByte();
+//          if (msg_type == _msg_type_data) {
+//             byte[] data = br.ReadBytes(msg.Length - 1);
+//             RemoteMessage m = new RemoteMessage(data, RemoteMessageType.Data, _peer_address);
+//             _message_handler.onData(m);
+//          } else {
+//             RemoteMessage m = new RemoteMessage(null, RemoteMessageType.Heartbeat, _peer_address);
+//             _message_handler.onData(m);
+//          }
+//       }
+
+//       // not very efficient
+//       private static byte[] _prefix_msg_type(byte[] data) {
+//          MemoryStream ms = new MemoryStream();
+//          BinaryWriter bw = new BinaryWriter(ms);
+//          bw.Write(_msg_type_data);
+//          bw.Write(data);
+//          ms.Position = 0;
+//          return ms.ToArray();
+//       }
+
+//       private async void _handle_error(Exception ex) {
+//          bts.Log.Gen.error($"TCPConnection._handle_error ex: {ex}");
+//          try {
+//             var socket = _nw_stream.Socket;
+//             socket.Close();
+//             socket.Dispose();
+//          } catch (Exception ex2) {
+//             bts.Log.Gen.error($"TCPConnection._handle_error got exception when closing Socket ex: {ex}");
+//          }
+//          _is_disposed = true;
+//          RemoteMessage rm = new RemoteMessage(new byte[] { }, RemoteMessageType.Disconnection, _peer_address);
+//          _message_handler.onData(rm);
+//       }
+
+//       public void send_with_confirmation(byte[] data, string confId, SendConfirmationDelegate del) { }
+
+//       public async Task send_with_confirmation_async(byte[] data, string conf_id, SendConfirmationDelegate del) {
+//          try {
+//             conf_id = (++_msg_id).ToString();
+//             await TCPSocketUtils.send_bytes_async(_nw_stream, data);
+//             del(conf_id, true);
+//          } catch (Exception ex) {
+//             _handle_error(ex);
+//          }
+//       }
+
+//       public void sendWithConfirmation(byte[] data, ref string conf_id, SendConfirmationDelegate del) {
+//          conf_id = (++_msg_id).ToString();
+//          send(data);
+//          del(conf_id, true);
+//       }
+
+//       public void send(byte[] data) {
+//          Task.Run(async () => await send_async(data));
+//          bts.Log.Gen.info($"AsyncTCPConnection2.send self: {get_self_address()} peer: {get_peer_address()} msg len: {data.Length} sent successfully");
+//          //_data_send_queue.enqueue(new QueuedMessage(data));
+//       }
+
+//       public async Task send_async(byte[] data) {
+//          // prefix 4 bytes for type (data vs hb)
+//          MemoryStream str = new MemoryStream();
+//          BinaryWriter bw = new BinaryWriter(str);
+//          bw.Write(_msg_type_data);
+//          bw.Write(data);
+//          try {
+//             await TCPSocketUtils.send_bytes_async(_nw_stream, str.ToArray());
+//          } catch (Exception ex) {
+//             bts.Log.Gen.error($"AsyncTCPConnection2.send_async ex: {ex}");
+//             _handle_error(ex);
+//          }
+//       }
+
+//       public void getPeerAddress(out string ip, out int port) {
+//          ip = _peer_address.ip;
+//          port = _peer_address.port;
+//          ///.Net has some problems getting the port, so using
+//          ///native code for that
+//          //sockaddr addr = new sockaddr();
+//          //int namelen = 16;
+//          //getsockname(mySock.Handle, ref addr, ref namelen);
+//          //port = addr.sin_port;
+//       }
+
+//       public RemoteConnectionInfo get_peer_address() {
+//          string ip; int port;
+//          getPeerAddress(out ip, out port);
+//          return new RemoteConnectionInfo(ip, port);
+//       }
+
+//       public RemoteConnectionInfo get_self_address() {
+//          string ip; int port;
+//          getSelfAddress(out ip, out port);
+//          return new RemoteConnectionInfo(ip, port);
+//       }
+
+
+//       public void getSelfAddress(out string ip, out int port) {
+//          ip = _self_address.ip;
+//          port = _self_address.port;
+//       }
+
+//       private async void _send_hb() {
+//          if (_last_hb_sent - DateTime.Now > _hb_period) {
+//             MemoryStream ms = new MemoryStream();
+//             BinaryWriter bw = new BinaryWriter(ms);
+//             bw.Write(_msg_type_hb);
+//             ms.Position = 0;
+//             byte[] hb_msg = ms.ToArray();
+//             await TCPSocketUtils.send_bytes_async(_nw_stream, hb_msg);
+//          }
+//       }
+
+//       public void doWork() {
+//          _send_hb();
+//       }
+
+//       public void Dispose() {
+//          _nw_stream.Close();
+//          _nw_stream.Dispose();
+//          _is_disposed = true;
+//       }
+
+//       public bool isDisposed {
+//          get {
+//             return _is_disposed;
+//          }
+//       }
+
+//    }
+// }

src/DataStreamSuite/PubSub/ManagedTCP/AsyncTCPConnectionAcceptor_APM.cs

@@ -15,6 +15,7 @@ namespace bts {
         private ITCPConnectionHandler _connection_handler;
         private HashSet<string> _key_map = new HashSet<string>();
         public AsyncTCPConnectionAcceptor_APM(int port, ITCPConnectionHandler connection_handler) {
+            Log.Gen.info($"AsyncTCPConnectionAcceptor_APM. cons started on port: {port}");
             _port = port;
             _connection_handler = connection_handler;
         }
@@ -25,7 +26,9 @@ namespace bts {
         }
 
         private void _init() {
+            Log.Gen.info($"AsyncTCPConnectionAcceptor_APM._init started on port: {_port}");
             _accept_socket = new Socket(AddressFamily.InterNetwork, SocketType.Stream, ProtocolType.Tcp);
+            TCPSocketUtils.set_server_socket_options(ref _accept_socket);
             IPEndPoint endpoint = new IPEndPoint(IPAddress.Any, _port);
             _accept_socket.Bind(endpoint);
             _accept_socket.Listen(250);
@@ -33,6 +36,7 @@ namespace bts {
 
         private void _begin_accepting() {
             var accept_key = Guid.NewGuid().ToString();
+            Log.Gen.info($"AsyncTCPConnectionAcceptor_APM._begin_accepting port: {_port} accept_key: {accept_key}");
             var res = _accept_socket.BeginAccept(new AsyncCallback(_on_accept), accept_key);
             if (res != null && res.IsCompleted && res.CompletedSynchronously)
                 _on_accept(res);
@@ -62,11 +66,8 @@ namespace bts {
             _connection_handler.onConnection(newConn);
             // go back to accepting more conns
             _begin_accepting();
-            string ip; int port;
-            newConn.getPeerAddress(out ip, out port);
-            string connMsg = String.Format("Connection to port {2} made from {0}:{1}", ip, port, _port);
-            //         BlueEventLog.logEvent(connMsg, PS3LogLevels.connection);
-            //Console.WriteLine(connMsg);       
+            var peer_rci = newConn.get_peer_address();
+            Log.Gen.info($"AsyncTCPConnectionAcceptor_APM._on_accept peer_rci: {peer_rci}");
         }
 
 

src/DataStreamSuite/PubSub/ManagedTCP/AsyncTCPConnectionAcceptor_AsyncWrapper.cs

@@ -0,0 +1,58 @@
+using System.Net.Sockets;
+using System.Net;
+
+using BCG;
+using System;
+using System.Collections.Generic;
+using System.Threading.Tasks;
+
+
+namespace bts {
+
+   // opens a TCP port and starts listening to connections. Unlike AsyncTCPConnectionAcceptor_APM,
+   // it uses TcpListener, instead of raw sockets.
+   public class AsyncTCPConnectionAcceptor_AsyncWrapper {
+      private readonly int _accept_port;
+      private Socket _accept_socket;
+      private ITCPConnectionHandler _connection_handler;
+      private TcpListener _server;
+      private Action<ITCPConnection> _cbs = (c) => { };
+
+      public AsyncTCPConnectionAcceptor_AsyncWrapper(int accept_port, ITCPConnectionHandler connection_handler) {
+         Log.Gen.info($"AsyncTCPConnectionAcceptor_AsyncWrapper.cons started on port: {accept_port}");
+         _accept_port = accept_port;
+         _connection_handler = connection_handler;
+      }
+
+      public void resgiter_callback(Action<ITCPConnection> cb) {
+         _cbs += cb;
+      }
+
+      public void start() {
+         Log.Gen.info($"AsyncTCPConnectionAcceptor_AsyncWrapper._init started on port: {_accept_port}");
+         _server = new TcpListener(IPAddress.Any, _accept_port);
+         _server.Start();
+         Task.Run(() => _accept_connections_forever());
+      }
+
+      private void _accept_connections_forever() {
+         Log.Gen.info($"AsyncTCPConnectionAcceptor_AsyncWrapper._begin_accepting port: {_accept_port}");
+         while (true) {
+            // this is a blocking call
+            Log.Gen.info($"AsyncTCPConnectionAcceptor_AsyncWrapper._begin_accepting port: {_accept_port} starting blocking call AcceptTcpClient");
+            TcpClient tcp_client = _server.AcceptTcpClient();
+            ITCPConnection conn = _create_connection(tcp_client);
+            Log.Gen.info($"AsyncTCPConnectionAcceptor_AsyncWrapper._begin_accepting port: {_accept_port}, new connection, self: {conn.get_self_address()} peer: {conn.get_peer_address()} calling onConnection now");
+            _connection_handler.onConnection(conn);
+         }
+         //_cbs(conn);
+      }
+
+      private static ITCPConnection _create_connection(TcpClient tcp_client) {
+         ITCPConnection conn = new AsyncTCPConnection_AsyncWrapper(tcp_client);
+         // ITCPConnection conn = new AsyncTCPConnection_APM(tcp_client.GetStream().Socket, TimeSpan.FromSeconds(1), ITCPConnection.ConnectionSide.Client);
+         return conn;
+      }
+   }
+
+}

src/DataStreamSuite/PubSub/ManagedTCP/AsyncTCPConnection_APM.cs

@@ -36,14 +36,17 @@ using System.Linq;
 using Bts.PubSub;
 using Spark.Core;
 
-namespace bts {
-   public class AsyncTCPConnection_APM : ITCPConnection, IWorker, IDisposable {
+namespace bts
+{
+   public class AsyncTCPConnection_APM : ITCPConnection, IWorker, IDisposable
+   {
 
       // this class is used as the argument of the completion function for BeginSend/Receive   
       // serves as the base class for both Send and ReeceiveMessage classes.
       // since both TCP sends and receives happen in multiple transfers, this
       // provides a wrapper around copying the partial transfers into the complete buffer
-      public class AsyncMessageBase {
+      public class AsyncMessageBase
+      {
          readonly string msg_id = Guid.NewGuid().ToString();
          // every BeginSend or BeginReceive has a unique key associated with it
          public int key;
@@ -55,37 +58,45 @@ namespace bts {
          public int transfer_ptr;
          protected int _n_transfers = 0;
          public bool completed_sync = false;
-         public AsyncMessageBase(byte[] the_buffer, int the_offset) {
+         public AsyncMessageBase(byte[] the_buffer, int the_offset)
+         {
             buffer = the_buffer;
             transfer_ptr = the_offset;
          }
-         public int remaining_size {
-            get {
+         public int remaining_size
+         {
+            get
+            {
                return buffer.Length - transfer_ptr;
             }
          }
 
          public int original_size => buffer.Length;
 
-         public void advance_transfer_pointer(int advance_by) {
+         public void advance_transfer_pointer(int advance_by)
+         {
             transfer_ptr += advance_by;
             _n_transfers++;
-            if (transfer_ptr > original_size) {
+            if (transfer_ptr > original_size)
+            {
                bts.Log.Gen.error($"transfer went beyond initial size {ToString()}");
             }
             // TODO: log if it goes past the buffer
          }
 
-         public bool fully_transfered() {
+         public bool fully_transfered()
+         {
             return transfer_ptr >= original_size;
          }
 
-         public override string ToString() {
+         public override string ToString()
+         {
             return $"{GetType().ToString()} msg_id: {msg_id} key: {key} original_length: {original_size} transfer_ptr: {transfer_ptr} n_transfers: {_n_transfers} " +
                    $"fully_transfered: {fully_transfered()} completed_sync: {completed_sync}";
          }
       }
-      public class AsyncSendMessage : AsyncMessageBase {
+      public class AsyncSendMessage : AsyncMessageBase
+      {
          public string message_id;
          public SendConfirmationDelegate on_send_del;
          //private int _n_transfers = 0;
@@ -94,35 +105,43 @@ namespace bts {
             : this(the_buffer, the_offset, String.Empty, null) { }
 
          public AsyncSendMessage(byte[] the_buffer, int the_transfer_ptr, string the_message_id, SendConfirmationDelegate the_on_send_del)
-         : base(the_buffer, the_transfer_ptr) {
+         : base(the_buffer, the_transfer_ptr)
+         {
             message_id = the_message_id;
             on_send_del = the_on_send_del;
          }
-         public override string ToString() {
+         public override string ToString()
+         {
             return base.ToString();
          }
 
       }
 
-      public class AsyncReceiveMessage : AsyncMessageBase {
-         public enum ReadState {
+      public class AsyncReceiveMessage : AsyncMessageBase
+      {
+         public enum ReadState
+         {
             reading_size,
             reading_data
          };
          readonly ReadState read_state;
 
          public AsyncReceiveMessage(int size, ReadState the_read_state) :
-         this(new byte[size], 0, the_read_state) { }
+         this(new byte[size], 0, the_read_state)
+         { }
          public AsyncReceiveMessage(byte[] the_buffer, int the_transfer_ptr, ReadState the_read_state)
-         : base(the_buffer, the_transfer_ptr) {
+         : base(the_buffer, the_transfer_ptr)
+         {
             read_state = the_read_state;
          }
 
-         public int read_int32(int start_idx = 0) {
+         public int read_int32(int start_idx = 0)
+         {
             return BitConverter.ToInt32(buffer, start_idx);
          }
 
-         public byte[] read_bytes(int start_idx) {
+         public byte[] read_bytes(int start_idx)
+         {
             byte[] rv = new byte[buffer.Length - start_idx];
             Array.Copy(buffer, start_idx, rv, 0, rv.Length);
             return rv;
@@ -131,16 +150,19 @@ namespace bts {
          public bool is_reading_size() => read_state == ReadState.reading_size;
       }
 
-      public class ErrorMessage {
+      public class ErrorMessage
+      {
          public readonly SocketError? socket_error = null;
          public readonly Exception exception = null;
          public readonly bool is_exception = false;
          public readonly string context = String.Empty;
-         public ErrorMessage(SocketError socket_error_, string context_) {
+         public ErrorMessage(SocketError socket_error_, string context_)
+         {
             socket_error = socket_error_;
             context = context_;
          }
-         public ErrorMessage(Exception exception_, string context_) {
+         public ErrorMessage(Exception exception_, string context_)
+         {
             is_exception = true;
             exception = exception_;
             context = context_;
@@ -148,46 +170,20 @@ namespace bts {
 
          public bool oom => exception != null && exception is OutOfMemoryException;
 
-         public override string ToString() {
+         public override string ToString()
+         {
             if (is_exception)
                return $"ErrorMessage exception: {exception} context: {context}";
             return $"socket_error: {socket_error} context: {context}";
          }
-
-
       }
 
-      #region work around for .Net socket's LocalEndPoint bug
-
-      // [StructLayout(LayoutKind.Sequential)]
-      // private struct in_addr {
-      //    [MarshalAs(UnmanagedType.ByValArray, SizeConst = 4)]
-      //    public byte[] sin_addr;
-      // }
-
-      // [StructLayout(LayoutKind.Sequential)]
-      // private struct sockaddr {
-      //    public short sin_family;
-      //    public ushort sin_port;
-      //    public in_addr sin_addr;
-      //    [MarshalAs(UnmanagedType.ByValArray, SizeConst = 8)]
-      //    public byte[] sin_zero;
-      // }
-
-      // [DllImport("wsock32.dll")]
-      // private static extern int getsockname(IntPtr s, ref sockaddr name, ref int namelen);
-
-      // [DllImport("wsock32.dll")]
-      // private static extern int getpeername(IntPtr s, ref sockaddr name, ref int namelen);
-
-      #endregion
-
       private const int DATA_MSG_SIGNAL = 0;
       private const int HB_MSG_SIGNAL = 1;
       private int _send_key = 0, _receive_key = 0;
       private HashSet<int> _read_key_map = new HashSet<int>(),
       _send_key_map = new HashSet<int>();
-      private readonly int _max_msg_size = PubSubConfig.Instance.MaxPacketSize.assert_value();
+      private readonly int _max_msg_size = Int32.MaxValue;// PubSubConfig.Instance.MaxPacketSize.assert_value();
       private readonly RemoteConnectionInfo _peer_conn_info, _self_conn_info;
       private readonly ITCPConnection.ConnectionSide _connection_side;
       private readonly Socket _sock;
@@ -201,38 +197,69 @@ namespace bts {
       private bool _hb_received_yet = false, _data_received_yet = false, _hb_sent_once = false;
       private DateTime _last_hb_received_time = new DateTime(1900, 1, 1);
 
-      public AsyncTCPConnection_APM(Socket sock, TimeSpan hb_period, ITCPConnection.ConnectionSide connection_side) {
+      public AsyncTCPConnection_APM(Socket sock, TimeSpan hb_period, ITCPConnection.ConnectionSide connection_side)
+      {
          _sock = sock;
          _hb_period = hb_period;
          _connection_side = connection_side;
          _sock.SetSocketOption(SocketOptionLevel.Socket, SocketOptionName.DontRoute, true);
-         string ip;
-         int port;
-         getPeerAddress(out ip, out port);
-         _peer_conn_info = new RemoteConnectionInfo(ip, port);
-         getSelfAddress(out ip, out port);
-         _self_conn_info = new RemoteConnectionInfo(ip, port);
+         #region testing options
+         switch (connection_side)
+         {
+            case ITCPConnection.ConnectionSide.Server:
+               TCPSocketUtils.set_server_socket_options(ref _sock);
+               break;
+            case ITCPConnection.ConnectionSide.Client:
+               TCPSocketUtils.set_client_socket_options(ref _sock);
+               break;
+         }
+         #endregion
+         _peer_conn_info = get_peer_address();
+         _self_conn_info = get_self_address();
          bts.Log.Gen.info($"AsyncTCPConnection_APM created {ToString()}");
+         _detect_connection_status_change();
       }
 
-      public override string ToString() {
+      private void _detect_connection_status_change()
+      {
+         Action testing_func = () =>
+         {
+            bool connected = _sock.Connected;
+            while (true)
+            {
+               if (connected != _sock.Connected)
+                  Log.Gen.error($"Connection status changed this: {ToString()} {connected} -> {_sock.Connected}");
+               connected = _sock.Connected;
+               System.Threading.Thread.Sleep(50);
+            }
+         };
+         ThreadPool.QueueUserWorkItem(_ => testing_func());
+      }
+
+      public override string ToString()
+      {
          return $"AsyncTCPConnection_APM side: {_connection_side} peer: {_peer_conn_info}, self: {_self_conn_info}";
       }
 
       #region ITCPConnection Members
 
-      public void sendWithConfirmation(byte[] data, ref string conf_id, SendConfirmationDelegate del) {
+      public void sendWithConfirmation(byte[] data, ref string conf_id, SendConfirmationDelegate del)
+      {
          conf_id = Guid.NewGuid().ToString();
          byte[] msg = _build_data_message(data);
          _begin_or_resume_sending(new AsyncSendMessage(msg, 0, conf_id, del));
       }
 
-      public void send(byte[] data) {
+      public void send(byte[] data)
+      {
+         //Log.Gen.info($"AsyncTCPConnection_APM.send data: {AsyncTCPConnection_AsyncWrapper.to_hex(data, 0, data.Length)}");
          byte[] msg = _build_data_message(data);
+         //Log.Gen.info($"AsyncTCPConnection_APM.send msg: {AsyncTCPConnection_AsyncWrapper.to_hex(msg, 0, msg.Length)}");
          _begin_or_resume_sending(new AsyncSendMessage(msg, 0));
       }
 
-      public void register(ITCPMessageHandler messageHandler) {
+      public void register(ITCPMessageHandler messageHandler)
+      {
          bts.Log.assert_d(_message_handler == null, "Message Handler is already assigned");
          //bts.Log.Gen.info($"AsyncTCPConnection_APM for peer = {_peer_conn_info} being registered");
          _message_handler = messageHandler;
@@ -241,27 +268,18 @@ namespace bts {
          _read_next_msg();
       }
 
-      public void getPeerAddress(out string ip, out int port) {
-         ip = "";
-         port = 0;
-         if (_is_alive) {
-            IPEndPoint ipEp = _sock.RemoteEndPoint as IPEndPoint;
-            port = ipEp.Port;
-            ip = ipEp.Address.ToString();
-         }
+      public RemoteConnectionInfo get_peer_address()
+      {
+         return TCPSocketUtils.get_peer_address(_sock);
       }
 
-      public void getSelfAddress(out string ip, out int port) {
-         ip = "";
-         port = 0;
-         if (_is_alive) {
-            IPEndPoint ipEp = _sock.LocalEndPoint as IPEndPoint;
-            ip = ipEp.Address.ToString();
-            port = ipEp.Port;
-         }
+      public RemoteConnectionInfo get_self_address()
+      {
+         return TCPSocketUtils.get_peer_address(_sock);
       }
 
-      public bool isDisposed {
+      public bool isDisposed
+      {
          get { return !_is_alive; }
       }
 
@@ -270,12 +288,14 @@ namespace bts {
       #region IWorker Members
 
       bool _last_connected = false;
-      public void doWork() {
+      public void doWork()
+      {
          if (!_is_alive)
             return;
          return;
          // disable hbs for now
-         if (!_hb_sent_once) {
+         if (!_hb_sent_once)
+         {
             bts.Log.Gen.info("Sending heartbeat to peer {_peer_conn_info} for the first time");
             _hb_sent_once = true;
          }
@@ -283,12 +303,14 @@ namespace bts {
          _begin_or_resume_sending(new AsyncSendMessage(msg, 0));
       }
 
-      private static int _tid() {
+      private static int _tid()
+      {
          var tid = System.Threading.Thread.CurrentThread.ManagedThreadId;
          return tid;
       }
 
-      private static string _cb_txt(IAsyncResult res) {
+      private static string _cb_txt(IAsyncResult res)
+      {
          return $"completed?: {res.IsCompleted} sync?: {res.CompletedSynchronously}";
       }
 
@@ -296,10 +318,13 @@ namespace bts {
 
       #region sending data
 
-      private void _begin_or_resume_sending(AsyncSendMessage message) {
+      private void _begin_or_resume_sending(AsyncSendMessage message)
+      {
          //bts.Log.Gen.info($"AsyncTCPConnection_APM._begin_or_resume_sending this: {ToString()} tid: {_tid()} message: {message} _is_alive: {_is_alive}");
-         if (_is_alive) {
-            try {
+         if (_is_alive)
+         {
+            try
+            {
                SocketError begin_send_err;
                // since we get message on on_send we dont need to keep the state around
                message.key = _send_key++;
@@ -307,35 +332,45 @@ namespace bts {
                                  out begin_send_err, new AsyncCallback(BeginSend_callback_function), message);
                if (begin_send_err != SocketError.Success)
                   _handle_error(new ErrorMessage(begin_send_err, "_begin_or_resume_sending"));
-            } catch (Exception ex) {
+            }
+            catch (Exception ex)
+            {
                _handle_error(new ErrorMessage(ex, "_begin_or_resume_sending"));
             }
          }
       }
 
-      private void BeginSend_callback_function(IAsyncResult res) {
-         if (res.CompletedSynchronously) {
+      private void BeginSend_callback_function(IAsyncResult res)
+      {
+         if (res.CompletedSynchronously)
+         {
             ThreadPool.QueueUserWorkItem(_ => _on_send_async(res));
-         } else
+         }
+         else
             _on_send_async(res);
       }
 
-      private void _on_send_async(IAsyncResult res) {
+      private void _on_send_async(IAsyncResult res)
+      {
          AsyncSendMessage message = res.AsyncState as AsyncSendMessage;
-         if (_send_key_map.Contains(message.key)) {
+         if (_send_key_map.Contains(message.key))
+         {
             bts.Log.Gen.warn($"AsyncTCPConnection_APM._on_send key: {message.key} already handled");
             return;
          }
          _send_key_map.Add(message.key);
          // bts.Log.Gen.info($"AsyncTCPConnection_APM._on_send: pre endSend this: {ToString()} tid: {_tid()} " +
          //                         $"res: {_cb_txt(res)} message: {res.AsyncState} _is_alive: {_is_alive}");
-         if (_is_alive) {
-            try {
+         if (_is_alive)
+         {
+            try
+            {
                SocketError send_err;
                int bytes_sent = _sock.EndSend(res, out send_err);
                //message.offset += bytesSent;
                message.advance_transfer_pointer(bytes_sent);
-               if (bytes_sent == 0) {
+               if (bytes_sent == 0)
+               {
                   bts.Log.Gen.error($"AsyncTCPConnection_APM._on_send this: {this} sent 0 bytes, throwing exception");
                   _handle_error(new ErrorMessage(new ApplicationException("Sent 0 bytes, peer socket is likely shutdown"),
                         "_on_send"));
@@ -343,22 +378,31 @@ namespace bts {
                }
                bts.Log.Gen.info($"AsyncTCPConnection_APM._on_send: post endSend this: {ToString()} tid: {_tid()} res: {_cb_txt(res)} " +
                $"message: {res.AsyncState} bytes_sent: {bytes_sent} _is_alive: {_is_alive} send_err: {send_err}");
-               if (send_err == SocketError.Success) {
+               if (send_err == SocketError.Success)
+               {
                   // successful send of some bytes
                   //if (message.offset == message.buffer.Length) {
-                  if (message.fully_transfered()) {
+                  if (message.fully_transfered())
+                  {
                      // fully sent, if confirmation needs to be sent out                     
-                     if (!String.IsNullOrEmpty(message.message_id)) {
+                     if (!String.IsNullOrEmpty(message.message_id))
+                     {
                         message.on_send_del(message.message_id, true);
                      }
-                  } else {
+                  }
+                  else
+                  {
                      // there is more to send
                      _begin_or_resume_sending(message);
                   }
-               } else {
+               }
+               else
+               {
                   _handle_error(new ErrorMessage(send_err, "_on_send"));
                }
-            } catch (Exception ex) {
+            }
+            catch (Exception ex)
+            {
                _handle_error(new ErrorMessage(ex, "_on_send"));
             }
          }
@@ -369,8 +413,10 @@ namespace bts {
       /// future references
       /// </summary>
       /// <returns></returns>
-      private byte[] _build_hb_message() {
-         if (_cached_hb_message == null) {
+      private byte[] _build_hb_message()
+      {
+         if (_cached_hb_message == null)
+         {
             BinaryWriter bw = new BinaryWriter(new MemoryStream());
             // the total msg =[msg_sz, msg_type] 
             int msg_sz = 4;
@@ -382,45 +428,54 @@ namespace bts {
       }
 
       // writes size prefixed msg
-      private byte[] _build_data_message(byte[] data) {
+      private byte[] _build_data_message(byte[] data)
+      {
          // TODO, avoid unecessary memory creation (write directly into byte[] using BitConverter)
-         BinaryWriter bw = new BinaryWriter(new MemoryStream());
+         MemoryStream str = new MemoryStream();
+         BinaryWriter bw = new BinaryWriter(str);
          // the total msg =[msg_sz, msg_type, data] 
-         var msg_sz = data.Length + 4;
+         Int32 msg_sz = data.Length + 4;
          bw.Write(msg_sz); // +4 for the type of msg
-         // to indicate data vs hb
+                           // to indicate data vs hb
          bw.Write(DATA_MSG_SIGNAL); // for data
-         // if we want to id every message, this is the place to do it
+                                    // if we want to id every message, this is the place to do it
          bw.Write(data);
-         return (bw.BaseStream as MemoryStream).ToArray();
+         str.Position = 0; //not needed really
+         return str.ToArray();
       }
 
       #endregion
 
       #region receiving data
 
-      private void _read_next_msg() {
+      private void _read_next_msg()
+      {
          _read_size();
       }
 
-      private void _read_size() {
+      private void _read_size()
+      {
          _begin_or_resume_reading(new AsyncReceiveMessage(4, AsyncReceiveMessage.ReadState.reading_size));
       }
 
-      private void _read_data(int data_size) {
+      private void _read_data(int data_size)
+      {
          // TODO, this could be a problem is the size is too big - maybe read it in chunks (later)
          _begin_or_resume_reading(new AsyncReceiveMessage(data_size, AsyncReceiveMessage.ReadState.reading_data));
       }
 
-      private static string IAsyncResult_txt(IAsyncResult res) {
+      private static string IAsyncResult_txt(IAsyncResult res)
+      {
          return $"IsCompleted: {res.IsCompleted} CompletedSynchronously: {res.CompletedSynchronously}";
       }
 
-      private void _begin_or_resume_reading(AsyncReceiveMessage rcv_msg) {
+      private void _begin_or_resume_reading(AsyncReceiveMessage rcv_msg)
+      {
          //bts.Log.Gen.info($"AsyncTCPConnection_APM._begin_or_resume_reading: this: {ToString()} tid: {_tid()} message: {rcv_msg} _is_alive: {_is_alive}");
          if (!_is_alive)
             return;
-         try {
+         try
+         {
             SocketError read_sock_err;
             rcv_msg.key = _receive_key++;
             var res = _sock.BeginReceive(rcv_msg.buffer, rcv_msg.transfer_ptr,
@@ -428,64 +483,94 @@ namespace bts {
                                  SocketFlags.None, out read_sock_err,
                                  new AsyncCallback(BeginReceive_callback_function),
                                  rcv_msg);
-            if (read_sock_err != SocketError.Success) {
+            if (read_sock_err != SocketError.Success)
+            {
                bts.Log.Gen.error($"_begin_or_resume_reading this: {this} error: {read_sock_err} rcv_msg: {rcv_msg}");
                _handle_error(new ErrorMessage(read_sock_err, "_beging_or_resume_reading"));
             }
-         } catch (Exception ex) {
+         }
+         catch (Exception ex)
+         {
             _handle_error(new ErrorMessage(ex, "_beging_or_resume_reading"));
          }
       }
 
+      private void _log_incoming_msg(IAsyncResult res)
+      {
+         // AsyncReceiveMessage rcv_message = res.AsyncState as AsyncReceiveMessage;
+         // var buff_txt = AsyncTCPConnection_AsyncWrapper.to_hex(rcv_message.buffer, 0, rcv_message.buffer.Length);
+         // Log.Gen.info($"BeginReceive_callback_function _log_incoming_msg buff: {buff_txt}");
+
+      }
+
       // this is the function passed to BeginReceive, we 
-      private void BeginReceive_callback_function(IAsyncResult res) {
-         if (res.CompletedSynchronously) {
+      private void BeginReceive_callback_function(IAsyncResult res)
+      {
+         _log_incoming_msg(res);
+         if (res.CompletedSynchronously)
+         {
             ThreadPool.QueueUserWorkItem(_ => _on_read_async(res));
-         } else
+         }
+         else
             _on_read_async(res);
       }
 
       // this will always be called on the non main-thread. the reason this is on its own thread is that this function ends up
       // calling _begin_or_resume_reading, which in tuns calls _on_read_async. So to not cause a stack overflow, this is broken
       // always called on a different thread than _begin_or_resume_reading
-      private void _on_read_async(IAsyncResult res) {
+      private void _on_read_async(IAsyncResult res)
+      {
          AsyncReceiveMessage rcv_message = res.AsyncState as AsyncReceiveMessage;
-         if (_read_key_map.Contains(rcv_message.key)) {
+         if (_read_key_map.Contains(rcv_message.key))
+         {
             bts.Log.Gen.warn($"AsyncTCPConnection_APM._on_read key: {rcv_message.key} already handled");
             return;
          }
          _read_key_map.Add(rcv_message.key);
          if (!_is_alive)
             return;
-         try {
+         try
+         {
             SocketError read_err;
             Log.Gen.info($"AsyncTCPConnection_APM._on_read pre-EndReceive rcv_message: {rcv_message} res: {IAsyncResult_txt(res)}");
             int bytes_read = _sock.EndReceive(res, out read_err);
             rcv_message.advance_transfer_pointer(bytes_read);
             Log.Gen.info($"AsyncTCPConnection_APM._on_read post-EndReceive rcv_message: {rcv_message} res: {IAsyncResult_txt(res)} bytes_read: {bytes_read}");
-            if (bytes_read == 0) {
+            if (bytes_read == 0)
+            {
                bts.Log.Gen.error($"AsyncTCPConnection_APM._on_read this: {this} read 0 bytes, throwing exception");
+               //bts.Log.Gen.error($"AsyncTCPConnection_APM._on_read this: {this} read 0 bytes, sleeping and ignoring");
+               //System.Threading.Thread.Sleep(50);
                _handle_error(new ErrorMessage(new ApplicationException("Read 0 bytes, peer socket is likely shutdown"), "_on_read"));
                return;
             }
             //return;
-            if (read_err == SocketError.Success) {
-               if (rcv_message.fully_transfered()) {
-                  if (rcv_message.is_reading_size()) {
+            if (read_err == SocketError.Success)
+            {
+               if (rcv_message.fully_transfered())
+               {
+                  if (rcv_message.is_reading_size())
+                  {
                      var msg_size = _on_read_size_completely(rcv_message);
-                     if (msg_size != null && msg_size.HasValue) {
+                     if (msg_size != null && msg_size.HasValue)
+                     {
                         _read_data(msg_size.Value);
                      }
                      return;
-                  } else {
+                  }
+                  else
+                  {
                      // data is read
                      int msg_type = rcv_message.read_int32();
                      bts.Log.assert(msg_type == HB_MSG_SIGNAL || msg_type == DATA_MSG_SIGNAL);
-                     if (msg_type == DATA_MSG_SIGNAL) {
+                     if (msg_type == DATA_MSG_SIGNAL)
+                     {
                         _on_read_data_completely(rcv_message);
                         _read_next_msg();
                         return;
-                     } else if (msg_type == HB_MSG_SIGNAL) {
+                     }
+                     else if (msg_type == HB_MSG_SIGNAL)
+                     {
                         _on_read_hb_completely(rcv_message);
                         _read_next_msg();
                         return;
@@ -493,43 +578,57 @@ namespace bts {
                      // go back to reading size for a new message
                      //_read_size();
                   }
-               } else {
+               }
+               else
+               {
                   // msg has not been fully rd, we have already advanced the transfer ptr
                   _begin_or_resume_reading(rcv_msg: rcv_message);
                }
-            } else {
-               _handle_error(new ErrorMessage(read_err, "_on_read"));
             }
-         } catch (Exception ex) {
-            _handle_error(new ErrorMessage(ex, "_on_read"));
+            else
+            {
+               _handle_error(new ErrorMessage(read_err, "_on_read_async"));
+            }
+         }
+         catch (Exception ex)
+         {
+            _handle_error(new ErrorMessage(ex, "_on_read_async"));
          }
       }
 
       #endregion
 
-      private int? _on_read_size_completely(AsyncReceiveMessage msg) {
+      private int? _on_read_size_completely(AsyncReceiveMessage msg)
+      {
          bts.Log.assert(msg.is_reading_size());
          bts.Log.assert(msg.fully_transfered());
          bts.Log.assert(msg.original_size == 4);
          var msg_sz = msg.read_int32();
+         //var buff_txt = AsyncTCPConnection_AsyncWrapper.to_hex(msg.buffer, 0, msg.buffer.Length);
+         //bts.Log.Gen.info($"AsyncTCPConnection_APM._on_read_size_completely buff: {buff_txt}");
          bts.Log.Gen.info($"AsyncTCPConnection_APM._on_read_size_completely msg: {msg} msg_sz: {msg_sz}");
-         if (msg_sz > _max_msg_size) {
+         if (msg_sz < 0 || msg_sz > _max_msg_size)
+         {
             string err_msg = $"Encountered malformed message, prefixed_size: {msg_sz}";
-            _handle_error(new ErrorMessage(new ApplicationException(err_msg), "_on_read"));
+            _handle_error(new ErrorMessage(new ApplicationException(err_msg), "_on_read_size_completely"));
             return null;
-         } else {
+         }
+         else
+         {
             // sz is determined, now we read that many bytes
             //_read_data(msg_sz);
             return msg_sz;
          }
       }
 
-      private void _on_read_hb_completely(AsyncReceiveMessage msg) {
+      private void _on_read_hb_completely(AsyncReceiveMessage msg)
+      {
          bts.Log.assert(!msg.is_reading_size());
          bts.Log.assert(msg.fully_transfered());
          bts.Log.assert(msg.original_size == 4);
          bts.Log.assert(msg.read_int32(0) == HB_MSG_SIGNAL);
-         if (!_hb_received_yet) {
+         if (!_hb_received_yet)
+         {
             _hb_received_yet = true;
             //bts.Log.Gen.info($"AsyncTCPConnection_APM HB received for the first time from peer {_peer_conn_info}");
          }
@@ -537,7 +636,8 @@ namespace bts {
          _message_handler.onData(new RemoteMessage(null, RemoteMessageType.Heartbeat, _peer_conn_info));
       }
 
-      private void _on_read_data_completely(AsyncReceiveMessage msg) {
+      private void _on_read_data_completely(AsyncReceiveMessage msg)
+      {
          bts.Log.assert(!msg.is_reading_size());
          bts.Log.assert(msg.fully_transfered());
          var type = msg.read_int32(0);
@@ -546,7 +646,8 @@ namespace bts {
          byte[] data = msg.read_bytes(4);
          ///callbacks can be made through thread pool to avoid this
          ///thread from being blocked, by this callback
-         if (!_data_received_yet) {
+         if (!_data_received_yet)
+         {
             _data_received_yet = true;
             //bts.Log.Gen.info($"AsyncTCPConnection_APM data received for the first time from peer {_peer_conn_info}");
          }
@@ -556,16 +657,24 @@ namespace bts {
 
       #region IDisposable Members
 
-      public void Dispose() {
+      public void Dispose()
+      {
          // socket is closed and disposed here
-         lock (this) {
-            if (_is_alive) {
+         lock (this)
+         {
+            bts.Log.Gen.info($"AsyncTCPConnection_APM.Dispose this: {this} Dispose called _dipose_impetus: {_dipose_impetus}");
+            if (_is_alive)
+            {
                _master_worker.removeWorker(this);
                _is_alive = false;
-               bts.Log.Gen.info($"AsycnTCPConnection.Dispose this: {this} being disposed, last HB was received at {_last_hb_received_time}");
-               try {
+               bts.Log.Gen.info($"AsyncTCPConnection_APM.Dispose this: {this} being disposed, last HB was received at {_last_hb_received_time}");
+               try
+               {
+                  bts.Log.Gen.info($"AsyncTCPConnection_APM.Dispose this: {this} shutting down socket now");
                   _sock.Shutdown(SocketShutdown.Both);
-               } catch (Exception ex) {
+               }
+               catch (Exception ex)
+               {
                   String.Format("AsyncTCPConnection_APM.Dispose exception: {ex} thrown when closing socket");
                   if (ex is OutOfMemoryException)
                      throw ex;
@@ -577,11 +686,15 @@ namespace bts {
 
       #endregion
 
-      private void _handle_error(ErrorMessage err_msg) {
-         lock (this) {
-            Log.Gen.info($"AsyncTCPConnection_APM._handle_error this: {this} error_msg: {err_msg}");
+      private string _dipose_impetus = String.Empty;
+      private void _handle_error(ErrorMessage err_msg)
+      {
+         lock (this)
+         {
+            Log.Gen.error($"AsyncTCPConnection_APM._handle_error this: {this} error_msg: {err_msg}");
             if (err_msg.oom)
                throw err_msg.exception;
+            _dipose_impetus = $"AsyncTCPConnection_APM._handle_error this: {this} error_msg: {err_msg}";
             Dispose();
          }
       }

src/DataStreamSuite/PubSub/ManagedTCP/AsyncTCPConnection_AsyncWrapper.cs

@@ -0,0 +1,339 @@
+namespace bts
+{
+   using BCG;
+   using BCG.Utilities.General;
+   using System;
+   using System.IO;
+   using System.Net.Sockets;
+   using System.Threading.Tasks;
+   using System.Threading;
+   using System.Net;
+   using System.Text;
+
+   // this ITCPConnection uses the TCPClient class instead of a raw socket (in the hope, that it would configure the socket correctly)
+
+   public class AsyncTCPConnection_AsyncWrapper : ITCPConnection, IWorker, IDisposable
+   {
+      public class ConnectionID
+      {
+         public readonly RemoteConnectionInfo self;
+         public readonly RemoteConnectionInfo peer;
+         public readonly Type conn_type;
+         public override string ToString()
+         {
+            return $"{conn_type} self: {self} peer: {peer}";
+         }
+         public ConnectionID(RemoteConnectionInfo self_, RemoteConnectionInfo peer_, Type conn_type_)
+         {
+            self = self_;
+            peer = peer_;
+            conn_type = conn_type_;
+         }
+      }
+      private const int DATA_MSG_SIGNAL = 0;
+      private const int HB_MSG_SIGNAL = 1;
+      private TcpClient _tcp_client;
+      private NetworkStream _nw_stream;
+      private ITCPMessageHandler _msg_handler;
+      private bool _is_alive = true;
+      ConnectionID _connection_id;
+
+      // public static string to_hex(byte[] byte_arr, int start_pos, int end_pos) {
+      //    var len = end_pos - start_pos;
+      //    StringBuilder hex_builder = new StringBuilder(len * 2);
+      //    int i = 0;
+
+      //    foreach (byte b in byte_arr) {
+      //       if (i >= start_pos && i < end_pos)
+      //          hex_builder.Append($"{b:X2} ");
+      //       i++;
+      //    }
+      //    return hex_builder.ToString().Trim();
+      // }
+
+      public AsyncTCPConnection_AsyncWrapper(TcpClient tcp_client)
+      {
+         _tcp_client = tcp_client;
+         _nw_stream = _tcp_client.GetStream();
+         _connection_id = new ConnectionID(TCPSocketUtils.get_self_address(_tcp_client.Client), TCPSocketUtils.get_peer_address(_tcp_client.Client), GetType());
+         Log.Gen.info("AsyncTCPConnection_AsyncWrapper.cons connection: {_connection_id}");
+      }
+
+      private void _detect_connection_status_change()
+      {
+         Action testing_func = () =>
+         {
+            bool connected = _tcp_client.Connected;
+            while (true)
+            {
+               if (connected != _tcp_client.Connected)
+                  Log.Gen.error($"Connection status changed this: {ToString()} {connected} -> {_tcp_client.Connected}");
+               connected = _tcp_client.Connected;
+               System.Threading.Thread.Sleep(50);
+               if (!connected)
+                  break;
+            }
+         };
+         ThreadPool.QueueUserWorkItem(_ => testing_func());
+      }
+
+      #region Sending 
+      public void _send_fixed_size_msg(byte[] buffer)
+      {
+         // string buff_txt = to_hex(buffer, 0, buffer.Length);
+         // Log.Gen.info($"AsyncTCPConnection_AsyncWrapper._send_fixed_size_msg buffer: {buff_txt}");
+
+         int total_bytes_sent = 0;
+         int n_bytes = buffer.Length;
+
+         int n_transfers = 0;
+         while (_is_alive && total_bytes_sent < n_bytes)
+         {
+            int remaining_bytes = n_bytes - total_bytes_sent;
+            _nw_stream.Write(buffer, total_bytes_sent, remaining_bytes);
+            total_bytes_sent += remaining_bytes;
+            n_transfers++;
+         }
+         Log.Gen.info($"AsyncTCPConnection_AsyncWrapper._send_fixed_size_msg sent {buffer.Length} bytes in {n_transfers} transfers");
+      }
+
+      private void _send_one_msg(byte[] msg)
+      {
+         try
+         {
+            // the total msg = [msg_sz, msg_type, data] 
+            int sz = msg.Length + 4;
+            MemoryStream str = new MemoryStream();
+            BinaryWriter bw = new BinaryWriter(str);
+            bw.Write(sz);
+            bw.Write(DATA_MSG_SIGNAL);
+            bw.Write(msg);
+            str.Position = 0;
+            byte[] buff = str.ToArray();
+            _send_fixed_size_msg(buff);
+         }
+         catch (Exception ex)
+         {
+            _handle_error(new ErrorMessage(ex, $"_send_one_msg call_stacK: {ex.StackTrace}"));
+         }
+      }
+
+      private void _send_forever()
+      {
+         while (_is_alive)
+         {
+            byte[] data = _send_queue.dequeue();
+            if (data != null)
+            {
+               _send_one_msg(data);
+            }
+         }
+      }
+
+      private SyncQueue<byte[]> _send_queue = new SyncQueue<byte[]>();
+      public void send(byte[] data) {
+         //Log.Gen.info($"AsyncTCPConnection_AsyncWrapper.send data: {to_hex(data, 0, data.Length}");
+         _send_queue.enqueue(data);
+      }
+      #endregion
+
+      public void register(ITCPMessageHandler message_handler)
+      {
+         _msg_handler = message_handler;
+         // starting thread pool threads to keep reading and sending
+         ThreadPool.QueueUserWorkItem(_ => _read_msgs_forever());
+         ThreadPool.QueueUserWorkItem(_ => _send_forever());
+      }
+
+      public RemoteConnectionInfo get_peer_address()
+      {
+         return _connection_id.peer;
+      }
+
+      public RemoteConnectionInfo get_self_address()
+      {
+         return _connection_id.self;
+      }
+
+      public bool isDisposed { get { return !_is_alive; } }
+
+      public void doWork()
+      {
+         // send hb
+      }
+
+      #region Reading
+      // This makes the blocking IO call, the callstack is _read_msgs_forever -> _read_one_message -> _read_fixed_size_msg
+      private byte[] _read_fixed_size_msg(int n_bytes)
+      {
+         if (n_bytes <= 0)
+         {
+            throw new ArgumentException("Number of bytes to read must be greater than zero.", nameof(n_bytes));
+         }
+         byte[] buffer = new byte[n_bytes];
+         int total_bytes_read = 0;
+         int n_transfers = 0;
+         while (_is_alive && total_bytes_read < n_bytes)
+         {
+            int bytes_read = _nw_stream.Read(buffer, total_bytes_read, n_bytes - total_bytes_read);
+            if (bytes_read <= 0)
+            {
+               throw new IOException("Read 0 bytes, the connection was closed before all data could be read.");
+            }
+            n_transfers++;
+            total_bytes_read += bytes_read;
+            Log.Gen.info($"AsyncTCPConnection_AsyncWrapper._read_fixed_size_msg n_bytes: {n_bytes} bytes_read: {bytes_read} total_bytes_read: {total_bytes_read} buff_len: {buffer.Length}");
+         }
+         Log.Gen.info($"AsyncTCPConnection_AsyncWrapper._read_fixed_size_msg read {n_bytes} bytes in {n_transfers} transfers");
+         return buffer;
+      }
+
+      private class OneRecvMessage
+      {
+         public int msg_type;
+         public byte[] data;
+         public OneRecvMessage(int msg_type_, byte[] data_)
+         {
+            msg_type = msg_type_;
+            data = data_;
+         }
+         public OneRecvMessage(byte[] msg)
+         {
+            MemoryStream str = new MemoryStream(msg);
+            BinaryReader br = new BinaryReader(str);
+            int msg_type = br.ReadInt32();
+            if (msg_type == DATA_MSG_SIGNAL)
+            {
+               // TODO, polish it, too much memory bneing allocated
+               data = br.ReadBytes(msg.Length - 4);
+            }
+         }
+
+         public override string ToString()
+         {
+            string rv = $"OneMessage msg_type: {msg_type} data_len: {data.Length}";
+            return rv;
+         }
+      }
+
+      private OneRecvMessage _read_one_message()
+      {
+         // read size first
+         byte[] sz_arr = _read_fixed_size_msg(4);
+         BinaryReader br = new BinaryReader(new MemoryStream(sz_arr));
+         br.BaseStream.Position = 0;
+         int msg_sz = br.ReadInt32();
+         Log.Gen.info($"AsyncTCPConnection_AsyncWrapper._read_one_message read msg_sz: {msg_sz}");
+         if (msg_sz < 0)
+         {
+            throw new IOException("Read 0 bytes, the connection was closed before all data could be read.");
+         }
+         byte[] msg = _read_fixed_size_msg(msg_sz);
+         Log.Gen.info($"AsyncTCPConnection_AsyncWrapper._read_one_message read msg sz: {msg.Length}");
+         // TODO, read first 4 bytes and decide hb or data
+         OneRecvMessage rv = new OneRecvMessage(msg);
+         return rv;
+      }
+
+      private void _read_msgs_forever()
+      {
+         while (_is_alive)
+         {
+            try
+            {
+               OneRecvMessage one_msg = _read_one_message();
+               if (one_msg.msg_type == DATA_MSG_SIGNAL)
+               {
+                  _msg_handler.onData(new RemoteMessage(one_msg.data, RemoteMessageType.Data, _connection_id.peer));
+               }
+               else if (one_msg.msg_type == HB_MSG_SIGNAL)
+               {
+                  // keep a last hb time an dudpate it here atomically
+                  _msg_handler.onData(new RemoteMessage(null, RemoteMessageType.Heartbeat, _connection_id.peer));
+               }
+               else
+               {
+                  throw new ApplicationException($"Unknown message type: {one_msg.msg_type}");
+               }
+            }
+            catch (Exception ex)
+            {
+               _handle_error(new ErrorMessage(ex, $"_read_msgs_forever, call_stack: {ex.StackTrace}"));
+            }
+         }
+         Log.Gen.info($"AsyncTCPConnection_AsyncWrapper._read_msgs_forever exiting loop");
+      }
+      #endregion
+
+      public bool is_disposed => !_is_alive;
+
+      public class ErrorMessage
+      {
+         public readonly SocketError? socket_error = null;
+         public readonly Exception exception = null;
+         public readonly bool is_exception = false;
+         public readonly string context = String.Empty;
+         public ErrorMessage(SocketError socket_error_, string context_)
+         {
+            socket_error = socket_error_;
+            context = context_;
+         }
+         public ErrorMessage(Exception exception_, string context_)
+         {
+            is_exception = true;
+            exception = exception_;
+            context = context_;
+         }
+
+         public bool oom => exception != null && exception is OutOfMemoryException;
+
+         public override string ToString()
+         {
+            if (is_exception)
+               return $"ErrorMessage exception: {exception} context: {context}";
+            return $"socket_error: {socket_error} context: {context}";
+         }
+      }
+
+      public void Dispose()
+      {
+         // socket is closed and disposed here
+         lock (this)
+         {
+            if (_is_alive)
+            {
+               //_master_worker.removeWorker(this);
+               _is_alive = false;
+               try
+               {
+                  bts.Log.Gen.info($"AsyncTCPConnection_APM.Dispose this: {this} closing tcp_client nin an orderly fashion now");
+                  _tcp_client.Close();
+               }
+               catch (Exception ex)
+               {
+                  String.Format("AsyncTCPConnection_APM.Dispose exception: {ex} thrown when closing socket");
+                  if (ex is OutOfMemoryException)
+                     throw ex;
+               }
+               _msg_handler.onData(new RemoteMessage(null, RemoteMessageType.Disconnection, _connection_id.peer));
+            }
+         }
+      }
+
+      private void _handle_error(ErrorMessage err_msg)
+      {
+         lock (this)
+         {
+            Log.Gen.error($"AsyncTCPConnection_APM._handle_error this: {this} error_msg: {err_msg}");
+            if (err_msg.oom)
+               throw err_msg.exception;
+            Dispose();
+         }
+      }
+
+      public override string ToString()
+      {
+         return _connection_id.ToString();
+      }
+   }
+}

src/DataStreamSuite/PubSub/ManagedTCP/AsyncTCPConnector_APM.cs

@@ -31,13 +31,13 @@ namespace bts {
             }
         }
 
-        public static void connect_to(BlutoSubAddress server_address, ITCPConnectionHandler handler) {
+        public static void connect_to(RemoteConnectionInfo server_address, ITCPConnectionHandler handler) {
             bts.Log.Gen.info($"AsyncTCPConnector_APM.connectTo server: {server_address}");
             // when connection is established, this sock passed to an ITCPCOnnection and sent back
             Socket sock = new Socket(AddressFamily.InterNetwork, SocketType.Stream, ProtocolType.Tcp);
-            EndPoint ep = new IPEndPoint(IPAddress.Parse(server_address.ip), server_address.port);
-            sock.BeginConnect(ep, new AsyncCallback(_on_connect),
-                            new SocketConnectionArgs(ep, sock, handler, server_address.ToString()));
+            EndPoint ip_ep = new IPEndPoint(IPAddress.Parse(server_address.ip), server_address.port);
+            sock.BeginConnect(ip_ep, new AsyncCallback(_on_connect),
+                            new SocketConnectionArgs(ip_ep, sock, handler, server_address.ToString()));
         }
 
         private static void _on_connect(IAsyncResult res) {

src/DataStreamSuite/PubSub/ManagedTCP/AsyncTCPConnector_AsyncWrapper.cs

@@ -0,0 +1,70 @@
+using System.Net.Sockets;
+using System.Net;
+
+using BCG;
+using System;
+using System.Collections.Generic;
+using System.Threading.Tasks;
+using System.Threading;
+using System.Reflection.Metadata.Ecma335;
+
+
+namespace bts {
+   // Unlike AsyncTCPConnectionAcceptor_APM this class is static since the requests to connect are stand alone requests
+   // whereas AsyncTCPConnectionAcceptor_APM is always listening for incoming connections which lends itself to more
+   // statefullness and hence non static
+   public static class AsyncTCPConnector_AsyncWrapper {
+      public static void connect_to(RemoteConnectionInfo server_address, ITCPConnectionHandler handler) {
+         bts.Log.Gen.info($"AsyncTCPConnector_AsyncWrapper.connect_to server: {server_address}");
+         //Task.Run(() => _connect_to(server_address, handler));
+         Action connect_once = () => { _connect_to(server_address, handler); };
+         ThreadPool.QueueUserWorkItem(_ => _keep_trying_blocking_function(connect_once, TimeSpan.FromSeconds(1), TimeSpan.FromSeconds(5)));
+      }
+
+      // this is the actual fuction thta calls TCPCLient sync methof and establishes a connection
+      private static void _connect_to(RemoteConnectionInfo server_rci, ITCPConnectionHandler handler) {
+         // TODO, random sleep
+         bts.Log.Gen.info($"AsyncTCPConnector_AsyncWrapper._connect_to server: {server_rci}");
+         TcpClient tcp_client = new TcpClient(server_rci.ip, server_rci.port);
+         bts.Log.Gen.info($"AsyncTCPConnector_AsyncWrapper._connect_to server new TcpClient returned, connected: {tcp_client.Connected}");
+         ITCPConnection conn = _create_connection(tcp_client);
+         var conn_rci = conn.get_peer_address();
+         bts.Log.Gen.info($"AsyncTCPConnector_AsyncWrapper._connect_to new connection, server_rci: {server_rci} connection.peer_address: {conn_rci}");
+         Log.assert(server_rci.Equals(conn_rci), $"{server_rci}!={conn_rci}");
+         bts.Log.Gen.info($"AsyncTCPConnector_AsyncWrapper._connect_to new connection, self: {conn.get_self_address()} peer: {conn.get_peer_address()}, calling onConnection now");
+         handler.onConnection(conn);
+      }
+
+      private static ITCPConnection _create_connection(TcpClient tcp_client) {
+         ITCPConnection conn = new AsyncTCPConnection_AsyncWrapper(tcp_client);
+         // ITCPConnection conn = new AsyncTCPConnection_APM(tcp_client.GetStream().Socket, TimeSpan.FromSeconds(1), ITCPConnection.ConnectionSide.Client);
+         return conn;
+      }
+
+      // this is a ganeric function that calls a sync action function, and if unsuccessful timesout after "timeout"
+      // wais for wait_before_retrying before trying again
+      public static void _keep_trying_blocking_function(Action action, TimeSpan timeout, TimeSpan wait_before_retrying) {
+         Func<bool> try_once = () => {
+            var cts = new CancellationTokenSource();
+            var token = cts.Token;
+
+            var task = Task.Run(action, token);
+
+            if (Task.WaitAny(task, Task.Delay(timeout, token)) == 0) {
+               // Task completed within timeout
+               task.GetAwaiter().GetResult(); // Ensure any exceptions/cancellation are rethrown
+               return true;
+            } else {
+               // Timeout
+               cts.Cancel();
+               return false;
+            }
+         };
+         bool connected = false;
+         while (!connected) {
+            connected = try_once();
+            System.Threading.Thread.Sleep(TimeSpan.FromSeconds(1));
+         }
+      }
+   }
+}

src/DataStreamSuite/PubSub/ManagedTCP/ClientToServerManagedTCPConnector.cs

@@ -0,0 +1,82 @@
+// using System;
+// using System.Collections.Generic;
+// using System.Text;
+// using System.Net;
+// using System.Net.Sockets;
+// using System.Threading;
+// using System.Threading.Tasks;
+// //using BCG.Utilities;
+// //using BCG.Utilities.General;
+
+// namespace BCG {
+//    internal class SocketConnectionArgs {
+//       public EndPoint ep;
+//       public Socket sock;
+//       public ITCPConnectionHandler handler;
+//       public string serverURL;
+
+//       internal SocketConnectionArgs(EndPoint theEp, Socket theSock, ITCPConnectionHandler theHandler,
+//          string theServerURL) {
+//          ep = theEp;
+//          sock = theSock;
+//          handler = theHandler;
+//          serverURL = theServerURL;
+//       }
+//    }
+
+//    public class ClientToServerManagedTCPConnector {
+//       private static Random myRnd = new Random(DateTime.Now.Millisecond);
+//       private static TimeSpan myHBTimeSpan = new TimeSpan(0, 0, 0, 0, PubSub3Config.instance.PS3_HeartBeatInterval);
+//       private static int myWaitTimeBeforeRetryingUpperLimit = 5000,
+//                            myWaitTimeBeforeRetryingLowerLimit = 1000;
+
+//       public static void connectTo(BlutoSubAddress serverAddress, ITCPConnectionHandler handler) {
+//          Console.WriteLine($"ClientToServerManagedTCPConnector.connectTo server: {serverAddress}");
+//          Socket sock = new Socket(AddressFamily.InterNetwork, SocketType.Stream, ProtocolType.Tcp);
+//          EndPoint ep = new IPEndPoint(IPAddress.Parse(serverAddress.ip), serverAddress.port);
+//          sock.BeginConnect(ep, new AsyncCallback(myOnConnect), new SocketConnectionArgs(ep, sock, handler, serverAddress.ToString()));
+//       }
+
+//       public static async Task connect_to_async(BlutoSubAddress serverAddress, ITCPConnectionHandler handler, TimeSpan wait_before_retrying, int max_n_attempts = 1024) {
+//          bts.Log.Gen.info($"ClientToServerManagedTCPConnector.connectTo server: {serverAddress}");
+//          await TCPSocketUtils.connect_to_server_async(serverAddress.ip, serverAddress.port, handler, wait_before_retrying, max_n_attempts);
+//       }
+
+//       public static void connect_to(BlutoSubAddress serverAddress, ITCPConnectionHandler handler, TimeSpan wait_before_retrying, int max_n_attempts = 1024) {
+//          bts.Log.Gen.info($"ClientToServerManagedTCPConnector.connectTo server: {serverAddress}");
+//          Task.Run(async () => {
+//             await TCPSocketUtils.connect_to_server_async(serverAddress.ip, serverAddress.port, handler, wait_before_retrying, max_n_attempts);
+//          });
+//       }
+
+//       private static void myOnConnect(IAsyncResult res) {
+//          Console.WriteLine("ClientToServerManagedTCPConnector.myOnConnect " + res.ToString() + " " + res.AsyncState.ToString());
+//          SocketConnectionArgs args = res.AsyncState as SocketConnectionArgs;
+//          try {
+//             args.sock.EndConnect(res);
+//          } catch (SocketException sex) {
+//             string errMsg = String.Format("{0}: Connection to {1} failed.", DateTime.Now.ToString(), args.serverURL.ToString());
+//             //BlueEventLog.logEvent(sex.Message + ";" + errMsg, PS3LogLevels.disconnection);
+//             //Console.WriteLine(errMsg);
+//             if (args.handler.isDisposed) {
+//                string doneMsg = String.Format("{0}: Connection to {1} was disposed. Not Retrying.", DateTime.Now.ToString(), args.serverURL.ToString());
+//                //Console.WriteLine(doneMsg);
+//                //               BlueEventLog.logEvent(doneMsg, PS3LogLevels.disconnection);
+//                return;
+//             } else {
+//                string retryMsg = String.Format("{0}: Retrying Connection to {1}.", DateTime.Now.ToString(), args.serverURL.ToString());
+//                //Console.WriteLine(retryMsg);
+//                //BlueEventLog.logEvent(retryMsg, PS3LogLevels.disconnection);
+
+//                int waitTime = myRnd.Next(myWaitTimeBeforeRetryingLowerLimit, myWaitTimeBeforeRetryingUpperLimit);
+//                Thread.Sleep(waitTime);
+//                args.sock.BeginConnect(args.ep, new AsyncCallback(myOnConnect), args);
+//                return;
+//             }
+//          }
+//          //BlueEventLog.logEvent(String.Format("Successful connection made to {0}.", args.serverURL.ToString()), PS3LogLevels.connection);
+//          args.handler.onConnection(new bts.AsyncTCPConnection_APM(args.sock, myHBTimeSpan, ITCPConnection.ConnectionSide.Client));
+//       }
+
+//    }
+// }

src/DataStreamSuite/PubSub/ManagedTCP/ITCPConnection.cs

@@ -1,5 +1,6 @@
 using System;
 using System.Collections.Generic;
+using System.Runtime.CompilerServices;
 using System.Text;
 
 namespace BCG {
@@ -20,7 +21,7 @@ namespace BCG {
       /// <param name="data"></param>
       /// <param name="confId"></param>
       /// <param name="del"></param>
-      void sendWithConfirmation(byte[] data, ref string confId, SendConfirmationDelegate del);
+      //void sendWithConfirmation(byte[] data, ref string confId, SendConfirmationDelegate del);
       /// <summary>
       /// sends data to the other side. theres no guaranatee that
       /// data is delievered.
@@ -41,7 +42,7 @@ namespace BCG {
       /// </summary>
       /// <param name="ip"></param>
       /// <param name="port"></param>
-      void getPeerAddress(out string ip, out int port);
+      RemoteConnectionInfo get_peer_address();
 
       /// <summary>
       /// Gets its own ip and port, as seen by the peer. So if this
@@ -51,7 +52,7 @@ namespace BCG {
       /// </summary>
       /// <param name="ip"></param>
       /// <param name="port"></param>
-      void getSelfAddress(out string ip, out int port);
+      RemoteConnectionInfo get_self_address();
       /// <summary>
       /// if the connection is disconnected, it disposes itself,
       /// hence this is an indicator of whether the connection is

src/DataStreamSuite/PubSub/ManagedTCP/RemoteConnectionInfo.cs

@@ -3,37 +3,66 @@ using System.Collections.Generic;
 using System.Text;
 using System.Net;
 using bts.utils;
+using System.Net.NetworkInformation;
+using bts;
 
 namespace BCG {
    public class RemoteConnectionInfo {
-      private string myIp;
-      private int myPort;
-      private IPEndPoint myEndPoint;
+      private string _ip;
+      private int _port;
+      //private IPEndPoint _ip_endpoint;
+      public static string ipv4_compat_prefix = "::ffff:";
+      public static RemoteConnectionInfo null_instance = new RemoteConnectionInfo(String.Empty, 0);
+      public static string prefix = "bts-cloud://";
+      //IP addresses that start with ::ffff: are IPv6-mapped IPv4 addresses. This format is used to represent an IPv4 address within an IPv6 address space.
+
+      public static string clean_ip(string ip) {
+         string rv = ip;
+         if (!String.IsNullOrEmpty(ip)) {
+            if (ip.StartsWith(ipv4_compat_prefix)) {
+               rv = ip.Substring(ipv4_compat_prefix.Length);
+            }
+         }
+         return rv;
+      }
 
       public RemoteConnectionInfo(string ip, int port) {
-         myIp = ip;
-         myPort = port;
+         _ip = clean_ip(ip);
+         _port = port;
+         Log.Gen.info($"RemoteConnectionInfo constructed ip_arg: {ip} _ip: {_ip} port: {_port}");
       }
 
-      public RemoteConnectionInfo(string ip, short port, IPEndPoint endPoint)
-         : this(ip, port) {
-         myEndPoint = endPoint;
+      public static RemoteConnectionInfo parse_url(string url) {
+         string[] toks = url.Split("//", StringSplitOptions.RemoveEmptyEntries);
+         var prefix = toks[0];
+         var ip_port = toks[1];
+         var ip_port_toks = toks[1].Split(':');
+         string ip = ip_port_toks[0];
+         int port = Int32.Parse(ip_port_toks[1]);
+         return new RemoteConnectionInfo(ip, port);
       }
 
+      // public RemoteConnectionInfo(string ip, int port, IPEndPoint ip_endpoint)
+      //    : this(ip, port) {
+      //    _ip_endpoint = ip_endpoint;
+      // }
+
       public string ip {
-         get { return myIp; }
+         get { return _ip; }
       }
 
       public int port {
-         get { return myPort; }
+         get { return _port; }
       }
 
-      public IPEndPoint endPoint {
-         get { return myEndPoint; }
-      }
+      // public IPEndPoint end_point {
+      //    // new IPEndPoint(IPAddress.Parse(server_address.ip), server_address.port);
+      //    get { return _ip_endpoint; }
+      // }
 
       public override string ToString() {
-         return ip + ":" + port;
+         //return ip + ":" + port;
+         return $"{prefix}{ip}:{port}";
       }
 
       public override int GetHashCode() {

src/DataStreamSuite/PubSub/ManagedTCP/ServerToClientManagedTCPConnectionAcceptor.cs

@@ -7,6 +7,7 @@ using System.Net;
 using BCG;
 
 namespace bts {
+   // TODO: replace it with AsyncTCPConnectionAcceptor_APM
    public class ServerToClientManagedTCPConnectionAcceptor {
 
       private readonly int _port;
@@ -53,12 +54,10 @@ namespace bts {
          ITCPConnection new_conn = new bts.AsyncTCPConnection_APM(new_sock, TimeSpan.FromSeconds(1), ITCPConnection.ConnectionSide.Server);
          _handler.onConnection(new_conn);
          _accept_socket.BeginAccept(new AsyncCallback(_on_accept), new object[] { });
-         string ip; int port;
-         new_conn.getPeerAddress(out ip, out port);
-         Log.Gen.info(
-            $"ServerToClientManagedTCPConnectionAcceptor._on_accept connection accepted successfully from ip: {ip} port: {port} (local_port: {_port})");
+         var peer_rci = new_conn.get_peer_address();
+         Log.Gen.info($"ServerToClientManagedTCPConnectionAcceptor._on_accept connection accepted successfully from rci: {peer_rci}");
       }
-
-
    }
+
+
 }

src/DataStreamSuite/PubSub/ManagedTCP/SingleServerToClientManagedTCPConnectionAcceptor.cs

@@ -1,104 +1,104 @@
-using System;
-using System.Collections.Generic;
-using System.Text;
-using System.Threading;
-using System.Net;
-using System.Net.Sockets;
-using System.Threading.Tasks;
-using bts;
-using Bts.PubSub;
-
-//using BCG.Utilities;
-
-namespace BCG {
-   public class SingleServerToClientManagedTCPConnectionAcceptor {
-      private TimeSpan myHBTimeSpan,
-                              myWaitTimeBeforeRetrying = new TimeSpan(0, 0, 0, 0, 250);
-      private ITCPConnectionHandler myHandler;
-      private Socket myAcceptSock;
-      private int myPort;
-
-
-      public int activateWithRandomPort(ITCPConnectionHandler handler, TimeSpan clientHBTimeSpan) {
-         myHBTimeSpan = clientHBTimeSpan;
-         myHandler = handler;
-         myAcceptSock = new Socket(AddressFamily.InterNetwork, SocketType.Stream, ProtocolType.Tcp);
-         int port = 0;
-         bool madeItThrough = false;
-         //         BlueEventLog.logEvent("Looking for open port to run the server on.", PS3LogLevels.connection);
-         Random randomizer = new Random();
-         PubSubPortRange theRange = Log.notnull(PubSubConfig.Instance.PortRange);
-
-
-         //if the range of ports is 1000, then trying 5,000 times means 
-         //your odds of missing an open port are less than 1%
-
-         int maxAttempts = (theRange.maxPort - theRange.minPort) * 5;
-         int totalAttempts = 0;
-         while (!madeItThrough) {
-            port = randomizer.Next();
-            port = theRange.minPort + (port % (theRange.maxPort - theRange.minPort));
-
-            try {
-               myAcceptSock.Bind(new IPEndPoint(IPAddress.Any, port));
-               madeItThrough = true;
-            } catch (Exception) {
-               totalAttempts++;
-               if (totalAttempts < maxAttempts) {
-                  //BlueEventLog.logEvent("Couldn't bind to port " + port.ToString() + "; retrying...", PS3LogLevels.connection);
-               } else {
-                  // BlueEventLog2.instance().abort("Couldn't find an open port to advertise the server on. Consider modifying PS3_PortRange. Talk to Mark for details.");
-               }
-            }
-         }
-         myPort = port;
-         myAcceptSock.Listen(250);
-         myAcceptSock.BeginAccept(new AsyncCallback(myOnAccept), new object[] { });
-
-         return myPort;
-      }
-
-      public int activate(ITCPConnectionHandler handler, int port, TimeSpan clientHBTimeSpan) {
-         myHBTimeSpan = clientHBTimeSpan;
-         myHandler = handler;
-         myAcceptSock = new Socket(AddressFamily.InterNetwork, SocketType.Stream, ProtocolType.Tcp);
-         IPEndPoint endpoint = new IPEndPoint(IPAddress.Any, port);
-         myAcceptSock.Bind(endpoint);
-         myAcceptSock.Listen(250);
-         myAcceptSock.BeginAccept(new AsyncCallback(myOnAccept), new object[] { });
-         myPort = endpoint.Port;
-         return myPort;  // not sure why we need this class variable myPort, but I am just keeping it up with the existing code
-      }
-
-      public int accept_on_port(ITCPConnectionHandler handler, string ip, int port, TimeSpan client_HB_timespan) {
-         Task.Run(async () => {
-            await TCPSocketUtils.accept_connections_async(ip, port, handler, TimeSpan.FromSeconds(1));
-         });
-         return port;
-      }
-
-      private void myOnAccept(IAsyncResult res) {
-         Socket newSock;
-         try {
-            newSock = myAcceptSock.EndAccept(res);
-         } catch (Exception ex) {
-            // BlueEventLog.logEvent(ex);
-            Thread.Sleep(myWaitTimeBeforeRetrying);
-            myAcceptSock.BeginAccept(new AsyncCallback(myOnAccept), new object[] { });
-            if (ex is OutOfMemoryException)
-               throw ex;
-            return;
-         }
-         newSock.LingerState = new LingerOption(false, 0);
-         newSock.Blocking = false;
-         ITCPConnection newConn = new bts.AsyncTCPConnection_APM(newSock, myHBTimeSpan, ITCPConnection.ConnectionSide.Server);
-         myHandler.onConnection(newConn);
-         myAcceptSock.BeginAccept(new AsyncCallback(myOnAccept), new object[] { });
-         string ip; int port;
-         newConn.getPeerAddress(out ip, out port);
-         string connMsg = String.Format("Connection to port {2} made from {0}:{1}", ip, port, myPort);
-         //         BlueEventLog.logEvent(connMsg, PS3LogLevels.connection);
-         //Console.WriteLine(connMsg);       
-      }
-   }
-}
+// using System;
+// using System.Collections.Generic;
+// using System.Text;
+// using System.Threading;
+// using System.Net;
+// using System.Net.Sockets;
+// using System.Threading.Tasks;
+// using bts;
+// using Bts.PubSub;
+
+// //using BCG.Utilities;
+
+// namespace BCG {
+//    public class SingleServerToClientManagedTCPConnectionAcceptor {
+//       private TimeSpan myHBTimeSpan,
+//                               myWaitTimeBeforeRetrying = new TimeSpan(0, 0, 0, 0, 250);
+//       private ITCPConnectionHandler myHandler;
+//       private Socket myAcceptSock;
+//       private int myPort;
+
+
+//       public int activateWithRandomPort(ITCPConnectionHandler handler, TimeSpan clientHBTimeSpan) {
+//          myHBTimeSpan = clientHBTimeSpan;
+//          myHandler = handler;
+//          myAcceptSock = new Socket(AddressFamily.InterNetwork, SocketType.Stream, ProtocolType.Tcp);
+//          int port = 0;
+//          bool madeItThrough = false;
+//          //         BlueEventLog.logEvent("Looking for open port to run the server on.", PS3LogLevels.connection);
+//          Random randomizer = new Random();
+//          PubSubPortRange theRange = Log.notnull(PubSubConfig.Instance.PortRange);
+
+
+//          //if the range of ports is 1000, then trying 5,000 times means 
+//          //your odds of missing an open port are less than 1%
+
+//          int maxAttempts = (theRange.maxPort - theRange.minPort) * 5;
+//          int totalAttempts = 0;
+//          while (!madeItThrough) {
+//             port = randomizer.Next();
+//             port = theRange.minPort + (port % (theRange.maxPort - theRange.minPort));
+
+//             try {
+//                myAcceptSock.Bind(new IPEndPoint(IPAddress.Any, port));
+//                madeItThrough = true;
+//             } catch (Exception) {
+//                totalAttempts++;
+//                if (totalAttempts < maxAttempts) {
+//                   //BlueEventLog.logEvent("Couldn't bind to port " + port.ToString() + "; retrying...", PS3LogLevels.connection);
+//                } else {
+//                   // BlueEventLog2.instance().abort("Couldn't find an open port to advertise the server on. Consider modifying PS3_PortRange. Talk to Mark for details.");
+//                }
+//             }
+//          }
+//          myPort = port;
+//          myAcceptSock.Listen(250);
+//          myAcceptSock.BeginAccept(new AsyncCallback(myOnAccept), new object[] { });
+
+//          return myPort;
+//       }
+
+//       public int activate(ITCPConnectionHandler handler, int port, TimeSpan clientHBTimeSpan) {
+//          myHBTimeSpan = clientHBTimeSpan;
+//          myHandler = handler;
+//          myAcceptSock = new Socket(AddressFamily.InterNetwork, SocketType.Stream, ProtocolType.Tcp);
+//          IPEndPoint endpoint = new IPEndPoint(IPAddress.Any, port);
+//          myAcceptSock.Bind(endpoint);
+//          myAcceptSock.Listen(250);
+//          myAcceptSock.BeginAccept(new AsyncCallback(myOnAccept), new object[] { });
+//          myPort = endpoint.Port;
+//          return myPort;  // not sure why we need this class variable myPort, but I am just keeping it up with the existing code
+//       }
+
+//       public int accept_on_port(ITCPConnectionHandler handler, string ip, int port, TimeSpan client_HB_timespan) {
+//          Task.Run(async () => {
+//             await TCPSocketUtils.accept_connections_async(ip, port, handler, TimeSpan.FromSeconds(1));
+//          });
+//          return port;
+//       }
+
+//       private void myOnAccept(IAsyncResult res) {
+//          Socket newSock;
+//          try {
+//             newSock = myAcceptSock.EndAccept(res);
+//          } catch (Exception ex) {
+//             // BlueEventLog.logEvent(ex);
+//             Thread.Sleep(myWaitTimeBeforeRetrying);
+//             myAcceptSock.BeginAccept(new AsyncCallback(myOnAccept), new object[] { });
+//             if (ex is OutOfMemoryException)
+//                throw ex;
+//             return;
+//          }
+//          newSock.LingerState = new LingerOption(false, 0);
+//          newSock.Blocking = false;
+//          ITCPConnection newConn = new bts.AsyncTCPConnection_APM(newSock, myHBTimeSpan, ITCPConnection.ConnectionSide.Server);
+//          myHandler.onConnection(newConn);
+//          myAcceptSock.BeginAccept(new AsyncCallback(myOnAccept), new object[] { });
+//          string ip; int port;
+//          newConn.get_peer_address(out ip, out port);
+//          string connMsg = String.Format("Connection to port {2} made from {0}:{1}", ip, port, myPort);
+//          //         BlueEventLog.logEvent(connMsg, PS3LogLevels.connection);
+//          //Console.WriteLine(connMsg);       
+//       }
+//    }
+// }

src/DataStreamSuite/PubSub/ManagedTCP/TCPSocketUtils.cs

@@ -6,14 +6,9 @@ using System.Net.Sockets;
 using System.Runtime.CompilerServices;
 using System.Text;
 using System.Threading.Tasks;
-using BCG;
-using System.Xml.Serialization;
-using System.Reflection;
-using BCG.Utilities.General;
-using System.Net.WebSockets;
 using bts;
 using System.Threading;
-using Google.Protobuf.WellKnownTypes;
+using System.Runtime.InteropServices;
 
 namespace BCG {
    // utility functions for TCP socket async communications. Does the following
@@ -22,6 +17,106 @@ namespace BCG {
    // send data
    // receive data
    public static class TCPSocketUtils {
+
+      struct TcpKeepAlive {
+         public uint OnOff;
+         public uint KeepAliveTime;
+         public uint KeepAliveInterval;
+      }
+
+      public static void set_server_socket_options(ref Socket server_socket) {
+
+         // Enable KeepAlive with detailed settings
+         TcpKeepAlive keepAliveSettings = new TcpKeepAlive {
+            OnOff = 1, // Enable KeepAlive
+            KeepAliveTime = 60000, // 1 minute in milliseconds
+            KeepAliveInterval = 10000 // 10 seconds in milliseconds
+         };
+
+         server_socket.SetSocketOption(SocketOptionLevel.Socket, SocketOptionName.KeepAlive, true);
+
+         // Set keep-alive interval and time (Linux-specific method)
+         if (OperatingSystem.IsLinux()) {
+            ;
+            const int keepAliveTime = 60; // Seconds
+            const int keepAliveInterval = 10; // Seconds
+            server_socket.SetSocketOption(SocketOptionLevel.Tcp, SocketOptionName.TcpKeepAliveTime, keepAliveTime);
+            server_socket.SetSocketOption(SocketOptionLevel.Tcp, SocketOptionName.TcpKeepAliveInterval, keepAliveInterval);
+         } else {
+            int size = Marshal.SizeOf(keepAliveSettings);
+            IntPtr ptr = Marshal.AllocHGlobal(size);
+            Marshal.StructureToPtr(keepAliveSettings, ptr, true);
+            byte[] inOptionValues = new byte[size];
+            Marshal.Copy(ptr, inOptionValues, 0, size);
+            Marshal.FreeHGlobal(ptr);
+
+            server_socket.IOControl(IOControlCode.KeepAliveValues, inOptionValues, null);
+         }
+
+         // Set other socket options as needed
+         server_socket.SetSocketOption(SocketOptionLevel.Socket, SocketOptionName.ReuseAddress, true);
+         server_socket.SetSocketOption(SocketOptionLevel.Tcp, SocketOptionName.NoDelay, true);
+         server_socket.SetSocketOption(SocketOptionLevel.Socket, SocketOptionName.ReceiveTimeout, 10000);
+         server_socket.SetSocketOption(SocketOptionLevel.Socket, SocketOptionName.SendTimeout, 10000);
+         server_socket.SetSocketOption(SocketOptionLevel.Socket, SocketOptionName.ReceiveBuffer, 8192);
+         server_socket.SetSocketOption(SocketOptionLevel.Socket, SocketOptionName.SendBuffer, 8192);
+
+         // Optionally configure linger option
+         LingerOption lingerOption = new LingerOption(true, 10);
+         server_socket.SetSocketOption(SocketOptionLevel.Socket, SocketOptionName.Linger, lingerOption);
+      }
+
+      public static RemoteConnectionInfo get_peer_address(Socket sock) {
+         IPEndPoint ip_ep = sock.RemoteEndPoint as IPEndPoint;
+         int port = ip_ep.Port;
+         string ip = ip_ep.Address.ToString();
+         return new RemoteConnectionInfo(ip, port);
+      }
+
+      public static RemoteConnectionInfo get_self_address(Socket socket) {
+         IPEndPoint ip_ep = socket.LocalEndPoint as IPEndPoint;
+         int port = ip_ep.Port;
+         string ip = ip_ep.Address.ToString();
+         return new RemoteConnectionInfo(ip, port);
+      }
+
+
+      public static void set_client_socket_options(ref Socket client_socket) {
+         TcpKeepAlive keep_alive_settings = new TcpKeepAlive {
+            OnOff = 1, // Enable KeepAlive
+            KeepAliveTime = 60000, // 1 minute in milliseconds
+            KeepAliveInterval = 10000 // 10 seconds in milliseconds
+         };
+
+         if (OperatingSystem.IsLinux()) {
+            const int keepAliveTime = 60; // Seconds
+            const int keepAliveInterval = 10; // Seconds
+            client_socket.SetSocketOption(SocketOptionLevel.Tcp, SocketOptionName.TcpKeepAliveTime, keepAliveTime);
+            client_socket.SetSocketOption(SocketOptionLevel.Tcp, SocketOptionName.TcpKeepAliveInterval, keepAliveInterval);
+         } else {
+            int size = Marshal.SizeOf(keep_alive_settings);
+            IntPtr ptr = Marshal.AllocHGlobal(size);
+            Marshal.StructureToPtr(keep_alive_settings, ptr, true);
+            byte[] inOptionValues = new byte[size];
+            Marshal.Copy(ptr, inOptionValues, 0, size);
+            Marshal.FreeHGlobal(ptr);
+
+            client_socket.IOControl(IOControlCode.KeepAliveValues, inOptionValues, null);
+         }
+
+         // Set other socket options as needed
+         client_socket.SetSocketOption(SocketOptionLevel.Socket, SocketOptionName.ReuseAddress, true);
+         client_socket.SetSocketOption(SocketOptionLevel.Tcp, SocketOptionName.NoDelay, true);
+         client_socket.SetSocketOption(SocketOptionLevel.Socket, SocketOptionName.ReceiveTimeout, 10000);
+         client_socket.SetSocketOption(SocketOptionLevel.Socket, SocketOptionName.SendTimeout, 10000);
+         client_socket.SetSocketOption(SocketOptionLevel.Socket, SocketOptionName.ReceiveBuffer, 8192);
+         client_socket.SetSocketOption(SocketOptionLevel.Socket, SocketOptionName.SendBuffer, 8192);
+
+         // Optionally configure linger option
+         LingerOption lingerOption = new LingerOption(true, 10);
+         client_socket.SetSocketOption(SocketOptionLevel.Socket, SocketOptionName.Linger, lingerOption);
+      }
+
       #region socket opening and connecting
       // keeps accepting connections forever
       public static async Task accept_connections_async(string ip, int port, ITCPConnectionHandler handler,

src/DataStreamSuite/PubSub/Misc/SyncQueue.cs

@@ -6,23 +6,20 @@ using System.Threading;
 using BCG;
 //using BCG.Utilities;
 
-namespace BCG
-{
+namespace BCG {
 
    /// <summary>
    /// Use this class for inter thread communication. It assumed a thread safe
    /// queue (this is right now implemented using locks, but for better performance
    /// will be switched over to lock free code, using cas).
    /// </summary>
-   public class SyncQueue <T> : ISyncQueue<T>
-   {
+   public class SyncQueue<T> : ISyncQueue<T> {
       private ThreadSafeQueue<T> myQueue = new ThreadSafeQueue<T>();
       private AutoResetEvent myEvent = new AutoResetEvent(false);
 
-      public SyncQueue()   {}
+      public SyncQueue() { }
 
-      public void enqueue(T t)
-      {
+      public void enqueue(T t) {
          myQueue.enqueue(t);
          myEvent.Set();
          //return true;
@@ -35,42 +32,31 @@ namespace BCG
       //   return true;
       //}
 
-      public T dequeue()
-      {
+      public T dequeue() {
          return dequeue(Int32.MaxValue);
       }
 
-      public T dequeue (int timeout)
-      {
+      public T dequeue(int timeout) {
          DateTime startTime = DateTime.Now;
          T rv = myQueue.dequeue();
-         if (rv != null || timeout <= 0)
-         {
+         if (rv != null || timeout <= 0) {
             return rv;
-         }
-         else
-         {
+         } else {
             int timeRemaining = timeout;
-            while (true)
-            {
+            while (true) {
                bool res = myEvent.WaitOne(timeRemaining, false);
-               if (res)
-               {
+               if (res) {
                   ///timeout has not expired, so thread has been signalled
                   rv = myQueue.dequeue();
-                  if (rv == null)
-                  {
+                  if (rv == null) {
                      timeRemaining = timeout - Convert.ToInt32(((TimeSpan)(DateTime.Now - startTime)).TotalMilliseconds);
                      if (timeRemaining <= 0)
                         return rv;
                      else
                         continue;
-                  }
-                  else
+                  } else
                      return rv;
-               }
-               else
-               {
+               } else {
                   ///timeout expired, return default;
                   return default(T);
                }
@@ -79,24 +65,18 @@ namespace BCG
          }
       }
 
-      public List<T> dequeueMultipleElements(int timeout, int maxElements)
-      {
-         if (timeout == 0)
-         {
+      public List<T> dequeueMultipleElements(int timeout, int maxElements) {
+         if (timeout == 0) {
             return myQueue.dequeueLimited(maxElements);
-         }
-         else
-         {
+         } else {
             DateTime startTime = DateTime.Now;
             List<T> rv = new List<T>();
             rv = myQueue.dequeueLimited(maxElements);
             if (rv.Count == maxElements)
                return rv;
-            else
-            {
+            else {
                int timeRemaining = timeout;
-               while (true)
-               {
+               while (true) {
                   T t = dequeue(timeRemaining);
                   if (t != null)
                      rv.Add(t);
@@ -108,19 +88,20 @@ namespace BCG
          }
       }
 
-      public List<T> dequeueMultipleElements(TimeSpan ts, int maxElements)
-      {
+      public List<T> dequeueMultipleElements(TimeSpan ts, int maxElements) {
          return dequeueMultipleElements(Convert.ToInt32(ts.TotalMilliseconds), maxElements);
       }
 
-      public int getCount()
-      {
+      public List<T> dequeue_all() {
+         return myQueue.dequeueAll();
+      }
+
+      public int getCount() {
          return myQueue.getCount();
       }
 
 
-      public void terminate()
-      {
+      public void terminate() {
       }
    }
 }

src/DataStreamSuite/PubSub/PublishedCache.cs

@@ -3,7 +3,7 @@ using System.Collections.Generic;
 
 namespace bts {
    // This caches all the published data. This class is created and held by
-   // PublisherCore. Whend ata is publsihed , PublisherCore passes it to this 
+   // PublisherCore. When data is publsihed , PublisherCore passes it to this 
    // class via the publish call. On a reconnection to the PubSubServer, 
    // PublisherCore calls get_content to get the data to re-publish, to 
    // re-seed the PubSubServer
@@ -14,6 +14,7 @@ namespace bts {
 
 
       public void publish(PublishMsg msg) {
+         Log.Gen.info($"PublishedCache.publish msg: {msg}");
          SingleTagData tag_data = null;
          if (!_data_map.TryGetValue(msg.tag, out tag_data)) {
             tag_data = new SingleTagData(msg.tag, msg.tag_type);
@@ -22,14 +23,15 @@ namespace bts {
          tag_data.handle_publish(msg);
       }
 
-      public byte[] get_content(string tag, out string tag_type) {
+      public Tuple<string, byte[]> get_content(string tag) {
          SingleTagData tag_data;
+         byte[] rv = new byte[] { };
          if (_data_map.TryGetValue(tag, out tag_data)) {
-            tag_type = tag_data.tag_type;
-            return tag_data.get_current_content();
+            string tag_type = tag_data.tag_type;
+            byte[] data = tag_data.get_current_content();
+            return new Tuple<string, byte[]>(tag_type, data);
          }
-         tag_type = String.Empty;
-         return new byte[] { };
+         return new Tuple<string, byte[]>(String.Empty, new byte[] { });
       }
 
       public void clear(string tag) {

src/DataStreamSuite/PubSub/PublisherCore.cs

@@ -14,26 +14,24 @@ namespace BCG {
       // to be used when communicating to a pubsubserver
       protected PubSubServerConnectionFactory _connection_factory = new PubSubServerConnectionFactory(false);
 
-      // internal queue used to serialize pubsub method calls
+      // internal queue used to serialize server calls. All calls to PublisherCore are written to this quwue,
+      // dequeued by the message pump thread
       protected SyncQueue<PubSubClientMessage> _main_queue = new SyncQueue<PubSubClientMessage>();
 
       // all the data published through this instance of PublisherCore
-      // is ketp there, in a form that can be used repopulate a pubsub server
-      // in the event of a crash
+      // is kept there, in a form that can be used repopulate a pubsub server
+      // in the event of a conection re-establishment to the PubSubServer
       protected PublishedCache _state = new PublishedCache();
 
-      // a different IPubSubServerConnection object is used to interact
-      // with each of the pubsub servers used by this publishercore
+      // every PubSubServer is communicated with a adifferent IPubSubServerConnection object.
 
       // a dictionary for fast retrieval of publishers
       // tag -> IPubSubServerConnection
       protected IDictionary<string, IPubSubServerConnection> _tag_sender_map =
          new Dictionary<string, IPubSubServerConnection>();
-      // url -> IPubSubServerConnection
-      private HashSet<IPubSubServerConnection> _connections = new HashSet<IPubSubServerConnection>();
-
-      // data waiting for transaction to end or abort
-      // tag->TagWaitingInTransaction
+      // pubsub server urls vs IPubSubServerConnection
+      private Dictionary<RemoteConnectionInfo, IPubSubServerConnection> _connection_map =
+         new Dictionary<RemoteConnectionInfo, IPubSubServerConnection>();
 
       // router object, used to select which pubsub should be used for each tag
       protected IPubSubRouter _router = null;
@@ -90,6 +88,7 @@ namespace BCG {
 
       public void publish(PubSubClientMessage msg) {
          Log.assert(msg.msg_type == PubSubMessageType.PUBLISH);
+         Log.assert(!String.IsNullOrEmpty(msg.pub_msg.tag_type));
          bts.Log.Gen.info($"PublisherCore.publish msg: {msg}");
          _main_queue.enqueue(msg);
       }
@@ -122,6 +121,7 @@ namespace BCG {
             try {
                while (true) {
                   PubSubClientMessage? msg = _main_queue.dequeue(100);
+                  // only msg tyes expected right now (6/21/24) are PUBLISh type
                   if (msg != null) {
                      bts.Log.Gen.info($"PublisherCore._message_pump msg: {msg}");
                      switch (msg.msg_type) {
@@ -138,8 +138,8 @@ namespace BCG {
                            break;
                      }
                   }
-
-                  foreach (var conn in _connections) {
+                  // all interactions with conn are called from the same thread
+                  foreach (var conn in _connection_map.Values) {
                      conn.doWork();
                   }
                }
@@ -156,102 +156,14 @@ namespace BCG {
       }
 
       private void _commit_transaction() {
-         _end_transaction(true);
+         //_end_transaction(true);
       }
 
       private void _abort_transaction() {
-         _end_transaction(false);
+         //_end_transaction(false);
       }
 
 
-
-      //based on success, sends a TA or TE, and sets myInTransaction to false
-      //TODO: if a transaction is sending to multiple pubsubs (in order to
-      //keep it consistent with the way it was done earlier), this should
-      //simply publish it non transactionally (easy way out), or 
-      //transactionally with respect to each pubsub
-      protected virtual void _end_transaction(bool success) {
-         // lock (this) {
-         //    // BlueEventLog.logAssert(inTransaction);
-         //    //BlueEventLog.logEvent("PublisherCore Finishing Transaction.", BlueEventLevel.Information);
-         //    _is_in_transaction = false;
-         //    if (success) {
-         //       // pubsub server -> tags with updates
-         //       Dictionary<string, List<string>> routesToTags = new Dictionary<string, List<string>>();
-
-         //       // group updates per pubsub server
-         //       foreach (string tag in myWaitingInTransactionMessages.Keys) {
-         //          string thisURL = _router.get_pub_url_for_tag(tag);
-         //          if (!routesToTags.ContainsKey(thisURL))
-         //             routesToTags[thisURL] = new List<string>();
-         //          routesToTags[thisURL].Add(tag);
-         //       }
-
-         //       // more than one server means we won't be able to honor the transactionality request
-         //       // log a warning if that is the case.
-         //       if (routesToTags.Keys.Count > 1) {
-         //          //this is going to be a long error message, but it's an importat one to be able to debug. 
-         //          StringBuilder errorMessageBuilder = new StringBuilder();
-         //          errorMessageBuilder.Append("Tags within this transaction are routed to different pubSub servers.\n");
-         //          errorMessageBuilder.Append("This will break the transactionality of clients receiving the data. Details:\n");
-
-         //          foreach (string route in routesToTags.Keys) {
-         //             foreach (string tag in routesToTags[route]) {
-         //                errorMessageBuilder.Append("\tTag ");
-         //                errorMessageBuilder.Append(tag);
-         //                errorMessageBuilder.Append(" is routed to ");
-         //                errorMessageBuilder.Append(route);
-         //                errorMessageBuilder.Append(".\n");
-         //             }
-
-         //          }
-         //          string errMsg = errorMessageBuilder.ToString();
-         //          // BlueEventLog.logEvent(errMsg, BlueEventLevel.Diagnosis);
-         //       }
-
-         //       // iterate through each server with pending messages
-         //       foreach (KeyValuePair<string, List<string>> kvp in routesToTags) {
-         //          IPubSubServerConnection transactionSender = null;
-         //          string publishingApp = ClientInfo.thisClient.processString;
-         //          string connection = kvp.Key;
-         //          Stream transactionData = new MemoryStream();
-
-         //          // iterate through each of the pending tag updates for one server
-         //          foreach (string tag in kvp.Value) {
-         //             ///it would be the same transaction sender that would 
-         //             ///return each time
-         //             transactionSender = myGetOrBuildSender(tag);
-
-
-         //             TagWaitingInTransaction waiting = myWaitingInTransactionMessages[tag];
-         //             // iterate through each of the updates for a specific tag, specific server
-         //             foreach (byte[] data in waiting.data)
-         //                // update internal state
-         //                _state.publish(waiting.tag, waiting.tagType, data);
-         //             // add a pubsub header and copy each message to the transactionData stream
-         //             waiting.write_to(transactionData, connection, publishingApp);
-         //          }
-         //          // now transactionData holds all the updates for all the tags 
-         //          // that are to be sent to the PubSub represented by transactionSender
-
-         //          // add a transaction header, indicating how many tags in the transaction
-         //          TransactionMetaData tMd = new TransactionMetaData(kvp.Value.Count);
-
-         //          // rewind transaction data to the beggining
-         //          transactionData.Position = 0;
-
-         //          // send the entire thing to the specific pubsub server.
-         //          // this is where the data is actually sent out !
-         //          var ps_msg = new PubSubMessage(new MetaData(tMd), transactionData);
-         //          Console.WriteLine($"serverConn.send metadata: {ps_msg.meta_data}");
-         //          bts.Log.Gen.info($"PublisherCore.send ps_msg: {ps_msg}");
-         //          transactionSender.send(ps_msg);
-         //       }
-         //    }
-         //    myWaitingInTransactionMessages.Clear();
-         // }
-      }
-
       protected virtual void _publish(PublishMsg msg) {
          bts.Log.Gen.info($"PublisherCore._publish msg: {msg}");
          // update the state
@@ -277,11 +189,13 @@ namespace BCG {
       protected IPubSubServerConnection _get_or_build_sender(string tag) {
          IPubSubServerConnection tag_conn = null;
          if (!_tag_sender_map.TryGetValue(tag, out tag_conn)) {
-            string route = _router.get_pub_url_for_tag(tag);
-            tag_conn = _connection_factory.getConnection(route);
-            if (!_connections.Contains(tag_conn)) {
-               tag_conn.register_connection_callback(new PubSubServerConnectionStatusChange(_handle_server_reboot));
-               _connections.Add(tag_conn);
+            RemoteConnectionInfo server_rci = _router.get_pub_route_for_tag(tag);
+            if (!_connection_map.TryGetValue(server_rci, out tag_conn)) {
+               tag_conn = _connection_factory.get_connection(server_rci);
+               tag_conn.register_connection_callback((server_url, connected) => {
+                  _handle_pubsubserver_reconnection(server_url, connected);
+               });
+               _connection_map.Add(server_rci, tag_conn);
             }
             _tag_sender_map.Add(tag, tag_conn);
          }
@@ -290,18 +204,19 @@ namespace BCG {
 
       ///in case of the server coming up, it publishes all the stored state 
       ///for that server, back to it
-      protected void _handle_server_reboot(string server_url, bool connection) {
-         bts.Log.Gen.info($"PublisherCore._handle_server_reboot server_url: {server_url} connection: {connection}");
+      private void _handle_pubsubserver_reconnection(string server_url, bool connection) {
+         bts.Log.Gen.info($"PublisherCore._handle_pubsubserver_reconnection server_url: {server_url} connection: {connection}");
          if (connection) {
             // find all tags that are being published to this server
             foreach (var kv in _tag_sender_map) {
                var tag = kv.Key;
                var conn = kv.Value;
                if (conn.server_url == server_url) {
-                  string tag_type;
-                  var content_bytes = _state.get_content(tag, out tag_type);
+                  var content = _state.get_content(tag);
+                  byte[] content_bytes = content.Item2;
+                  string tag_type = content.Item1;
                   string content_bytes_log = content_bytes == null ? "NULL" : content_bytes.Length.ToString();
-                  Log.Gen.info($"PublisherCore._handle_server_reboot content_bytes: {content_bytes_log}");
+                  Log.Gen.info($"PublisherCore._handle_pubsubserver_reconnection server_url: {server_url} content_bytes: {content_bytes_log}");
                   bts.PublishMsg publish_msg = new bts.PublishMsg() {
                      connection = kv.Key,
                      publishing_app = ClientInfo.thisClient.processString,

src/DataStreamSuite/PubSub/Server/Communication/IPubsubMessageHub.cs

@@ -15,11 +15,6 @@ namespace BCG {
    /// this event is prompted by the doWork.
    /// </summary>
    public interface IPubsubMessageHub : IWorker {
-
-      /// <summary>
-      /// The port that the message hub is listening on.  Return -1 if you're not listening on a port, i.e. for hte case of multicast
-      /// </summary>
-      int port { get; }
       /// <summary>
       /// register or unregister callback for data. The data can either be received on this callback
       /// on an arbitrary thread, or polled via getUpdates
@@ -49,6 +44,8 @@ namespace BCG {
       /// <param name="sender"></param>
       /// <param name="data"></param>
       List<Pair<string, byte[]>> get_updates(TimeSpan timeout, int maxElements);
+      // same as above, but gets all updates that are queued up
+      List<Pair<string, byte[]>> get_all_updates();
       /// <summary>
       /// sends data to the receiver.
       /// </summary>

src/DataStreamSuite/PubSub/Server/Communication/ManagedTCP/ManagedTCPMessageHub.cs

@@ -15,17 +15,13 @@ using bts;
 
 namespace BCG {
 
-   // this is the main TCP message exchange in the pubsub server. It accepts connections, 
+   // this is the main TCP message exchange in the pubsub server. It accepts connections, data, as well as send data
    public class ManagedTCPMessageHub : IPubsubMessageHub,      // external facing interface to be used by PubSubMesage Reactor
                                        ITCPConnectionHandler,  // interface to receive connections when calling ServerToClientManagedTCPConnectionAcceptor.accept_connections
                                        ITCPMessageHandler {    // interface to receive data from ITCPConnection
-      private int _port;
+      private readonly int _port;
       private bool _is_disposed = false;
-      public int port {
-         get { return _port; }
-         //property is read only
-
-      }
+      // class used internally to queue messages from one trhead ot another
       private class ConnectionStatusInfo {
          public readonly string client_url;
          public readonly PubSubCallbacks.ConnectionStatus status;
@@ -39,16 +35,16 @@ namespace BCG {
       private PubSubServerDataDelegate _data_cb;
 
       /// <summary>
-      /// client to connection
+      /// client to connection map
       /// </summary>
       private Dictionary<string, ITCPConnection> _connection_map = new Dictionary<string, ITCPConnection>();
       /// <summary>
       /// data that is received from the client (on the onData thread) is queued in this
-      /// to be fetched by the client of this class in the getUpdates.
+      /// to be fetched by the client of this class in the get_updates.
       /// </summary>
       private SyncQueue<Pair<string, byte[]>> _received_data_queue;
       /// <summary>
-      /// data to be sent to the client is stored  here and is fetched by the
+      /// data to be sent to the client is enqueued here and is fetched by the
       /// myDoItAll thread to be sent to the clients. the send is not directly sent out
       /// since there is some state management needed, so we do it on the main message
       /// pump thread
@@ -62,27 +58,25 @@ namespace BCG {
       // _connection_cb arguments are queued here                                      
       private ThreadSafeQueue<ConnectionStatusInfo> _conn_callback_queue = new ThreadSafeQueue<ConnectionStatusInfo>();
       /// <summary>
-      /// new connections received (on the onConnection thread) is queued
+      /// new connections received (on the onConnection thread) are queued
       /// in this and will be fetched by the message_pump thread
       /// </summary>
       private ThreadSafeQueue<ITCPConnection> _new_connections = new ThreadSafeQueue<ITCPConnection>();
-      /// <summary>
-      /// clients that sent heartbeats are stored in here
-      /// </summary>
-      private ThreadSafeQueue<string> _received_hbs = new ThreadSafeQueue<string>();
 
       private readonly int _read_send_queue_timeout = 100;// just used to pass it on
       private readonly TimeSpan _hb_send_interval;
       private TimeSpan _subscriber_drop_threshold;
-      private TimeSpan _collect_disposed_connections_period = new TimeSpan(0, 0, 0, 0, 1000);
+      private TimeSpan _collect_disposed_connections_period = TimeSpan.FromMilliseconds(500);
       private DateTime _last_HB_collection_time = DateTime.Now;
       private DateTime _last_disposed_dead_collection_time = DateTime.Now;
-      private ServerToClientManagedTCPConnectionAcceptor _connection_acceptor;
+      // this si always active, accepting new client connections
+      private AsyncTCPConnectionAcceptor_AsyncWrapper _connection_acceptor;
 
       // TODO, get 
       public ManagedTCPMessageHub(int port, TimeSpan hb_send_interval, int missing_HBs_for_gc, TimeSpan collect_disposed_connections_period) {
          _port = port;
-         _connection_acceptor = new ServerToClientManagedTCPConnectionAcceptor(port, this);
+         // what kind of connection_acceptor is controlled here
+         _connection_acceptor = new AsyncTCPConnectionAcceptor_AsyncWrapper(_port, this);
          _hb_send_interval = hb_send_interval;
          _subscriber_drop_threshold = new TimeSpan(0, 0, 0, 0, Convert.ToInt32(hb_send_interval.TotalMilliseconds * missing_HBs_for_gc));
          _collect_disposed_connections_period = collect_disposed_connections_period;
@@ -91,6 +85,7 @@ namespace BCG {
 
       #region IPubsubMessageHub Members
 
+      // this is unused right now, since the get_updates on a polling thread is called to by the client of this class to get messages
       public void register_data_callback(PubSubServerDataDelegate del) {
          bts.Log.assert_d(!_activated, "connection callbacks should be registered before the call to activate");
          bts.Log.Gen.info("ManagedTCPMessageHub.register_data_callback called");
@@ -104,33 +99,37 @@ namespace BCG {
       }
 
       public void activate() {
-         _connection_acceptor.activate();
+         _connection_acceptor.start();
          bts.Log.Gen.info($"ManagedTCPMessageHub.activate TCP Component is ready, listening on port {_port} for incoming connections");
          _activated = true;
+         // as the name says, all messages hand-off happen here
          Thread do_it_all_thread = new Thread(new ThreadStart(_main_message_pump));
          do_it_all_thread.IsBackground = true;
+         do_it_all_thread.Priority = ThreadPriority.Highest;
          do_it_all_thread.Start();
       }
 
-      public List<Pair<string, byte[]>> get_updates(TimeSpan timeout, int maxElements) {
-         return _received_data_queue.dequeueMultipleElements(timeout, maxElements);
+      // retutns the incoming either when timeout expires or n_elems are dequeued   
+      public List<Pair<string, byte[]>> get_updates(TimeSpan timeout, int max_n_elems) {
+         return _received_data_queue.dequeueMultipleElements(timeout, max_n_elems);
       }
 
-      private bool _enqueue_send(string receiver, byte[] data) {
-         bts.Log.Gen.info($"ManagedTCPMessageHub._enqueue_send receiver: {receiver} data len: {data.Length}");
-         _send_data_queue.enqueue(new Pair<string, byte[]>(receiver, data));
-         return true;
+      // similiar to above, but returns all the elements that have been received since the last time this function was called
+      public List<Pair<string, byte[]>> get_all_updates() {
+         return _received_data_queue.dequeue_all();
       }
 
-      // public bool send(string receiver, System.IO.Stream str) {
-      //    BinaryReader br = new BinaryReader(str);
-      //    return send(receiver, br.ReadBytes(Convert.ToInt32(str.Length - str.Position)));
+      // private bool _enqueue_send(string receiver, byte[] data) {
+      //    bts.Log.Gen.info($"ManagedTCPMessageHub._enqueue_send receiver: {receiver} data len: {data.Length}");
+      //    _send_data_queue.enqueue(new Pair<string, byte[]>(receiver, data));
+      //    return true;
       // }
 
       public bool send(string receiver, bts.PubSubServerMessage ps_msg) {
-         bts.Log.Gen.info($"ManagedTCPMessageHub.send receiver: {receiver} ps_msg: {ps_msg}");
          var ps_msg_bytes = ps_msg.serialize_into_bytes();
-         return _enqueue_send(receiver, ps_msg_bytes);
+         bts.Log.Gen.info($"ManagedTCPMessageHub.send receiver: {receiver} ps_msg: {ps_msg} serialized msg sz: {ps_msg_bytes.Length}");
+         _send_data_queue.enqueue(new Pair<string, byte[]>(receiver, ps_msg_bytes));
+         return true;
       }
 
       #endregion
@@ -155,18 +154,13 @@ namespace BCG {
       #region ITCPConnectionHandler Members
 
       /// <summary>
-      /// Enqeues both the connection name (to be reported back
-      /// via ConnectionStatusChange) and connection (to be
-      /// stored in the hashtable and used for later communication).
-      /// This is called on an arbitrary thread
+      /// This is the only place where
+      /// clients connect to the pubsubserver.
+      /// The connection in qneueued, so we can report it back in the same thread
       /// </summary>
       /// <param name="connection"></param>
       public void onConnection(ITCPConnection connection) {
-         // this is where we get new connections, dead connections are collected in _collect_disposed_connections  
-         string ip;
-         int port;
-         connection.getPeerAddress(out ip, out port);
-         bts.Log.Gen.info($"ManagedTCPMessageHub.onConnection ip: {ip} port: {port}");
+         bts.Log.Gen.info($"ManagedTCPMessageHub.onConnection connection: {connection}");
          _new_connections.enqueue(connection);
       }
 
@@ -175,23 +169,25 @@ namespace BCG {
       #region ITCPMessageHandler Members
       // called by ITCPConnection on an arbitrary thread
       public void onData(RemoteMessage message) {
-         bts.Log.Gen.info($"ManagedTCPMessgageHub.onData recv msg: {message}");
-         string sender = new BlutoSubAddress(message.remoteConnectionInfo.ip, message.remoteConnectionInfo.port).ToString();
+         bts.Log.Gen.info($"ManagedTCPMessgageHub.onData message: {message}");
+         var sender_rci = message.remoteConnectionInfo;
          switch (message.messageType) {
             case RemoteMessageType.Data:
                // if callback is registered fire it right away, else enqueue it
                if (_data_cb != null)
-                  _data_cb(sender, message.message);
+                  _data_cb(sender_rci.ToString(), message.message);
                else
-                  _received_data_queue.enqueue(new Pair<string, byte[]>(sender, message.message));
+                  _received_data_queue.enqueue(new Pair<string, byte[]>(sender_rci.ToString(), message.message));
                break;
             case RemoteMessageType.Connection:
-               // wont happen
+               // this conditional will never hit, because onData is ITCPConnection's callback, and ITCPConnection
+               // will not report a connection, connections are reported through ConnectionAcceptor
                break;
             case RemoteMessageType.Disconnection:
+               // we ignore this here, since in the pump thread function, we poll the connection and GC it if its disconnected
                break;
             case RemoteMessageType.Heartbeat:
-               // dont care about, if it goes missing, we'll get a disconnection message
+               // we dont need this either, since the HB time can be queried from the conneciton and GC'ed there and then
                break;
             default:
                bts.Log.assert_d(false, "Unknown message type");
@@ -205,10 +201,9 @@ namespace BCG {
          while (true) {
             try {
                while (true) {
-                  // these are the connections that were received on onConnection
+                  // these are the connections that were received on onConnection, written to _conn_callback_queue
                   _handle_new_connections();
-
-                  ///dead connections that are disposed
+                  ///dead connections that are disposed, writtent to _conn_callback_queue
                   _collect_disposed_connections();
 
                   Pair<string, byte[]> p = _send_data_queue.dequeue(_read_send_queue_timeout);
@@ -219,7 +214,7 @@ namespace BCG {
                      _connection_map.TryGetValue(p.first, out conn);
                      if (conn != null) {
                         conn.send(p.second);
-                        Log.Gen.info("$ManagedTCPMessageHub._main_message_pump sent n_bytes : {p.second.Length} to: {p.first}");
+                        Log.Gen.info($"ManagedTCPMessageHub._main_message_pump sending data client: {p.first} size: {p.second.Length}");
                      } else {
                         string error_msg = $"sending data of sz: {p.second.Count()} to {p.first} failed, this client might have " +
                            "disconnected, in which case it will be garbage collected within 60 secs";
@@ -250,8 +245,7 @@ namespace BCG {
                   disposed_urls.Add(url);
             }
             foreach (string url in disposed_urls) {
-               string dispose_msg = $"Collecting disposed connections : {url} is disposed by the connection itself, " +
-                  "connection count before deleting this: {_connection_map.Count}";
+               string dispose_msg = $"Collecting disposed connection: {url} connection count before deleting this: {_connection_map.Count}";
                bts.Log.Gen.warn(dispose_msg);
                _connection_map.Remove(url);
                _conn_callback_queue.enqueue(new ConnectionStatusInfo(url, PubSubCallbacks.ConnectionStatus.DISCONNECTED));
@@ -259,22 +253,20 @@ namespace BCG {
          }
       }
 
-      // these are the connections that were added to _new_connections, on on_connect
+      // these are the connections that were added to _new_connections on onConnection
       // (which gets called in another thread)
       private void _handle_new_connections() {
          var new_conns = _new_connections.dequeueAll();
          foreach (ITCPConnection conn in new_conns) {
-            string ip;
-            int port;
-            conn.getPeerAddress(out ip, out port);
-            string client_url = new BlutoSubAddress(ip, port).ToString();
-            bts.Log.Gen.info($"ManagedTCPMessageHub._handle_new_connections client_url: {client_url}");
+            var peer_rci = conn.get_peer_address();
+            bts.Log.Gen.info($"ManagedTCPMessageHub._handle_new_connections conn: {conn}");
             ///to stop it from being garbabge collected immediatly
-            _received_hbs.enqueue(client_url);
+            // ot get data
             conn.register(this);
-            _connection_map.Add(client_url, conn);
+            _connection_map.Add(peer_rci.ToString(), conn);
             // enqueued here to call the user sub in doWork
-            _conn_callback_queue.enqueue(new ConnectionStatusInfo(client_url, PubSubCallbacks.ConnectionStatus.CONNECTED));
+            // why not fire the handler here ?
+            _conn_callback_queue.enqueue(new ConnectionStatusInfo(peer_rci.ToString(), PubSubCallbacks.ConnectionStatus.CONNECTED));
          }
       }
 
@@ -293,4 +285,4 @@ namespace BCG {
 
 
 
-}
+}

src/DataStreamSuite/PubSub/Server/PubSubMessageReactor.cs

@@ -6,6 +6,8 @@ using bts;
 using Bts.PubSub;
 
 using Spark.Core;
+using System.Text;
+using System.Diagnostics;
 
 namespace BCG {
    // This is the main PubSub Server class where it all starts (called by Program.cs). Its constructed and start is called and thats it.
@@ -18,26 +20,9 @@ namespace BCG {
       private Thread _msg_polling_thread = null;
       // all the data,list of subscribers etc are in the state
       private PubSubState _state = new PubSubState();
-      private bool myStop = false;
       private MasterWorker _mainloop_master_worker = MasterWorker.getInstance(false);
-      // private double myProcessingWarningThreshold =
-      //    Convert.ToDouble(PubSub3Config.instance.PS3_MessageProcessingWarningThreshold);
-      // private HighPerformanceTimer myTimer = new HighPerformanceTimer();
-
-      private delegate void SingleTagTimeSliceDelegate(string tag, double processingTimeSeconds);
-      private SingleTagTimeSliceDelegate onSingleTagPub;
       private static SyncQueue<PublishMsg> _internal_pub_queue = new SyncQueue<PublishMsg>();
-      //this thing keeps track of whether a given tag is compressed or not. It is used for logging messages.
-
-      /* This delegate allows pubsub components to be notified when the identity ofe the pubsub is known*/
-      // private delegate void IdentityFoundDelegate(string identity, bool isRoot);
-      // private event IdentityFoundDelegate onIdentityFound;
-      // private void fireIdentityFound(string identity, bool isRoot) {
-      //    if (onIdentityFound != null)
-      //       onIdentityFound(identity, isRoot);
-      // }
-
-      //private Dictionary<string, bool> myTagCompressionState = new Dictionary<string, bool>();
+      private Stopwatch _stop_watch = new Stopwatch();
 
       public PubSubMessageReactor(int port_no) {
          bts.Log.Gen.info("PubSubMessageReactor.cons System Initialized");
@@ -51,11 +36,12 @@ namespace BCG {
             TimeSpan.FromMilliseconds(PubSubConfig.Instance.HeartbeatInterval.assert_value()),
             PubSubConfig.Instance.MissingHeartbeatsBeforeGc.assert_value(),
             TimeSpan.FromMilliseconds(PubSubConfig.Instance.DisposedConnectionCheckPeriod.assert_value()));
-
+         // this is the only event this class gets from hub
          _msg_hub.register_connection_callback(new PubSubCallbacks.PubSubServerConnectionStatusChange(_handle_client_connection));
 
       }
 
+      // the connection callback is the only event we get back from the message_hub.
       private void _handle_client_connection(string client, PubSubCallbacks.ConnectionStatus conn_status) {
          bts.Log.Gen.info($"PubsubMessageReactor._handle_client_connection client: {client} conn_status: {conn_status}");
          ///interested in disconnections only, as connections of no
@@ -65,10 +51,6 @@ namespace BCG {
             _state.remove_connection(client);
       }
 
-      private void stop() {
-         myStop = true;
-      }
-
       public void start() {
          _msg_polling_thread.Start();
       }
@@ -79,10 +61,13 @@ namespace BCG {
       /// hanlder
       /// </summary>
       private void _polling_thread_func() {
+         Log.Gen.info("PubSubMessageReactor._polling_thread_func starting thread function _polling_thread_func");
          TimeSpan poll_timeout = new TimeSpan(0, 0, 0, 0, 2500);
          _msg_hub.activate();
          Action handle_internal_pubs = () => {
-            var internal_pubs = _internal_pub_queue.dequeueMultipleElements(1, 64000);
+            Log.Gen.info($"PubSubMessageReactor._polling_thread_func handle_internal_pubs");
+            var internal_pubs = _internal_pub_queue.dequeueMultipleElements(TimeSpan.FromSeconds(1), 64000);
+            //Log.Gen.info($"PubSubMessageReactor._polling_thread_func internal_pubs count: {internal_pubs.Count}");
             foreach (var msg in internal_pubs) {
                try {
                   _handle_pub("Internal", msg, true);
@@ -94,12 +79,16 @@ namespace BCG {
 
          while (true) {
             try {
-               var received_data = _msg_hub.get_updates(poll_timeout, 1);
+               //bts.Log.Gen.info($"PubSubMessageReactor._polling_thread_func pre get_updates");
+               var received_data = _msg_hub.get_all_updates();
                var n_msgs = received_data.Count;
-               if (n_msgs > 0) {
-                  bts.Log.Gen.info($"PubSubMessageReactor._polling_thread_func n_msgs: {n_msgs}");
+               if (n_msgs == 0) {
+                  System.Threading.Thread.Sleep(TimeSpan.FromMilliseconds(250));
+                  continue;
                }
+               bts.Log.Gen.info($"PubSubMessageReactor._polling_thread_func post get_updates n_msgs: {n_msgs}");
                foreach (var kv in received_data) {
+                  // multiple pubs for the same tags should be batched here
                   _handle_client_msg(kv.first, kv.second);
                }
                handle_internal_pubs();
@@ -111,8 +100,25 @@ namespace BCG {
             }
          }
       }
+      public static string to_hex(byte[] byte_arr, int start_pos, int end_pos) {
+         var len = end_pos - start_pos;
+         StringBuilder hex_builder = new StringBuilder(len * 2);
+         int i = 0;
+
+         foreach (byte b in byte_arr) {
+            if (i >= start_pos && i < end_pos)
+               hex_builder.Append($"{b:X2} ");
+            i++;
+         }
+         return hex_builder.ToString().Trim();
+      }
+
       // all messages (polled from the message_hub) sent to the client are handled here
       private void _handle_client_msg(string sender, byte[] msg) {
+         //_stop_watch.Restart();
+         var stop_watch = Stopwatch.StartNew();
+         //string msg_txt = to_hex(msg, 0, msg.Length);
+         Log.Gen.info($"PubSubMessageReactor._handle_client_msg sender: {sender} msg: {msg.Length}");
          // TODO: reduce object creation, C# has an ObjectPool
          PubSubClientMessage ps_client_msg = new PubSubClientMessage();
          ps_client_msg.consruct_deserialize(msg);
@@ -131,12 +137,15 @@ namespace BCG {
                _handle_pub(sender, ps_client_msg.pub_msg, false);
                break;
          }
+         stop_watch.Stop();
+         Log.Gen.info($"PubSubMessageReactor._handle_client_msg processed ps_client_msg: {ps_client_msg} processing time: {stop_watch.ElapsedMilliseconds} msecs");
       }
 
       private void _handle_sub(string sender, SubscribeMsg sub_msg) {
-         //TODO:  we make this correction here, since non managed clients are ntosending this correctly         
+         //TODO:  we make this correction here, since C++ clients are not sending this correctly         
          bts.Log.Gen.info($"PubSubMessageReactor._handle_sub sender: {sender} sub_msg: {sub_msg}");
-         if (String.IsNullOrEmpty(sub_msg.connection) || sub_msg.connection.Contains("unset")) {
+         if (String.IsNullOrEmpty(sub_msg.connection) || String.Compare(sender, sub_msg.connection) != 0) {
+            bts.Log.Gen.error($"PubSubMessageReactor._handle_sub correcting sub_msg.connection sender: {sender} sub_msg.connection: {sub_msg.connection}");
             sub_msg.connection = sender;
          }
          bts.Log.Gen.info($"PubSubMessageReactor._handle_sub(corrected) sender: {sender} sub_msg: {sub_msg}");
@@ -149,10 +158,10 @@ namespace BCG {
             //ie data has been published once
             if (data != null) {
                bts.Log.Gen.info($"ManagedTCPMessgeHub._handle_sub get_content tag: {tag} data len: {data.Length}");
-               string tagType = _state.getTagType(tag);
+               string tag_type_ = _state.getTagType(tag);
                var msg_to_send = new PubSubServerMessage() {
                   tag = tag,
-                  tag_type = tagType,
+                  tag_type = tag_type_,
                   data = data
                };
                bts.Log.Gen.info($"ManagedTCPMessgeHub._handle_sub sending msg_to_send: {msg_to_send} connection: {sub_msg.connection}");
@@ -170,7 +179,7 @@ namespace BCG {
          // this is where pubsub's data state is updated
          _state.update_pub_data(pub_msg);
          var sub_connections_for_tag = _state.get_subscriber_connections_for_tag(pub_msg.tag);
-         //bts.Log.Gen.info($"PubSubMessageReactor._handle_pub {sub_connections_for_tag.Count} subs for tag: {pub_msg.tag}");
+         bts.Log.Gen.info($"PubSubMessageReactor._handle_pub {sub_connections_for_tag.Count} subs for tag: {pub_msg.tag}");
          bts.PubSubServerMessage msg_to_send = new PubSubServerMessage() {
             tag = pub_msg.tag,
             tag_type = pub_msg.tag_type,

src/DataStreamSuite/PubSub/Server/PubSubSingleTagState.cs

@@ -1,126 +1,126 @@
-using System;
-using System.Collections.Generic;
-using System.Linq;
-using System.Text;
+// using System;
+// using System.Collections.Generic;
+// using System.Linq;
+// using System.Text;
 
-/*
- *  This class contains the recent channges of state of a single pubsub tag, with respect to who has subscribed, who has published, etc. 
- *  It has a list of new subscription events. 
- * 
- */
-namespace BCG {
-   public class PubSubSingleTagState {
-      private string myTag;
-      public string tag {
-         get { return myTag; }
-         set { myTag = value; }
+// /*
+//  *  This class contains the recent channges of state of a single pubsub tag, with respect to who has subscribed, who has published, etc. 
+//  *  It has a list of new subscription events. 
+//  * 
+//  */
+// namespace BCG {
+//    public class PubSubSingleTagState {
+//       private string myTag;
+//       public string tag {
+//          get { return myTag; }
+//          set { myTag = value; }
 
-      }
-      //we have this here as SortedList so that we can compare for equality
-      private SortedList<string, string> myPublishers;
-      private SortedList<string, string> mySubscribers;
-      private Dictionary<string, PubSubSingleTagStateEvent> myNewSubscribers;
-      private Dictionary<string, PubSubSingleTagStateEvent> myRemovedSubscribers;
-      private Dictionary<string, PubSubSingleTagStateEvent> myNewPublishers;
-      private Dictionary<string, PubSubSingleTagStateEvent> myRemovedPublishers;
+//       }
+//       //we have this here as SortedList so that we can compare for equality
+//       private SortedList<string, string> myPublishers;
+//       private SortedList<string, string> mySubscribers;
+//       private Dictionary<string, PubSubSingleTagStateEvent> myNewSubscribers;
+//       private Dictionary<string, PubSubSingleTagStateEvent> myRemovedSubscribers;
+//       private Dictionary<string, PubSubSingleTagStateEvent> myNewPublishers;
+//       private Dictionary<string, PubSubSingleTagStateEvent> myRemovedPublishers;
 
 
 
-      public PubSubSingleTagState() {
-         //the total state of the tag
-         myPublishers = new SortedList<string, string>();
-         mySubscribers = new SortedList<string, string>();
+//       public PubSubSingleTagState() {
+//          //the total state of the tag
+//          myPublishers = new SortedList<string, string>();
+//          mySubscribers = new SortedList<string, string>();
 
-         //how has the state changed recentlty
-         myNewSubscribers = new Dictionary<string, PubSubSingleTagStateEvent>();
-         myNewPublishers = new Dictionary<string, PubSubSingleTagStateEvent>();
-         myRemovedPublishers = new Dictionary<string, PubSubSingleTagStateEvent>();
-         myRemovedSubscribers = new Dictionary<string, PubSubSingleTagStateEvent>();
-      }
+//          //how has the state changed recentlty
+//          myNewSubscribers = new Dictionary<string, PubSubSingleTagStateEvent>();
+//          myNewPublishers = new Dictionary<string, PubSubSingleTagStateEvent>();
+//          myRemovedPublishers = new Dictionary<string, PubSubSingleTagStateEvent>();
+//          myRemovedSubscribers = new Dictionary<string, PubSubSingleTagStateEvent>();
+//       }
 
-      //returns a list of all the events that need to be published.
-      public PubSubSingleTagStateEvent[] getNewEventsForPublishing() {
-         PubSubSingleTagStateEvent[] newEvents = new PubSubSingleTagStateEvent[myNewSubscribers.Count + myNewPublishers.Count + myRemovedPublishers.Count + myRemovedSubscribers.Count];
+//       //returns a list of all the events that need to be published.
+//       public PubSubSingleTagStateEvent[] getNewEventsForPublishing() {
+//          PubSubSingleTagStateEvent[] newEvents = new PubSubSingleTagStateEvent[myNewSubscribers.Count + myNewPublishers.Count + myRemovedPublishers.Count + myRemovedSubscribers.Count];
 
-         myNewSubscribers.Values.CopyTo(newEvents, 0);
-         int index = myNewSubscribers.Count;
-         myNewPublishers.Values.CopyTo(newEvents, index);
-         index += myNewPublishers.Count;
-         myRemovedSubscribers.Values.CopyTo(newEvents, index);
-         index += myRemovedSubscribers.Count;
-         myRemovedPublishers.Values.CopyTo(newEvents, index);
+//          myNewSubscribers.Values.CopyTo(newEvents, 0);
+//          int index = myNewSubscribers.Count;
+//          myNewPublishers.Values.CopyTo(newEvents, index);
+//          index += myNewPublishers.Count;
+//          myRemovedSubscribers.Values.CopyTo(newEvents, index);
+//          index += myRemovedSubscribers.Count;
+//          myRemovedPublishers.Values.CopyTo(newEvents, index);
 
-         return newEvents;
+//          return newEvents;
 
-      }
+//       }
 
-      //call this after you have published the new events
-      //this is made public for code readability reasons 
-      //i.e. so that someone reading the SubInfoStateWatcher can understand what's going on more easily
-      public void resetNewEvents() {
-         myNewSubscribers = new Dictionary<string, PubSubSingleTagStateEvent>();
-         myNewPublishers = new Dictionary<string, PubSubSingleTagStateEvent>();
-         myRemovedSubscribers = new Dictionary<string, PubSubSingleTagStateEvent>();
-         myRemovedPublishers = new Dictionary<string, PubSubSingleTagStateEvent>();
-      }
+//       //call this after you have published the new events
+//       //this is made public for code readability reasons 
+//       //i.e. so that someone reading the SubInfoStateWatcher can understand what's going on more easily
+//       public void resetNewEvents() {
+//          myNewSubscribers = new Dictionary<string, PubSubSingleTagStateEvent>();
+//          myNewPublishers = new Dictionary<string, PubSubSingleTagStateEvent>();
+//          myRemovedSubscribers = new Dictionary<string, PubSubSingleTagStateEvent>();
+//          myRemovedPublishers = new Dictionary<string, PubSubSingleTagStateEvent>();
+//       }
 
-      public void newPublish(PubSubSingleTagStateEvent newPublisher) {
-         //don't do anything here if we already knew about the publishers because we were already dealing with this publishers
-         if (!myPublishers.ContainsKey(newPublisher.app)) {
-            myPublishers.Add(newPublisher.app, newPublisher.app);
+//       public void newPublish(PubSubSingleTagStateEvent newPublisher) {
+//          //don't do anything here if we already knew about the publishers because we were already dealing with this publishers
+//          if (!myPublishers.ContainsKey(newPublisher.app)) {
+//             myPublishers.Add(newPublisher.app, newPublisher.app);
 
-            //if we were going to delete it, that means it was already being published by this app
-            //that means we should no logner delete it, but we don't have to announce that it was published again
-            if (myRemovedPublishers.ContainsKey(newPublisher.app))
-               myRemovedPublishers.Remove(newPublisher.app);
-            else
-               myNewPublishers[newPublisher.app] = newPublisher;
+//             //if we were going to delete it, that means it was already being published by this app
+//             //that means we should no logner delete it, but we don't have to announce that it was published again
+//             if (myRemovedPublishers.ContainsKey(newPublisher.app))
+//                myRemovedPublishers.Remove(newPublisher.app);
+//             else
+//                myNewPublishers[newPublisher.app] = newPublisher;
 
 
-         }
-      }
+//          }
+//       }
 
-      public void removePublisher(PubSubSingleTagStateEvent removedPublisher) {
-         if (myPublishers.ContainsKey(removedPublisher.app)) {
-            myPublishers.Remove(removedPublisher.app);
+//       public void removePublisher(PubSubSingleTagStateEvent removedPublisher) {
+//          if (myPublishers.ContainsKey(removedPublisher.app)) {
+//             myPublishers.Remove(removedPublisher.app);
 
-            //same kind of logic here. If we were about to publish this new publisher, we no longer need to announce it
-            if (myNewPublishers.ContainsKey(removedPublisher.app))
-               myNewPublishers.Remove(removedPublisher.app);
-            else
-               myRemovedPublishers[removedPublisher.app] = removedPublisher;
-         }
+//             //same kind of logic here. If we were about to publish this new publisher, we no longer need to announce it
+//             if (myNewPublishers.ContainsKey(removedPublisher.app))
+//                myNewPublishers.Remove(removedPublisher.app);
+//             else
+//                myRemovedPublishers[removedPublisher.app] = removedPublisher;
+//          }
 
-      }
+//       }
 
-      public void newSubscription(PubSubSingleTagStateEvent newSubscriber) {
-         if (!mySubscribers.ContainsKey(newSubscriber.app)) {
-            mySubscribers.Add(newSubscriber.app, newSubscriber.app);
+//       public void newSubscription(PubSubSingleTagStateEvent newSubscriber) {
+//          if (!mySubscribers.ContainsKey(newSubscriber.app)) {
+//             mySubscribers.Add(newSubscriber.app, newSubscriber.app);
 
-            if (myRemovedSubscribers.ContainsKey(newSubscriber.app))
-               myRemovedSubscribers.Remove(newSubscriber.app);
-            else
-               myNewSubscribers[newSubscriber.app] = newSubscriber;
+//             if (myRemovedSubscribers.ContainsKey(newSubscriber.app))
+//                myRemovedSubscribers.Remove(newSubscriber.app);
+//             else
+//                myNewSubscribers[newSubscriber.app] = newSubscriber;
 
 
-         }
-      }
+//          }
+//       }
 
-      public void removeSubscriber(PubSubSingleTagStateEvent removedSubscriber) {
-         if (mySubscribers.ContainsKey(removedSubscriber.app)) {
-            mySubscribers.Remove(removedSubscriber.app);
-            //if we were going to add this subscriber, we no logner want to
-            //this means no event needs to be published
-            if (myNewSubscribers.ContainsKey(removedSubscriber.app))
-               myNewSubscribers.Remove(removedSubscriber.app);
-            else
-               //we have to remove this guy in this updated
-               myRemovedSubscribers[removedSubscriber.app] = removedSubscriber;
-         }
+//       public void removeSubscriber(PubSubSingleTagStateEvent removedSubscriber) {
+//          if (mySubscribers.ContainsKey(removedSubscriber.app)) {
+//             mySubscribers.Remove(removedSubscriber.app);
+//             //if we were going to add this subscriber, we no logner want to
+//             //this means no event needs to be published
+//             if (myNewSubscribers.ContainsKey(removedSubscriber.app))
+//                myNewSubscribers.Remove(removedSubscriber.app);
+//             else
+//                //we have to remove this guy in this updated
+//                myRemovedSubscribers[removedSubscriber.app] = removedSubscriber;
+//          }
 
 
-      }
+//       }
 
 
-   }
-}
+//    }
+// }

src/DataStreamSuite/PubSub/Server/PubSubState.cs

@@ -42,29 +42,8 @@ namespace BCG {
 
       #region subscription based state changes
 
-      /// <summary>
-      /// everytime a subscription is made, this gets called
-      /// by PubsubServer2. Adds a
-      /// TODO : subscibers sending sub twice from the client
-      /// </summary>
-      /// <param name="sub"></param>
-      /// <param name="md"></param>
-      // public void sub(SubMetaData sub) {
-      //    ///tag based key ds update
-      //    foreach (string tag in sub.tags) {
-      //       if (!_tag_to_connection_map.ContainsKey(tag)) {
-      //          _tag_to_connection_map.Add(tag, new SingleTagConnectionData(tag));
-      //       }
-      //       _tag_to_connection_map[tag].onSub(sub);
-
-      //       if (onSingleTagChange != null)
-      //          onSingleTagChange(new PubSubSingleTagStateEvent(tag, PubSubSingleTagStateEvent.EventType.subscription, sub.connection, sub.subscribing_app));
-      //    }
-      //    if (onTagMapChanged != null)
-      //       onTagMapChanged(sub.tags);
-      // }
-
       public void handle_sub(bts.SubscribeMsg sub_msg) {
+         Log.Gen.info($"PubSubState.handle_sub sub_msg: {sub_msg}");
          foreach (string tag in sub_msg.tags) {
             SingleTagConnectionData tag_conn_map = null;
             if (!_tag_to_connection_map.TryGetValue(tag, out tag_conn_map)) {
@@ -81,24 +60,6 @@ namespace BCG {
          // TODO
       }
 
-      // public void unsub(UnSubMetaData unsub) {
-      //    foreach (string tag in unsub.tags) {
-      //       if (_tag_to_connection_map.ContainsKey(tag)) {
-      //          ///maybe check here, that if there are no subscribers
-      //          ///and publishers for the tag then remove it.
-      //          _tag_to_connection_map[tag].onUnsub(unsub);
-      //          if (onSingleTagChange != null)
-      //             onSingleTagChange(new PubSubSingleTagStateEvent(tag, PubSubSingleTagStateEvent.EventType.unsubscription, unsub.connection, unsub.unsubscribing_app));
-      //       } else {
-      //          string warningMsg = String.Format("{0} was never published or subscribed on this pubsub, hence" +
-      //             " the unsub call by app {1} is voided", tag, unsub.unsubscribing_app);
-      //          //BlueEventLog.logEvent(warningMsg, BlueEventLevel.Warning);
-      //       }
-      //    }
-      //    if (onTagMapChanged != null)
-      //       onTagMapChanged(unsub.tags);
-      // }
-
       public void handle_unsub(bts.UnsubscribeMsg ubnsub_msg) {
          foreach (string tag in ubnsub_msg.tags) {
             SingleTagConnectionData conn_data = null;
@@ -138,48 +99,11 @@ namespace BCG {
          }
          if (onTagMapChanged != null)
             onTagMapChanged(tags_effected.ToArray());
-
-
       }
 
       #endregion
 
       #region data based state changes
-      /// <summary>
-      /// this is the only place in the state class where the
-      /// data is modified ; happens when data is published
-      /// </summary>
-      /// <param name="tag"></param>
-      /// <param name="msg"></param>
-      // public void updatePubData(PubMetaData pub, Stream msg) {
-      //    int sz = (int)(msg.Length - msg.Position);
-      //    bts.Log.Gen.info($"PubSubData.updatePubData pub: msg : {msg.Position} / {msg.Length} sz: {sz}");
-      //    ///this is the only place where SingleTagData should be created
-      //    if (!_tag_to_data_map.ContainsKey(pub.tag)) {
-      //       ///first time data is published for this tag
-      //       _tag_to_data_map.Add(pub.tag, new SingleTagData(pub.tag, pub.type));
-      //    }
-      //    _tag_to_data_map[pub.tag].pubUpdate(msg);
-
-      //    if (!_tag_to_connection_map.ContainsKey(pub.tag)) {
-      //       _tag_to_connection_map.Add(pub.tag, new SingleTagConnectionData(pub.tag));
-      //    }
-      //    bool pubChanged = false;
-      //    _tag_to_connection_map[pub.tag].onPub(pub, newPublisher: out pubChanged);
-      //    if (pubChanged) {
-      //       if (onTagMapChanged != null)
-      //          onTagMapChanged(new string[] { pub.tag });
-
-      //       if (onSingleTagChange != null)
-      //          onSingleTagChange(
-      //             new PubSubSingleTagStateEvent(pub.tag,
-      //                PubSubSingleTagStateEvent.EventType.publish,
-      //                pub.connection, pub.publishingApp));
-      //    }
-      //    if (onPublishStats != null)
-      //       onPublishStats(pub.tag, sz);
-      // }
-
       public void update_pub_data(bts.PublishMsg pub_msg) {
          try {
             bts.Log.Gen.info($"PubSubData.update_pub_data pub_msg: {pub_msg}");

src/DataStreamSuite/PubSub/Server/SingleTagConnectionData.cs

@@ -78,6 +78,7 @@ namespace BCG {
       }
 
       public bool handle_sub(SubscribeMsg sub_msg) {
+         Log.Gen.info($"SingleTagConnectionData.handle_sub sub_msg: {sub_msg}");
          if (_sub_conn_to_app_map.ContainsKey(sub_msg.connection)) {
             bts.Log.Gen.info($"tag: {_tag} has already been subscribed by connection: {sub_msg.connection}");
             return false;

src/DataStreamSuite/PubSub/Server/SingleTagData.cs

@@ -10,7 +10,8 @@ using BCG;
 
 namespace bts {
    /// <summary>
-   /// this is the data (not pub/subs/connections) for one tag. This is owned  by PubSubState and offers
+   /// this is the data (not pub/subs/connections) for one tag. This exists in both the PublisherCOre on the 
+   /// publisher side, as well as the PubSubServer
    /// 2 main functions
    /// 1. handle_publish, called in response to a pubsub publish
    /// 2. get_current_content, to retreive the state built in response to 1.
@@ -47,7 +48,7 @@ namespace bts {
       // PubsubState calls this
       public void handle_publish(PublishMsg pub_msg) {
          string handler_str = _tag_handler != null ? _tag_handler.GetType().ToString() : "NULL";
-         bts.Log.Gen.info($"SingleTagData.handle_publish pub_msg: {pub_msg} handler: {handler_str} ");
+         bts.Log.Gen.info($"SingleTagData.handle_publish tag: {_tag} pub_msg: {pub_msg} handler: {handler_str} ");
 
          try {
             _tag_handler.update(_tag, pub_msg.data_to_publish);
@@ -62,7 +63,9 @@ namespace bts {
       }
 
       public byte[] get_current_content() {
-         return _tag_handler.get_content(_tag);
+         byte[] rv = _tag_handler.get_content(_tag);
+         bts.Log.Gen.info($"SingleTagData.get_current_content tag: {_tag} rv: {rv}");
+         return rv;
       }
 
       private void _on_handler_internal_publish(ICollection<Triplet<string, string, byte[]>> cbItems) {

src/DataStreamSuite/PubSub/SingleServerSubscriberCore.cs

@@ -3,7 +3,9 @@ using bts;
 using System;
 
 using System.Collections.Generic;
+using System.ComponentModel;
 using System.Linq;
+using System.Runtime.InteropServices;
 
 namespace BCG {
    /// <summary>
@@ -27,9 +29,9 @@ namespace BCG {
 
       public event PubSubConnectionChange onPubSubConnectionChange;
 
-      public SingleServerSubscriberCore(string server_url, PubSubServerConnectionFactory connection_factory) {
-         bts.Log.Gen.info($"SingleServerSubscriberCore constructed for server_url: {server_url}");
-         _connection = connection_factory.getConnection(server_url);
+      public SingleServerSubscriberCore(RemoteConnectionInfo server_rci, PubSubServerConnectionFactory connection_factory) {
+         bts.Log.Gen.info($"SingleServerSubscriberCore constructed for server_url: {server_rci}");
+         _connection = connection_factory.get_connection(server_rci);
          _connection.register_connection_callback(new PubSubServerConnectionStatusChange(_handle_server_reboot));
          _connection.register_data_callback(_on_data);
       }
@@ -47,7 +49,11 @@ namespace BCG {
          foreach (var m in conn_data) {
             var msg = new PubSubServerMessage();
             msg.consruct_deserialize(m);
-            rv.Add(msg);
+            if (_tags.Contains(msg.tag))
+               rv.Add(msg);
+            else {
+               Log.Gen.warn("SingleSeverSubsciberCore.get_updates msg: {msg} does not have a subscription");
+            }
          }
          return rv;
       }
@@ -121,6 +127,7 @@ namespace BCG {
                tags = tag_args
             }
          };
+         Log.Gen.info($"SingleServerSubscriberCore._subscribe sending msg: {msg}");
          _connection.send(msg);
       }
 

src/DataStreamSuite/PubSub/SubscriberCore.cs

@@ -1,125 +1,126 @@
-using bts;
-
-using Bts.PubSub;
-
-using System;
-using System.Collections.Generic;
-
-namespace BCG {
-
-   /// <summary>
-   /// This class serves as the subscrition interface to the pubsub. The only methods
-   /// it exposes are getUpdates and subscribe. Internally, it manages a SingleServerSubscriberCore
-   /// foreach server. So this class does the routing part, and each SingleServerSubscriberCore
-   /// handles dealing sending (sub/unsub) and receiving (data) messages from the transport.
-   /// There is no sharing with respect to this class. Ie, it creates new SingleServerSubscriberCore
-   /// to the servers even though there might be so created by other instances of the SubscriberCore
-   /// 
-   /// </summary>
-   public class SubscriberCore : ISubscriberCore {
-      private Dictionary<string, ISingleServerSubscriberCore> _server_url_to_subcore_map = new Dictionary<string, ISingleServerSubscriberCore>();
-      private IPubSubRouter _tag_router;
-      private PubSubServerConnectionFactory _connection_factory = new PubSubServerConnectionFactory(true);
-
-      public event PubSubConnectionChange onPubSubConnectionChange;
-
-
-      #region constructors
-
-      public SubscriberCore() {
-         //this defaults to a basic pubsubrouter if the config key is not filled out
-         _tag_router = new PubSubRouter(Log.notnull(PubSubConfig.Instance.SubscriberRoutes).ToArray());
-      }
-      public SubscriberCore(string site_name) {
-         _tag_router = new PubSubRouter(PubSubConfig.Instance.SubscriberRoutesFor(site_name).ToArray());
-      }
-
-      private IPubSubRouter _get_router() =>
-         new PubSubRouter(Log.notnull(PubSubConfig.Instance.SubscriberRoutes).ToArray());
-
-      public SubscriberCore(PubSubRoute[] routes) {
-         _tag_router = new PubSubRouter(routes);
-      }
-
-      public SubscriberCore(IPubSubRouter router) {
-         _tag_router = router;
-      }
-
-      #endregion
-
-      public IEnumerable<bts.PubSubServerMessage> get_updates(TimeSpan timeout) {
-         List<bts.PubSubServerMessage> rv = new List<bts.PubSubServerMessage>();
-         foreach (var kv in _server_url_to_subcore_map) {
-            var sub_core = kv.Value;
-            rv.AddRange(sub_core.get_updates(timeout));
-         }
-         return rv;
-      }
-
-      private IDictionary<string, int> _n_update_map = new Dictionary<string, int>();
-      private void _log_n_updates(IEnumerable<PubSubUpdate> updates) {
-         foreach (PubSubUpdate u in updates) {
-            if (!_n_update_map.ContainsKey(u.tag)) {
-               _n_update_map.Add(u.tag, 0);
-            }
-            _n_update_map[u.tag] = _n_update_map[u.tag] + 1;
-         }
-         // log it now
-         foreach (var kv in _n_update_map) {
-            bts.Log.Gen.info($"StaticSubscrberCore tag: {kv.Key} n_updates: {kv.Value}");
-         }
-      }
-
-      public void subscribe(params string[] tags) {
-         lock (_server_url_to_subcore_map) {
-            foreach (string tag in tags) {
-               string server_url = _tag_router.get_sub_url_for_tag(tag);
-               var server_sub_core = _find_subscribercore(server_url);
-               server_sub_core.subscribe(tag);
-            }
-         }
-      }
-
-      private ISingleServerSubscriberCore _find_subscribercore(string server_url) {
-         ISingleServerSubscriberCore rv = null;
-         if (!_server_url_to_subcore_map.TryGetValue(server_url, out rv)) {
-            rv = new SingleServerSubscriberCore(server_url, _connection_factory);
-            _server_url_to_subcore_map.Add(server_url, rv);
-            rv.onPubSubConnectionChange += _handle_pubsubserver_connection_change;
-         }
-         return rv;
-      }
-
-
-      public void _handle_pubsubserver_connection_change(bool is_connected, string server_url) {
-         if (onPubSubConnectionChange != null)
-            onPubSubConnectionChange(is_connected, server_url);
-
-      }
-
-      public void unsubscribe(params string[] tags) {
-         lock (_server_url_to_subcore_map) {
-
-            foreach (string tag in tags) {
-               string server_url = _tag_router.get_sub_url_for_tag(tag);
-               var server_subcore = _find_subscribercore(server_url);
-               server_subcore.unsubscribe(tag);
-            }
-         }
-      }
-
-      public IPubSubRouter get_router() {
-         return _tag_router;
-      }
-
-      #region IDisposable Members
-
-      public void Dispose() {
-         foreach (var sssc in _server_url_to_subcore_map.Values)
-            sssc.Dispose();
-         _connection_factory.Dispose();
-      }
-
-      #endregion
-   }
-}
+using bts;
+
+using Bts.PubSub;
+
+using System;
+using System.Collections.Generic;
+
+namespace BCG {
+
+   /// <summary>
+   /// This class serves as the subscrition interface to the pubsub. The only methods
+   /// it exposes are getUpdates and subscribe. Internally, it manages a SingleServerSubscriberCore
+   /// foreach server. So this class does the routing part, and each SingleServerSubscriberCore
+   /// handles dealing sending (sub/unsub) and receiving (data) messages from the transport.
+   /// There is no sharing with respect to this class. Ie, it creates new SingleServerSubscriberCore
+   /// to the servers even though there might be so created by other instances of the SubscriberCore
+   /// 
+   /// </summary>
+   public class SubscriberCore : ISubscriberCore {
+      private Dictionary<RemoteConnectionInfo, ISingleServerSubscriberCore> _server_route_to_subcore_map =
+         new Dictionary<RemoteConnectionInfo, ISingleServerSubscriberCore>();
+      private IPubSubRouter _tag_router;
+      private PubSubServerConnectionFactory _connection_factory = new PubSubServerConnectionFactory(true);
+
+      public event PubSubConnectionChange onPubSubConnectionChange;
+
+
+      #region constructors
+
+      public SubscriberCore() {
+         //this defaults to a basic pubsubrouter if the config key is not filled out
+         _tag_router = new PubSubRouter(Log.notnull(PubSubConfig.Instance.SubscriberRoutes).ToArray());
+      }
+      public SubscriberCore(string site_name) {
+         _tag_router = new PubSubRouter(PubSubConfig.Instance.SubscriberRoutesFor(site_name).ToArray());
+      }
+
+      private IPubSubRouter _get_router() =>
+         new PubSubRouter(Log.notnull(PubSubConfig.Instance.SubscriberRoutes).ToArray());
+
+      public SubscriberCore(PubSubRoute[] routes) {
+         _tag_router = new PubSubRouter(routes);
+      }
+
+      public SubscriberCore(IPubSubRouter router) {
+         _tag_router = router;
+      }
+
+      #endregion
+
+      public IEnumerable<bts.PubSubServerMessage> get_updates(TimeSpan timeout) {
+         List<bts.PubSubServerMessage> rv = new List<bts.PubSubServerMessage>();
+         foreach (var kv in _server_route_to_subcore_map) {
+            var sub_core = kv.Value;
+            rv.AddRange(sub_core.get_updates(timeout));
+         }
+         return rv;
+      }
+
+      private IDictionary<string, int> _n_update_map = new Dictionary<string, int>();
+      private void _log_n_updates(IEnumerable<PubSubUpdate> updates) {
+         foreach (PubSubUpdate u in updates) {
+            if (!_n_update_map.ContainsKey(u.tag)) {
+               _n_update_map.Add(u.tag, 0);
+            }
+            _n_update_map[u.tag] = _n_update_map[u.tag] + 1;
+         }
+         // log it now
+         foreach (var kv in _n_update_map) {
+            bts.Log.Gen.info($"StaticSubscrberCore tag: {kv.Key} n_updates: {kv.Value}");
+         }
+      }
+
+      public void subscribe(params string[] tags) {
+         lock (_server_route_to_subcore_map) {
+            foreach (string tag in tags) {
+               RemoteConnectionInfo server_rci = _tag_router.get_sub_route_for_tag(tag);
+               var server_sub_core = _find_subscribercore(server_rci);
+               server_sub_core.subscribe(tag);
+            }
+         }
+      }
+
+      private ISingleServerSubscriberCore _find_subscribercore(RemoteConnectionInfo server_rci) {
+         ISingleServerSubscriberCore rv = null;
+         if (!_server_route_to_subcore_map.TryGetValue(server_rci, out rv)) {
+            rv = new SingleServerSubscriberCore(server_rci, _connection_factory);
+            _server_route_to_subcore_map.Add(server_rci, rv);
+            rv.onPubSubConnectionChange += _handle_pubsubserver_connection_change;
+         }
+         return rv;
+      }
+
+
+      public void _handle_pubsubserver_connection_change(bool is_connected, string server_url) {
+         if (onPubSubConnectionChange != null)
+            onPubSubConnectionChange(is_connected, server_url);
+
+      }
+
+      public void unsubscribe(params string[] tags) {
+         lock (_server_route_to_subcore_map) {
+
+            foreach (string tag in tags) {
+               var server_rci = _tag_router.get_sub_route_for_tag(tag);
+               var server_subcore = _find_subscribercore(server_rci);
+               server_subcore.unsubscribe(tag);
+            }
+         }
+      }
+
+      public IPubSubRouter get_router() {
+         return _tag_router;
+      }
+
+      #region IDisposable Members
+
+      public void Dispose() {
+         foreach (var sssc in _server_route_to_subcore_map.Values)
+            sssc.Dispose();
+         _connection_factory.Dispose();
+      }
+
+      #endregion
+   }
+}

src/DataStreamSuite/PubSub/Testing/SimpleTCPClient.cs

@@ -15,7 +15,7 @@ using System.Transactions;
 namespace BCG {
 
    public class SimpleTCPClient : ITCPConnectionHandler, ITCPMessageHandler {
-      private readonly BlutoSubAddress _server_addr;
+      private readonly RemoteConnectionInfo _server_addr;
       private int _m_msgs_received = 0;
       private readonly UInt32 _msg_sz = 1024;// * 1024;
       private readonly int _max_n_msgs = Int32.MaxValue;
@@ -23,10 +23,11 @@ namespace BCG {
       int _send_wait_period_ms = 5000;
       Action<string> _listener;
 
-      public SimpleTCPClient(string server_ip, int server_port, UInt32 msg_sz, int send_wait_period_ms) {
+      public SimpleTCPClient(string server_ip, int server_port, UInt32 msg_sz, int send_wait_period_ms, int max_n_msgs) {
+         _max_n_msgs = max_n_msgs;
          _msg_sz = msg_sz;
          _send_wait_period_ms = send_wait_period_ms;
-         _server_addr = new BlutoSubAddress(server_ip, server_port);
+         _server_addr = new RemoteConnectionInfo(server_ip, server_port);
       }
 
       public void start() {
@@ -34,7 +35,10 @@ namespace BCG {
       }
 
       private void _connect() {
-         bts.AsyncTCPConnector_APM.connect_to(_server_addr, this);
+         Log.Gen.info($"SimpleTCPClient._connect to: {_server_addr}");
+
+         bts.AsyncTCPConnector_AsyncWrapper.connect_to(_server_addr, this);
+         //bts.AsyncTCPConnector_APM.connect_to(_server_addr, this);
       }
 
       public void register_listener(Action<string> cb) {
@@ -43,10 +47,10 @@ namespace BCG {
 
       public void onConnection(ITCPConnection connection) {
          string ip; int port;
-         connection.getPeerAddress(out ip, out port);
-         bts.Log.Gen.info($"SimpleTCPClientAsync.onConnection connected to server ip: {ip} port: {port}, starting task to send data in a loop");
+         var peer_rci = connection.get_peer_address();
+         bts.Log.Gen.info($"SimpleTCPClient.onConnection connected to server: {peer_rci}, starting task to send data in a loop");
          if (_listener != null) {
-            _listener.Invoke($"Connected to server ip: {ip} port {port}");
+            _listener.Invoke($"Connected to server {peer_rci}");
          }
          connection.register(this);
          // start sending data
@@ -56,14 +60,23 @@ namespace BCG {
 
       private void _keep_sending_data(ITCPConnection connection) {
          byte[] buff = new byte[_msg_sz];
-         for (int i = 0; i < buff.Length; i++) {
-            buff[i] = (byte)i;
-         }
+         MemoryStream str = new MemoryStream(buff);
+         BinaryWriter bw = new BinaryWriter(str);
+         Action<int> fill_msg = (rep_i) => {
+            str.Position = 0;
+            int n_ints = (int)_msg_sz / 4;
+            for (int i = 0; i < n_ints; i++) {
+               bw.Write(rep_i);
+            }
+         };
+
+
          for (int i = 0; i < _max_n_msgs; i++) {
             if (!_send_flag) {
                return;
             }
-            bts.Log.Gen.info("Sending msg: {i} sz: {buff.Length}");
+            fill_msg(1);
+            bts.Log.Gen.info($"Sending msg: {i} sz: {buff.Length}");
             System.Threading.Thread.Sleep(_send_wait_period_ms);
             connection.send(buff);
             _listener.Invoke($"Data sent, sz: {buff.Length}");
@@ -72,9 +85,11 @@ namespace BCG {
 
       public void onData(RemoteMessage msg) {
          _m_msgs_received++;
-         Console.WriteLine($"SimpleTCPClientAsync.onData n_msgs: {_m_msgs_received} msg: {msg}");
+         if (msg.messageType == RemoteMessageType.Connection)
+            Log.Gen.error($"SimpleTCPClient.onData n_msgs: {_m_msgs_received} msg: {msg}");
          _listener.Invoke($"Message received msg: {msg}");
          if (msg.messageType == RemoteMessageType.Disconnection) {
+            Log.Gen.error($"SimpleTCPClient.onData disconnected, reconnecting");
             _send_flag = false;
             _connect();
          }

src/DataStreamSuite/PubSub/Testing/SimpleTCPServer.cs

@@ -8,25 +8,23 @@ namespace BCG {
    public class SimpleTCPServer : ITCPConnectionHandler, ITCPMessageHandler {
       int _n_msgs_received = 0;
       bool _isDisposed = false;
-      AsyncTCPConnectionAcceptor_APM _acceptor;
+      AsyncTCPConnectionAcceptor_AsyncWrapper _acceptor;
       IDictionary<RemoteConnectionInfo, ITCPConnection> _conn_map = new Dictionary<RemoteConnectionInfo, ITCPConnection>();
       readonly bool _echo_msg_back;
 
       public SimpleTCPServer(int port, bool echo_msg_back) {
-         _acceptor = new AsyncTCPConnectionAcceptor_APM(port, this);
+         Log.Gen.info($"SimpleTCPServer.const port: {port} echo_msg_back: {echo_msg_back}");
+         _acceptor = new AsyncTCPConnectionAcceptor_AsyncWrapper(port, this);
          _echo_msg_back = echo_msg_back;
       }
 
       public void init() {
-         //_acceptor = new AsyncTCPConnectionAcceptor_APM(12345, this);
-         _acceptor.init();
+         _acceptor.start();
       }
 
 
       public void onConnection(ITCPConnection connection) {
-         string ip; int port;
-         connection.getPeerAddress(out ip, out port);
-         RemoteConnectionInfo rci = new RemoteConnectionInfo(ip, port);
+         RemoteConnectionInfo rci = connection.get_peer_address();
          _conn_map[rci] = connection;
          bts.Log.Gen.info($"SimpleTCPServer.onConnection new client connected conn: {rci}");
          connection.register(this);
@@ -34,7 +32,8 @@ namespace BCG {
 
 
       public void onData(RemoteMessage msg) {
-         var log_msg = $"SimpleTCPServer.onData msg: {msg}";
+         _n_msgs_received++;
+         var log_msg = $"SimpleTCPServer.onData msgs_received: {_n_msgs_received} msg: {msg} _echo_msg_back: {_echo_msg_back}";
          Log.Gen.info(log_msg);
          Console.WriteLine(log_msg);
          if (!_echo_msg_back) {
@@ -44,11 +43,10 @@ namespace BCG {
          ITCPConnection this_conn;
          bool found = _conn_map.TryGetValue(rci, out this_conn);
          if (!found) {
-            Console.WriteLine("Could not find connection !!!!!!");
+            Log.Gen.error($"Could not find connection: {rci}");
             return;
          }
-         _n_msgs_received++;
-         Console.WriteLine($"\n\nSimpleTCPServerAsync.onData msgs_received: {_n_msgs_received} msg: {msg} ci: {rci} NOW echoing it back\n\n");
+         Log.Gen.info($"\n\n\nSimpleTCPServerAsync.onData msgs_received: {_n_msgs_received} msg: {msg} ci: {rci} NOW echoing it back\n\n\n");
          if (msg.messageType == RemoteMessageType.Data) {
             this_conn.send(msg.message);
          } else {

src/DataStreamSuite/PubSub/documentation/internal_readme.txt

@@ -0,0 +1,117 @@
+Basic TCP:
+ITCPConnection implemented by AsyncTCPConnection_APM is the innermost class resposinsble for TCP sends and received. Its used 
+on both client and server symmetrically. All the hard async send/receive logic is there.
+AsyncTCPConnectionAcceptor_APM opens a port and listens and accepts to connections from the client. It creates the ITCPConnection
+on the server.
+AsyncTCPConnector_APM connects to the server (above) and creates an ITCPConnection on the client side.
+
+IPubSubServerConnection:
+All communication to PubSubServer happens using this. It has 3 main types of calls:
+1. send: to asyncronoulsy end data
+2. get data: either through a polling call or callback
+3. get connection info: through callbacks
+
+The only implementation of the above is PubSubServerManagedTCPConnection
+(there used to be an MSMQ one and another wrapped around native code - but neither exists anymore)
+
+PubSubMessages:
+PubSubClientMessage is the only messages thats sent from client to pubsubserver. Its serialized on the
+client and sent over to the pubsubserver, where uints received ad deserialized
+
+SubscriberCore:ISubscriberCore (retired now)
+|
+V
+SingleServerSubscriberCore:ISingleServerSubscriberCore
+
+PublisherCore:IPublisherCore (transactions not supported for now), keeps AllPublishedTagState to cache state on the publisher
+|
+V
+IPubSubServerConnection
+
+PubSubServer:
+PubSubMessageReactor (top level, called from main), contain PubSubState (which has all data and connection state)
+ManagedTCPMessageHub : IPubsubMessageHub (this is what handles each ITCPConnection for each connection)
+
+PubSubState keeps the total pubsub data in 2 classes
+Dictionary<string, SingleTagData> and 
+Dictionary<string, SingleTagConnectionData>
+SingleTagData has the tag and IPubSubServerTagHandler (or byte[]) (IPubSubServerTagHandler holds the data state)
+for streams the IPubSubServerTagHandler is the PubsubServerTagHandler, which is designed for multiple tags and has a IDictionary<string, TagStateBank>
+TagStateBank containes the PubsubDeserial izationController, which knows how to deserialize the stream publishes and build state, ad simialirly produce the state on demand
+PubsubTagHandlerFactory returns a IPubSubServerTagHandler for each tag, called by the server state handler, SingleTagData and publisher tag state, AllPublishedTagState
+
+SingleTagConnectionData contains connection vs sub, and connection vs pub
+PubSubState calls
+SystemSubscriptionMapPublisher.init(this);
+SystemStatsPublisher.init(this);
+SystemHeartbeatPublisher.init();
+---------------------------------------------------------------
+How internal tags are published?
+There are 3 internal publishers (ie compute somerthing an dpublish tit PubSubServer, such that clients can subscribe to it):
+SystemStatsPublisher, SystemSubscriptionMapPublisher and SystemHeartbeatPublisher
+These are somewhat tricky because they are both listeners to PubUsbState event and intrnally publish, but are also IPubSubServerTagHandler
+SystemSubscriptionMapPublisher:
+just one instance, a static init is called by PubSubState and it passes itself. SystemSubscriptionMapPublisher subscribes to the event
+gets the affected tags, marks them dirty, in doWork creates the state and calls the internal_publish.
+Now, as the result of an internal_publish (which is pretty much like a rmote publish as far as the PubsubServer is concerned), SingleTagData
+ends up calling SystemSubscriptionMapPublisher (since it also serves as the tagHandler for the tag it publishes via internal_publish)
+
+
+
+PubSubState has onPublishStats (tag, sz) that is fired on update_pub_data
+onPublishStats is subscribed by SystemStatsPublisher
+SystemStatsPublisher.onPublishStats updates the state and on the next doWork calls PubSubMessageReactor.internal_send
+SystemStatsPublisher is also an IPubSubServerTagHandler so SingleTagData calls it getContent to get full state
+
+Streams:
+PubTableBroker allows publishing of streams through createTable, addRow, deleteRow, changeRow, makeBatchChanges ->
+PubTablePublisher:IPubTablePublisher per tag (addRow, deleteRow, changeRow, createTable, batchRemoveAdd)
+PubTablePublisher has BlueDataSerializationController:IBlueDataSerializer to serialize the rows to byte[]
+BlueDataSerializationController uses a IBlueDataSerializer, which is specialized for the tag (using UniversalSerializer for now)
+
+IBlueDataSerializer has just one function
+void serialize (byte op, IBlueRowCollection[] rows, out byte[] serializedData);
+the second arg is IBlueRowCollection[] because in case of op = StreamOpCode.multipleRemoveAdd, it has multiple IBlueRowCollection for rem and add
+(BlueDataSerializer.serialize's comment has the serialization layout)
+
+For each serializer we have 2 matching deserializers, IPubsubDeserializer (for PubSubservr) and IClientDeserializer (for clients subscribing to streams)
+For UniversalSerializer, we have PubsubUniversalDeserializer and ClientUniversalDeserializer
+
+-------------------------------------------------------------------
+TCP publishing:
+To run the base TCP layer run PubSubServer as an echo server, 
+(set run_echo_tcp_server_only in spark\src\DataStreamSuite\PubSubServer\Program.cs)
+to run a client that sends a tcp message tot he echo server, run 
+SimpleTCPEchoStream
+
+
+Pubsub publishing:
+-----------------
+PubSubTestingStream (bounces pubs across the pubsub server)
+mutableAppend(pubSubTesting(100,10000000,100000,"test_1"),
+pubSubTesting(250,10000000,100000,"test_2"))
+
+
+Stream publishing:
+------------------
+PublisherStream (publishes any stream to the pubsubserver)
+r=second(100)
+r=project(r,"id")
+r=math(r,"x = id*id")
+publisher(r, "test_x")
+
+
+tracking message into the pubsubserver
+--------------------------------------
+connection:
+ServerToClientManagedTCPConnectionAcceptor._on_accept
+ var log_msg = $"ServerToClientManagedTCPConnectionAcceptor._on_accept called port: {_port} state: {res.AsyncState}";
+         Log.Gen.info(log_msg);
+
+data:
+ManagedTCPMessageHub.onData
+bts.Log.Gen.info($"ManagedTCPMessgageHub.onData recv msg: {message}");
+PubSubMessageReactor._polling_thread_func
+bts.Log.Gen.info($"PubSubMessageReactor._polling_thread_func n_msgs: {n_msgs}");
+PubSubMessageReactor._handle_client_msg
+bts.Log.Gen.info($"PubSubMessageReator._handle_client_msg ps_client_msg: {ps_client_msg}");

src/DataStreamSuite/PubSub/documentation/readme.txt

@@ -1,117 +1,58 @@
-Basic TCP:
-ITCPConnection implemented by AsyncTCPConnection_APM is the innermost class resposinsble for TCP sends and received. Its used 
-on both client and server symmetrically. All the hard async send/receive logic is there.
-AsyncTCPConnectionAcceptor_APM opens a port and listens and accepts to connections from the client. It creates the ITCPConnection
-on the server.
-AsyncTCPConnector_APM connects to the server (above) and creates an ITCPConnection on the client side.
+If the DSQA comes up empty, we need to take it one step at a time and ensure it in the following order:
 
-IPubSubServerConnection:
-All communication to PubSubServer happens using this. It has 3 main types of calls:
-1. send: to asyncronoulsy end data
-2. get data: either through a polling call or callback
-3. get connection info: through callbacks
+--------------------
+PubsubServer routes:
+--------------------
+the client (both publisher and subscriber) need to connect tot he rigth pubsubserver. There are 3 days to do this:
+1. Bring up a DSQA and check the title bar, it should be tag:regExp: PubSunRoute
+the ip and port matter, the prefix is just fo the human eye.
+2. Run the stream pss_routes().
+It will give a row per tag reg exp vs the pss route (pss == pubsub server0)
+3. Check the clirnt logs for:
+PubSubRouter.get_sub_url_for_tag tag: and PubSubRouter.get_pub_url_for_tag tag:
 
-The only implementation of the above is PubSubServerManagedTCPConnection
-(there used to be an MSMQ one and another wrapped around native code - but neither exists anymore)
+-------------------
+Connection Attempt:
+-------------------
+Check the pss logs for:
+"AsyncTCPConnectionAcceptor_AsyncWrapper.cons started on port:"
+Make sure it is listening to the correct port
 
-PubSubMessages:
-PubSubClientMessage is the only messages thats sent from client to pubsubserver. Its serialized on the
-client and sent over to the pubsubserver, where uints received ad deserialized
-
-SubscriberCore:ISubscriberCore (retired now)
-|
-V
-SingleServerSubscriberCore:ISingleServerSubscriberCore
-
-PublisherCore:IPublisherCore (transactions not supported for now), keeps AllPublishedTagState to cache state on the publisher
-|
-V
-IPubSubServerConnection
-
-PubSubServer:
-PubSubMessageReactor (top level, called from main), contain PubSubState (which has all data and connection state)
-ManagedTCPMessageHub : IPubsubMessageHub (this is what handles each ITCPConnection for each connection)
-
-PubSubState keeps the total pubsub data in 2 classes
-Dictionary<string, SingleTagData> and 
-Dictionary<string, SingleTagConnectionData>
-SingleTagData has the tag and IPubSubServerTagHandler (or byte[]) (IPubSubServerTagHandler holds the data state)
-for streams the IPubSubServerTagHandler is the PubsubServerTagHandler, which is designed for multiple tags and has a IDictionary<string, TagStateBank>
-TagStateBank containes the PubsubDeserial izationController, which knows how to deserialize the stream publishes and build state, ad simialirly produce the state on demand
-PubsubTagHandlerFactory returns a IPubSubServerTagHandler for each tag, called by the server state handler, SingleTagData and publisher tag state, AllPublishedTagState
-
-SingleTagConnectionData contains connection vs sub, and connection vs pub
-PubSubState calls
-SystemSubscriptionMapPublisher.init(this);
-SystemStatsPublisher.init(this);
-SystemHeartbeatPublisher.init();
----------------------------------------------------------------
-How internal tags are published?
-There are 3 internal publishers (ie compute somerthing an dpublish tit PubSubServer, such that clients can subscribe to it):
-SystemStatsPublisher, SystemSubscriptionMapPublisher and SystemHeartbeatPublisher
-These are somewhat tricky because they are both listeners to PubUsbState event and intrnally publish, but are also IPubSubServerTagHandler
-SystemSubscriptionMapPublisher:
-just one instance, a static init is called by PubSubState and it passes itself. SystemSubscriptionMapPublisher subscribes to the event
-gets the affected tags, marks them dirty, in doWork creates the state and calls the internal_publish.
-Now, as the result of an internal_publish (which is pretty much like a rmote publish as far as the PubsubServer is concerned), SingleTagData
-ends up calling SystemSubscriptionMapPublisher (since it also serves as the tagHandler for the tag it publishes via internal_publish)
+On the client, make sure it is attemping to connect to the right pss
+"AsyncTCPConnector_AsyncWrapper.connect_to server:"
 
 
+------------------------------------
+Successful connection establishment:
+------------------------------------
 
-PubSubState has onPublishStats (tag, sz) that is fired on update_pub_data
-onPublishStats is subscribed by SystemStatsPublisher
-SystemStatsPublisher.onPublishStats updates the state and on the next doWork calls PubSubMessageReactor.internal_send
-SystemStatsPublisher is also an IPubSubServerTagHandler so SingleTagData calls it getContent to get full state
+On pss, a successful connection should result in the log:
+"calling onConnection now"
+The whole log is 
+Log.Gen.info($"AsyncTCPConnectionAcceptor_AsyncWrapper._begin_accepting port: {_accept_port}, new connection, self: {conn.get_self_address()} peer: {conn.get_peer_address()} calling onConnection now");
 
-Streams:
-PubTableBroker allows publishing of streams through createTable, addRow, deleteRow, changeRow, makeBatchChanges ->
-PubTablePublisher:IPubTablePublisher per tag (addRow, deleteRow, changeRow, createTable, batchRemoveAdd)
-PubTablePublisher has BlueDataSerializationController:IBlueDataSerializer to serialize the rows to byte[]
-BlueDataSerializationController uses a IBlueDataSerializer, which is specialized for the tag (using UniversalSerializer for now)
-
-IBlueDataSerializer has just one function
-void serialize (byte op, IBlueRowCollection[] rows, out byte[] serializedData);
-the second arg is IBlueRowCollection[] because in case of op = StreamOpCode.multipleRemoveAdd, it has multiple IBlueRowCollection for rem and add
-(BlueDataSerializer.serialize's comment has the serialization layout)
-
-For each serializer we have 2 matching deserializers, IPubsubDeserializer (for PubSubservr) and IClientDeserializer (for clients subscribing to streams)
-For UniversalSerializer, we have PubsubUniversalDeserializer and ClientUniversalDeserializer
-
--------------------------------------------------------------------
-TCP publishing:
-To run the base TCP layer run PubSubServer as an echo server, 
-(set run_echo_tcp_server_only in spark\src\DataStreamSuite\PubSubServer\Program.cs)
-to run a client that sends a tcp message tot he echo server, run 
-SimpleTCPEchoStream
+On the client, a successful connection should result in the log:
+"AsyncTCPConnector_AsyncWrapper._connect_to new connection,"
 
 
-Pubsub publishing:
------------------
-PubSubTestingStream (bounces pubs across the pubsub server)
-mutableAppend(pubSubTesting(100,10000000,100000,"test_1"),
-pubSubTesting(250,10000000,100000,"test_2"))
+------------------------------------
+Successful message sent from client:
+------------------------------------
+"AsyncTCPConnection_AsyncWrapper._send_fixed_size_msg sent"
 
-
-Stream publishing:
-------------------
-PublisherStream (publishesa any stream to the pubsubserver)
-r=second(100)
-r=project(r,"id")
-r=math(r,"x = id*id")
-publisher(r, "test_x")
-
-
-tracking message into the pubsubserver
 --------------------------------------
-connection:
-ServerToClientManagedTCPConnectionAcceptor._on_accept
- var log_msg = $"ServerToClientManagedTCPConnectionAcceptor._on_accept called port: {_port} state: {res.AsyncState}";
-         Log.Gen.info(log_msg);
+Successful message received on client:
+--------------------------------------
+"AsyncTCPConnection_AsyncWrapper._read_fixed_size_msg read"
 
-data:
-ManagedTCPMessageHub.onData
-bts.Log.Gen.info($"ManagedTCPMessgageHub.onData recv msg: {message}");
-PubSubMessageReactor._polling_thread_func
-bts.Log.Gen.info($"PubSubMessageReactor._polling_thread_func n_msgs: {n_msgs}");
-PubSubMessageReactor._handle_client_msg
-bts.Log.Gen.info($"PubSubMessageReator._handle_client_msg ps_client_msg: {ps_client_msg}");
+
+pss's send/receive is the same log, since it uses the same class AsyncTCPConnection_AsyncWrapper
+---------------------------------
+Successful message sent from pss:
+---------------------------------
+"AsyncTCPConnection_AsyncWrapper._send_fixed_size_msg sent"
+
+-----------------------------------
+Successful message received on pss:
+-----------------------------------
+"AsyncTCPConnection_AsyncWrapper._read_fixed_size_msg read"

src/DataStreamSuite/PubSubServer/Configurations/PubsubServer_Linux.json

@@ -0,0 +1,61 @@
+{
+   "user": "bts",
+   "pubsub": {
+      "milliseconds_between_sub_info_updates": 1000,
+      "milliseconds_between_time_slice_updates": 5000,
+      "milliseconds_between_no_buffer_space_retries": 100,
+      "retries_on_no_buffer_space": 1,
+      "do_it_all_seconds_between_logs": 0,
+      "heartbeat_interval": 15000,
+      "missing_heartbeats_before_gc": 50,
+      "use_managed_sockets": true,
+      "state_update_period": 5000,
+      "statistics_update_period": 5000,
+      "disposed_connection_check_period": 1000,
+      "message_processing_warning_threshold": 1000,
+      "server_port": 12345,
+      "publisher_routes": [
+         {
+            "name": "default",
+            "server_address": "bts-cloud://192.168.1.80:12345",
+            "tag": ".*"
+         }
+      ],
+      "subscriber_routes": [
+         {
+            "name": "default",
+            "server_address": "bts-cloud://192.168.1.80:12345",
+            "tag": ".*"
+         }
+      ],
+      "max_packet_size": 500000000,
+      "blue_comm_must_die": true,
+      "crushable_tags": [],
+      "tag_type_to_assembly_map": [
+         {
+            "handler_class": "BCG.PubSubServerTagHandler",
+            "assembly": "DataStreamCore.dll"
+         }
+      ],
+      "server_queue": "PubSubQueue",
+      "port_range": {
+         "first": 30400,
+         "last": 31700
+      },
+      "bind_to_random_port": false,
+      "debug_tags": "^NOTAGS$",
+      "latency_report_threshold_ms": 1500,
+      "use_substations": false,
+      "use_dynamic_clients": false,
+      "is_substation": false,
+      "internally_publish_sub_info": true,
+      "use_managed_clients_in_native_code": false,
+      "compressed_tags": "^NOTAGS$",
+      "tags_to_report_compression_ratio": "^NOTAGS$",
+      "disable_msmq": false,
+      "use_prepublished_server_state": false,
+      "ip_translation": null,
+      "gc_activation_threshold": 0,
+      "wait_period_after_gc": 1000
+   }
+}

src/DataStreamSuite/PubSubServer/Program.cs

@@ -12,6 +12,7 @@ using mbts.config;
 using Spark.Core;
 
 using System;
+using System.Collections.Specialized;
 using System.Threading;
 using System.Threading.Tasks;
 
@@ -24,8 +25,11 @@ int Run(PubSubServerConfig cfg) {
       cfg.MergeWith(PubSubServerConfig.Default());
       PubSubConfig.SetGlobalConfig(Log.notnull(cfg.PubSub));
       Configuration.initialize(cfg.Sparkd);
+      bool run_echo_server = false;
+      bool echo_msg_back = false;
+      cfg.SimpleEchoServer = run_echo_server;
       return cfg.SimpleEchoServer ?? false ?
-         RunEchoServer(cfg.PubSub.ServerPort.assert_value()) :
+         RunEchoServer(cfg.PubSub.ServerPort.assert_value(), echo_msg_back) :
          RunPubSubServer();
    } catch (Exception ex) {
       // Write the error to console first, in case the logging system is compromised.
@@ -58,11 +62,14 @@ int RunPubSubServer() {
    return 0;
 }
 
-int RunEchoServer(short port) {
-   Console.WriteLine("Running as TCP Echo server, Hit enter to continue");
+int RunEchoServer(short port, bool echo) {
+   string log_msg = $"Running as TCP Echo server, port: {port} echo: {echo} Hit enter to continue:";
+   Console.WriteLine(log_msg);
    Console.ReadLine();
-   Console.WriteLine($"ready to listen to port: {port}");
-   var sts = new SimpleTCPServer(port, true);
+   var ready_msg = $"ready to listen to port: {port}";
+   Log.Gen.info(ready_msg);
+   Console.WriteLine(ready_msg);
+   var sts = new SimpleTCPServer(port, echo);
    sts.init();
    Console.CancelKeyPress += OnCancelKeyPress;
    while (true) {

src/DataStreamSuite/SharedStreams/Business/BenzingaNews/BenzingaNewsStream.cs

@@ -7,26 +7,45 @@ namespace bts {
    using System.Text.Json;
    using System.Text.Json.Nodes;
    [DataStreamUserRespresentation("benzingaNews")]
-   public class BenzingaNewStream : ViewStream {
+   public class BenzingaNewsStream : ViewStream {
       private IBlueTableSchema _schema = null;
       private int _id = 0;
-      // constructor takes in the raw mongo db stream, and the time to live (ttl), "7d" for example
-      public BenzingaNewStream(ViewStream mongodb_raw_stream, string ttl) {
+
+
+      private static ViewStream _pre_process_raw_kvstream(ViewStream mongo_db_str) {
+         ViewStream r = new KeyValueToRelationalStream(mongo_db_str, "doc_id", "key", "value");
+         r = new DateStandardizationStream(r, "created_at", "ISO8601");
+         return r;
+      }
+
+      public BenzingaNewsStream(string connection_string, string db_name, string collection_name, string poll_period, string ttl)
+      : this(new MongoDBStream(connection_string, db_name, collection_name, poll_period), ttl) {
+      }
+
+      // public BenzingaNewsStream()
+      // : this(new MongoDBStream(DataStreamPub connection_string, db_name, collection_name, poll_period), ttl) {
+      // }
+
+      // assumes it gets a raw mongodb_raw_kv_stream with key value pairs
+      public BenzingaNewsStream(ViewStream mongodb_raw_kv_stream, string ttl) {
+         ViewStream output_stream = _pre_process_raw_kvstream(mongodb_raw_kv_stream);
          // Hiding body (with project, columns can be hidden, or renamed)
-         ViewStream output_stream = new ProjectStream(mongodb_raw_stream, "!body");
+         output_stream = new ProjectStream(output_stream, "!body");
          // add column news_source = 'Benzinga', so we can append other news sources
          output_stream = new AddColumnsStream(output_stream, @"news_source : Benzinga");
          // now SuperMapStream is used to map one row into multiple, one per symbol,
          // (SuperMapStream takes 2 lambdas, one to get the enw schema and another to 
          // return an array of rows, per one row)
-         output_stream = new SuperMapStream(mongodb_raw_stream,
+         output_stream = new SuperMapStream(output_stream,
 
         (IBlueTableSchema s) => {
            // this lambda returns the new schema (ie with extra cols symbol, exchange, is_primary_exchange)
            if (_schema == null) {
               var out_schema = s;
               out_schema = BlueDataUtility.remove_columns(out_schema, "securities");
-              out_schema = BlueDataUtility.add_column(s, "doc_id_internal", typeof(string));
+              out_schema = BlueDataUtility.remove_columns(out_schema, "body");
+              out_schema = BlueDataUtility.remove_columns(out_schema, "tags");
+              out_schema = BlueDataUtility.add_column(out_schema, "doc_id_internal", typeof(string));
               out_schema = BlueDataUtility.add_column(out_schema, "symbol", typeof(string));
               out_schema = BlueDataUtility.add_column(out_schema, "exchange", typeof(string));
               out_schema = BlueDataUtility.add_column(out_schema, "is_primary_exchange", typeof(bool));
@@ -44,11 +63,13 @@ namespace bts {
            }
            foreach (JsonNode jnode in json_objs) {
               if (jnode is JsonObject json_object) {
+                 string symbol = jnode["symbol"] == null ? String.Empty : jnode["symbol"].ToString();
+                 string exchange = jnode["exchange"] == null ? String.Empty : jnode["exchange"].ToString();
                  MutableBlueRow nr = new MutableBlueRow(_schema, r);
                  nr.setValue("id", _id++);
                  nr.setValue("doc_id_internal", $"{r["doc_id"]}_{jnode["symbol"]}");
-                 nr.setValue("symbol", jnode["symbol"]);
-                 nr.setValue("exchange", jnode["exchange"]);
+                 nr.setValue("symbol", symbol);
+                 nr.setValue("exchange", exchange);
                  nr.setValue("is_primary_exchange", Boolean.Parse(jnode["primary"].ToString()));
                  expanded_rows.Add(nr);
               }

src/DataStreamSuite/SharedStreams/Business/BenzingaNews/BenzingaNewsSymbolStateStream.cs

@@ -0,0 +1,33 @@
+namespace bts {
+   using BCG;
+
+   using System.Collections.Generic;
+   using System.Linq;
+   using System.Security.Cryptography;
+
+   //[DataStreamUserRespresentation("benzingaNewsSymbols")]
+   public class BenzingaNewsSymbolStateStream : ViewStream, INonIncGroupByEvaluator {
+      public BenzingaNewsSymbolStateStream(ViewStream news_str) {
+         ViewStream r = new NonIncGroupByStream(news_str, this);
+         mySetRealSource(r);
+      }
+      public BenzingaNewsSymbolStateStream(string connection_string, string db_name, string collection_name, string poll_period, string ttl) :
+      this(new BenzingaNewsStream(connection_string, db_name, collection_name, poll_period, ttl)) {
+      }
+
+      public IBlueRowCollection evaluate(IDictionary<int, List<IBlueRow>> grouped_rows) {
+         BlueTable rv = null;
+         foreach (var kv in grouped_rows) {
+            var symbol_rows = kv.Value;
+            IBlueRow state_row = BlueDataUtility.find_max(symbol_rows, "created_at");
+            rv = rv == null ? new BlueTable(state_row.schema) : rv;
+            rv.addRow(state_row);
+         }
+         return rv;
+      }
+
+      public string[] get_group_by_cols() {
+         return new string[] { "symbol" };
+      }
+   }
+}

src/DataStreamSuite/SharedStreams/Business/BenzingaNews/BenzingaNewsSymbolsStream.cs

@@ -0,0 +1,14 @@
+namespace bts {
+   using BCG;
+   [DataStreamUserRespresentation("benzingaNewsSymbols")]
+
+   public class BenzingaNewsSymbolsStream : ViewStream {
+      public BenzingaNewsSymbolsStream(ViewStream news_str) {
+         ViewStream rs = new DistinctRowStream(news_str, "symbol");
+         mySetRealSource(rs);
+      }
+      public BenzingaNewsSymbolsStream(string connection_string, string db_name, string collection_name, string poll_period, string ttl) :
+      this(new BenzingaNewsStream(connection_string, db_name, collection_name, poll_period, ttl)) {
+      }
+   }
+}

src/DataStreamSuite/SharedStreams/Business/MarketData/BookStream.cs

@@ -62,7 +62,7 @@ namespace BCG {
       }
 
       private void _on_books(TableUpdate<BookTuple, uint> update) {
-         _log_updates(update);
+         //_log_updates(update);
          bts.Log.Gen.info($"BookStream._on_books state: {update.state.Count()} changes: {update.changes.Count()} added: {update.added.Count()}");
          _cached_state = update.state;
       }

src/DataStreamSuite/SharedStreams/Config/SharedStreamConfig.cs

@@ -0,0 +1,48 @@
+namespace Bts.DataStreamPublisher;
+
+using Bts.PubSub;
+
+//using CommandLine;
+
+using System;
+using System.Collections.Generic;
+using System.IO;
+using System.Text.Json;
+using System.Text.Json.Serialization;
+
+public class SharedStreamConfig {
+
+   [JsonPropertyName("mongodb_url")]
+   public string MONGO_DB_URL { get; set; }
+
+   [JsonPropertyName("mongodb_db_name")]
+   public string MONGO_DB_NAME { get; set; }
+
+   [JsonPropertyName("mongodb_db_collection")]
+   public string MONGO_DB_COLLECTION { get; set; }
+
+   [JsonPropertyName("benzinga_news_ttl")]
+   public string BENZINGA_NEWS_TTL { get; set; }
+
+   [JsonPropertyName("pubsub")]
+   public PubSubConfig? PubSub { get; set; }
+
+
+   /// <summary>
+   /// Merges options with those defined in `ConfigFile` (does not overwrite
+   /// non-null existing values).
+   /// </summary>
+   /// <exception cref="InvalidOperationException">
+   /// JsonSerializer.Deserialize returns null
+   /// </exception>
+   public void MergeWithConfigFile() {
+      // if (ConfigFile is null) {
+      //    return;
+      // }
+      // var cfgString = File.ReadAllText(ConfigFile);
+      // var parsedCfg = JsonSerializer.Deserialize<DataStreamPublisherConfig>(cfgString) ??
+      //    throw new InvalidOperationException($"Failed to parse json from {ConfigFile}");
+      // MergeWith(parsedCfg);
+      // ConfigFile = null;
+   }
+}

src/DataStreamSuite/SharedStreams/System/PS3/Health/DeadPublisherTagsStream.cs

@@ -1,19 +1,14 @@
-using System;
-using System.Collections.Generic;
-using System.Text;
+using bts;
 
-namespace BCG
-{
+namespace BCG {
    /// <summary>
    /// List of all the tags that result from a
    /// dead publisher, ie tags that have data
    /// but no active publisher
    /// </summary>
-   [DataStreamUserRespresentation ("deadPublisherTags")]
-   public class DeadPublisherTagsStream : ViewStream
-   {
-      public DeadPublisherTagsStream()
-      {
+   [DataStreamUserRespresentation("deadPublisherTags")]
+   public class DeadPublisherTagsStream : ViewStream {
+      public DeadPublisherTagsStream() {
          ViewStream serverState = new PubSubServerStateStream();
          serverState = new FilterStream(serverState, "publishers='' and deadPublishers<>''");
          serverState = new ProjectStream(serverState, "!subscribers publishers deadPublishers");

src/DataStreamSuite/SharedStreams/System/PS3/PubsubRoutesStream.cs

@@ -0,0 +1,28 @@
+namespace BCG {
+   [DataStreamUserRespresentation("pss_routes")]
+   public class PubsubRoutesStream : ViewStream {
+      private ViewStreamDatabase _vsd;
+      private IPubSubRouter _router = new PubSubRouter();
+
+      public PubsubRoutesStream()
+      : this(ViewStreamDatabase.instance) { }
+
+      public PubsubRoutesStream(ViewStreamDatabase vsd) {
+         _vsd = vsd;
+      }
+
+      protected override void myOnFirstSubscriber() {
+         IBlueTableSchema schema = BlueDataUtility.form_schema("PubSubRoutes",
+                                                               Tuple.Create("id", TypeCode.Int32),
+                                                               Tuple.Create("tag_regexp", TypeCode.String),
+                                                               Tuple.Create("server_url", TypeCode.String));
+         BlueTable rows = new BlueTable(schema);
+         var routes = _router.sub_routes();
+         foreach (var route in routes) {
+            IBlueRow nr = new BlueRow(schema, rows.Count, route.tag_reg_exp, route.server_address);
+            rows.addRow(nr);
+         }
+         myFireEvent(new DataPacket(rows, rows, new BlueTable(schema)));
+      }
+   }
+}

src/DataStreamSuite/SharedStreams/System/PS3/Stats/PubSubServerTagStatsStream.cs

@@ -5,9 +5,7 @@ using BCG.Utilities.General;
 using MathNet.Numerics.RootFinding;
 
 namespace bts {
-
-
-   [DataStreamUserRespresentation("tagStats")]
+   [DataStreamUserRespresentation("pss_stats")]
    public class PubSubServerTagStatsStream : ViewStream, IWorker {
       private SubscriberCore _sub_core;
       private ViewStreamDatabase _vsd;

src/DataStreamSuite/SharedStreams/System/PS3/Subscription/AllTagsStream.cs

@@ -1,50 +1,50 @@
-using System;
-using System.Collections;
-using System.IO;
-
-using BCG.Utilities.General;
-using BCG.Utilities;
-using Bts.PubSub;
-using Spark.Core;
-
-namespace BCG {
-
-   public class AllTagsStream : ViewStream, IWorker {
-      private IWorker myRealWorkerStream;
-      public AllTagsStream()
-         : this("") { }
-
-      public AllTagsStream(string prefix)
-         : this(prefix, ViewStreamDatabase.instance) { }
-
-      public AllTagsStream(ViewStreamDatabase vsd, string regExp) {
-         ViewStream rs;
-         mySetRealSource(rs = new TagSynopsisStream(vsd, regExp, true));
-         myRealWorkerStream = rs as IWorker;
-      }
-
-      public AllTagsStream(string prefix, ViewStreamDatabase db)
-         : this(prefix, db, db.subscriber.get_router()) { }
-
-      public AllTagsStream(string prefix, ViewStreamDatabase db, IPubSubRouter router) {
-         ViewStream rs = null;
-         //if we're using substations, we don't care about the router
-         if (PubSubConfig.Instance.UseSubstations.assert_value()) {
-            mySetRealSource(rs = new TagSynopsisStream(db, "^" + prefix + ".*", true));
-         } else {
-            mySetRealSource(rs = new TagSynopsisStream(db, router.get_all_pubsub_sub_urls(), "^" + prefix + ".*", true));
-         }
-         myRealWorkerStream = rs as IWorker;
-      }
-
-      #region IWorker Members
-
-      public void doWork() {
-         if (myRealWorkerStream != null)
-            myRealWorkerStream.doWork();
-      }
-
-      #endregion
-   }
-
-}
+// using System;
+// using System.Collections;
+// using System.IO;
+
+// using BCG.Utilities.General;
+// using BCG.Utilities;
+// using Bts.PubSub;
+// using Spark.Core;
+
+// namespace BCG {
+
+//    public class AllTagsStream : ViewStream, IWorker {
+//       private IWorker myRealWorkerStream;
+//       public AllTagsStream()
+//          : this("") { }
+
+//       public AllTagsStream(string prefix)
+//          : this(prefix, ViewStreamDatabase.instance) { }
+
+//       public AllTagsStream(ViewStreamDatabase vsd, string regExp) {
+//          ViewStream rs;
+//          mySetRealSource(rs = new TagSynopsisStream(vsd, regExp, true));
+//          myRealWorkerStream = rs as IWorker;
+//       }
+
+//       public AllTagsStream(string prefix, ViewStreamDatabase db)
+//          : this(prefix, db, db.subscriber.get_router()) { }
+
+//       public AllTagsStream(string prefix, ViewStreamDatabase db, IPubSubRouter router) {
+//          ViewStream rs = null;
+//          //if we're using substations, we don't care about the router
+//          if (PubSubConfig.Instance.UseSubstations.assert_value()) {
+//             mySetRealSource(rs = new TagSynopsisStream(db, "^" + prefix + ".*", true));
+//          } else {
+//             mySetRealSource(rs = new TagSynopsisStream(db, router.get_all_pubsub_sub_urls(), "^" + prefix + ".*", true));
+//          }
+//          myRealWorkerStream = rs as IWorker;
+//       }
+
+//       #region IWorker Members
+
+//       public void doWork() {
+//          if (myRealWorkerStream != null)
+//             myRealWorkerStream.doWork();
+//       }
+
+//       #endregion
+//    }
+
+// }

src/DataStreamSuite/SharedStreams/System/PS3/Subscription/PubSubServerStateStream.cs

@@ -1,16 +1,12 @@
-using System;
-using System.Collections.Generic;
-using System.Text;
 using BCG.Utilities.General;
-using System.Collections;
-using System.IO;
-using MongoDB.Driver.Core.WireProtocol.Messages;
+using BCG;
 
-namespace BCG {
+namespace bts {
    /// <summary>
    /// Gets the subscription and publication state for one
    /// server.
    /// </summary>
+   [DataStreamUserRespresentation("pss_state")]
    public class PubSubServerStateStream : ViewStream, IWorker {
       private IDictionary<string, TagPubSubInfo> _tag_map = new Dictionary<string, TagPubSubInfo>();
       private readonly string _tag = "System_AllSubscriptions";

src/DataStreamSuite/SharedStreams/System/PS3/Subscription/TagSynopsisStream.cs

@@ -1,117 +1,117 @@
-using System;
-using System.Collections.Generic;
-using System.Text;
-using BCG.Utilities.General;
-using System.Collections;
-using System.IO;
-using Bts.PubSub;
-using bts;
-
-namespace BCG {
-   [DataStreamUserRespresentation("tagSynopsis")]
-   public class TagSynopsisStream : ViewStream, IStreamMapper, IWorker {
-      private List<IWorker> myWorkers = new List<IWorker>();
-      IBlueTableSchema mySchema = new BlueTableSchema("AllTags",
-                                             new ColumnInfo("id", 0, typeof(int)),
-                                             new ColumnInfo("tag", 1, typeof(string)),
-                                             new ColumnInfo("subscribers", 2, typeof(int)),
-                                             new ColumnInfo("pubSub", 3, typeof(string)),
-                                             new ColumnInfo("dataPresent", 4, typeof(bool)));
-
-      public TagSynopsisStream() {
-         mySetRealSource(new TagSynopsisStream(ViewStreamDatabase.instance, ViewStreamDatabase.instance.subscriber.get_router().get_all_pubsub_sub_urls()));
-      }
-
-      public TagSynopsisStream(string tagRegExp) {
-         mySetRealSource(new TagSynopsisStream(ViewStreamDatabase.instance, ViewStreamDatabase.instance.subscriber.get_router().get_all_pubsub_sub_urls(), tagRegExp));
-      }
-
-      public TagSynopsisStream(ViewStreamDatabase vsd, string route) {
-
-         ViewStream vs = new PubSubServerStateStream(/*route,*/ vsd);
-         vs = new MapStream(vs, this);
-         myWorkers.Add(vs as IWorker);
-
-         mySetRealSource(vs);
-      }
-
-
-      public TagSynopsisStream(bool cons_overloader, string route) :
-      this(ViewStreamDatabase.instance, new string[] { route }) {
-      }
-
-      public TagSynopsisStream(ViewStreamDatabase vsd, ICollection<string> routes) {
-         List<ViewStream> vss = new List<ViewStream>();
-         foreach (string r in routes) {
-            ViewStream vs = new PubSubServerStateStream(/*r,*/ vsd);
-            vs = new MapStream(vs, this);
-            vss.Add(vs);
-            myWorkers.Add(vs as IWorker);
-         }
-         mySetRealSource(new AppendStream(vss.ToArray()));
-      }
-
-      public TagSynopsisStream(ViewStreamDatabase vsd, string tagRegExp, bool filterOutNonPublished) {
-         // if (PubSub3Config.instance.PS3_UseSubstations)
-         // {
-         //    mySetRealSource(new ThreadSafeStream(new SubstationTagSynopsisStream(vsd, tagRegExp, filterOutNonPublished)));
-
-         // }
-         // else
-         {
-            PubSubRouter theRouter = new(Log.notnull(PubSubConfig.Instance.SubscriberRoutes).ToArray());
-            ICollection<string> thisVSDRoutes = theRouter.get_all_pubsub_sub_urls();
-
-            ViewStream vs = new FilterStream(new TagSynopsisStream(vsd, thisVSDRoutes), "tag", tagRegExp);
-
-            if (filterOutNonPublished)
-               vs = new FilterStream(vs, "dataPresent=true");
-            mySetRealSource(vs);
-         }
-
-      }
-
-      public TagSynopsisStream(ViewStreamDatabase vsd, ICollection<string> routes, string tagRegExp) {
-         mySetRealSource(new FilterStream(new TagSynopsisStream(vsd, routes), "tag", tagRegExp));
-      }
-
-      public TagSynopsisStream(ViewStreamDatabase vsd, ICollection<string> routes, string tagRegExp, bool filterOutNonPublished) {
-         ViewStream vs = new TagSynopsisStream(vsd, routes, tagRegExp);
-         if (filterOutNonPublished)
-            vs = new FilterStream(vs, "dataPresent=true");
-         mySetRealSource(vs);
-      }
-
-
-      #region IStreamMapper Members
-
-      public IBlueTableSchema getOutputSchema(IBlueTableSchema inputSchema) {
-         return mySchema;
-      }
-
-      #endregion
-
-      #region IDynamicStreamMapper Members
-
-      public IBlueRow map(IBlueRow row) {
-         string subValue = row["subscribers"].ToString().Trim();
-         return new BlueRow(getOutputSchema(null),
-                           row["id"],
-                           row["tag"],
-                           subValue.Length == 0 ? 0 : subValue.Split(',').Length,
-                           row["server"],
-                           row["publishers"].ToString().Length > 0 || row["deadPublishers"].ToString().Length > 0);
-      }
-
-      #endregion
-
-      #region IWorker Members
-
-      public void doWork() {
-         foreach (IWorker w in myWorkers)
-            w.doWork();
-      }
-
-      #endregion
-   }
-}
+// using System;
+// using System.Collections.Generic;
+// using System.Text;
+// using BCG.Utilities.General;
+// using System.Collections;
+// using System.IO;
+// using Bts.PubSub;
+// using bts;
+
+// namespace BCG {
+//    [DataStreamUserRespresentation("tagSynopsis")]
+//    public class TagSynopsisStream : ViewStream, IStreamMapper, IWorker {
+//       private List<IWorker> myWorkers = new List<IWorker>();
+//       IBlueTableSchema mySchema = new BlueTableSchema("AllTags",
+//                                              new ColumnInfo("id", 0, typeof(int)),
+//                                              new ColumnInfo("tag", 1, typeof(string)),
+//                                              new ColumnInfo("subscribers", 2, typeof(int)),
+//                                              new ColumnInfo("pubSub", 3, typeof(string)),
+//                                              new ColumnInfo("dataPresent", 4, typeof(bool)));
+
+//       public TagSynopsisStream() {
+//          mySetRealSource(new TagSynopsisStream(ViewStreamDatabase.instance, ViewStreamDatabase.instance.subscriber.get_router().get_all_pubsub_sub_urls()));
+//       }
+
+//       public TagSynopsisStream(string tagRegExp) {
+//          mySetRealSource(new TagSynopsisStream(ViewStreamDatabase.instance, ViewStreamDatabase.instance.subscriber.get_router().get_all_pubsub_sub_urls(), tagRegExp));
+//       }
+
+//       public TagSynopsisStream(ViewStreamDatabase vsd, string route) {
+
+//          ViewStream vs = new PubSubServerStateStream(/*route,*/ vsd);
+//          vs = new MapStream(vs, this);
+//          myWorkers.Add(vs as IWorker);
+
+//          mySetRealSource(vs);
+//       }
+
+
+//       public TagSynopsisStream(bool cons_overloader, string route) :
+//       this(ViewStreamDatabase.instance, new string[] { route }) {
+//       }
+
+//       public TagSynopsisStream(ViewStreamDatabase vsd, ICollection<string> routes) {
+//          List<ViewStream> vss = new List<ViewStream>();
+//          foreach (string r in routes) {
+//             ViewStream vs = new PubSubServerStateStream(/*r,*/ vsd);
+//             vs = new MapStream(vs, this);
+//             vss.Add(vs);
+//             myWorkers.Add(vs as IWorker);
+//          }
+//          mySetRealSource(new AppendStream(vss.ToArray()));
+//       }
+
+//       public TagSynopsisStream(ViewStreamDatabase vsd, string tagRegExp, bool filterOutNonPublished) {
+//          // if (PubSub3Config.instance.PS3_UseSubstations)
+//          // {
+//          //    mySetRealSource(new ThreadSafeStream(new SubstationTagSynopsisStream(vsd, tagRegExp, filterOutNonPublished)));
+
+//          // }
+//          // else
+//          {
+//             PubSubRouter theRouter = new(Log.notnull(PubSubConfig.Instance.SubscriberRoutes).ToArray());
+//             ICollection<string> thisVSDRoutes = theRouter.get_all_pubsub_sub_urls();
+
+//             ViewStream vs = new FilterStream(new TagSynopsisStream(vsd, thisVSDRoutes), "tag", tagRegExp);
+
+//             if (filterOutNonPublished)
+//                vs = new FilterStream(vs, "dataPresent=true");
+//             mySetRealSource(vs);
+//          }
+
+//       }
+
+//       public TagSynopsisStream(ViewStreamDatabase vsd, ICollection<string> routes, string tagRegExp) {
+//          mySetRealSource(new FilterStream(new TagSynopsisStream(vsd, routes), "tag", tagRegExp));
+//       }
+
+//       public TagSynopsisStream(ViewStreamDatabase vsd, ICollection<string> routes, string tagRegExp, bool filterOutNonPublished) {
+//          ViewStream vs = new TagSynopsisStream(vsd, routes, tagRegExp);
+//          if (filterOutNonPublished)
+//             vs = new FilterStream(vs, "dataPresent=true");
+//          mySetRealSource(vs);
+//       }
+
+
+//       #region IStreamMapper Members
+
+//       public IBlueTableSchema getOutputSchema(IBlueTableSchema inputSchema) {
+//          return mySchema;
+//       }
+
+//       #endregion
+
+//       #region IDynamicStreamMapper Members
+
+//       public IBlueRow map(IBlueRow row) {
+//          string subValue = row["subscribers"].ToString().Trim();
+//          return new BlueRow(getOutputSchema(null),
+//                            row["id"],
+//                            row["tag"],
+//                            subValue.Length == 0 ? 0 : subValue.Split(',').Length,
+//                            row["server"],
+//                            row["publishers"].ToString().Length > 0 || row["deadPublishers"].ToString().Length > 0);
+//       }
+
+//       #endregion
+
+//       #region IWorker Members
+
+//       public void doWork() {
+//          foreach (IWorker w in myWorkers)
+//             w.doWork();
+//       }
+
+//       #endregion
+//    }
+// }

src/DataStreamSuite/SharedStreams/System/PS3/TableMetaDataStream.cs

@@ -0,0 +1,114 @@
+namespace bts
+{
+   using BCG;
+   using BCG.Utilities.General;
+
+   public class TableMetaDataStream : ViewStream, IWorker
+   {
+      private class TableState
+      {
+         public ViewStream table_stream;
+         public IBlueTableSchema schema = null;
+         public bool processed = false;
+         public TableState(ViewStream table_stream_)
+         {
+            table_stream = table_stream_;
+            table_stream.onChange += _handle_table_update;
+         }
+         private void _handle_table_update(object dp_obj)
+         {
+            DataPacket dp = dp_obj as DataPacket;
+            schema = dp.schema;
+         }
+         public void unsub()
+         {
+            table_stream.onChange -= _handle_table_update;
+         }
+      }
+      private IBlueTableSchema _schema = BlueDataUtility.form_schema("TableMetaData",
+         Tuple.Create("id", TypeCode.Int32),
+         Tuple.Create("table_name", TypeCode.String),
+         Tuple.Create("col_idx", TypeCode.Int32),
+         Tuple.Create("col_type_code", TypeCode.Int32),
+         Tuple.Create("col_name", TypeCode.String));
+
+      private ViewStreamDatabase _vsd;
+      private ViewStream _base_stream;
+      private IDictionary<string, TableState> _table_map = new Dictionary<string, TableState>();
+      private BlueTable _state;
+      public TableMetaDataStream()
+      : this(ViewStreamDatabase.instance) { }
+      public TableMetaDataStream(ViewStreamDatabase vsd)
+      {
+         _vsd = vsd;
+         _base_stream = new PubSubServerStateStream(_vsd);
+         _state = new BlueTable(_schema);
+      }
+      protected override void myOnFirstSubscriber()
+      {
+         _base_stream.onChange += _handle_base_stream;
+         _vsd.addWorker(this);
+      }
+
+      protected override void myOnLastSubscriber()
+      {
+         _base_stream.onChange -= _handle_base_stream;
+         _vsd.removeWorker(this);
+      }
+
+      public void doWork()
+      {
+         bool changed = false;
+         foreach (TableState ts in _table_map.Values)
+         {
+            if (!ts.processed && ts.schema != null)
+            {
+               foreach (ColumnInfo ci in ts.schema.columnInfo)
+               {
+                  int id = _state.Count;
+                  var table_name = ts.schema.tableName;
+                  var col_idx = ci.colNo;
+                  var col_name = ci.colName;
+                  TypeCode type_code = BlueDataUtility.get_typecode(ci.colType);
+                  BlueRow nr = new BlueRow(_schema, id, table_name, col_idx, Convert.ToInt32(type_code),
+                                          col_name);
+                  changed = true;
+                  _state.addRow(nr);
+               }
+               ts.processed = true;
+               ts.unsub();
+            }
+         }
+         if (changed)
+         {
+            var dp_out = BlueDataUtility.build_dp_inc(_state, myLastValue as DataPacket);
+            myFireEvent(dp_out);
+         }
+      }
+
+      private void _handle_base_stream(object dp_obj)
+      {
+         DataPacket dp = dp_obj as DataPacket;
+         foreach (IBlueRow r in dp.newRows)
+         {
+            string tag = r.get<string>("tag");
+            if (tag.StartsWith("DS_"))
+            {
+               _handle_table(tag);
+            }
+         }
+      }
+
+      private void _handle_table(string tag_name)
+      {
+         Log.assert(tag_name.StartsWith("DS_"));
+         string table_name = tag_name.Remove(0, 3);
+         TableState ts = null;
+         if (!_table_map.TryGetValue(table_name, out ts))
+         {
+            ts = new TableState(new TableStream(table_name, _vsd));
+            _table_map.Add(table_name, ts);
+         }
+      }
+   }
+}

src/Spark.Admin.WebApp/src/Accounts/Types.fs

@@ -17,7 +17,7 @@ type Tab =
   | PositionManagementTab
   | MarginTab
 
-let venues = [| Venue.BINANCE; Venue.BinanceUs; Venue.BinanceUsdsFutures; Venue.COINBASE; Venue.DERIBIT; Venue.GLBX; Venue.ICE; Venue.SIM; |]
+let venues = [| Venue.BINANCE; Venue.BinanceUs; Venue.BinanceUsdsFutures; Venue.CBOE; Venue.COINBASE; Venue.DERIBIT; Venue.GLBX; Venue.ICE; Venue.TAIFEX; Venue.SIM; |]
 
 let securityTypes = [| SecurityType.SPOT; SecurityType.FUTURE; SecurityType.OPTION; SecurityType.SPREAD; |]
 
@@ -69,6 +69,11 @@ type ICEExchangeField =
   | AccountCode of string
   | MiscTagMapping of JSMap<string>
 
+type TaifexExchangeField =
+  | CTI of CTICode
+  | Origin of CustomerOrFirm
+  | ClearingAccount of string
+
 type SIMExchangeField =
   | CTI of CTICode
   | Origin of CustomerOrFirm

src/Spark.Admin.WebApp/src/Accounts/View.fs

@@ -797,9 +797,9 @@ let venueToString (venue:bts.api.Venue) =
     | bts.api.Venue.BINANCE -> "BINANCE"
     | bts.api.Venue.BinanceUs -> "BinanceUs"
     | bts.api.Venue.BinanceUsdsFutures -> "BinanceUsdsFutures"
-    | bts.api.Venue.COINBASE -> "COINBASE"
     | bts.api.Venue.FTX -> "FTX"
     | bts.api.Venue.OKX -> "OKX"
+    | bts.api.Venue.TAIFEX -> "TAIFEX"
 
 let exchange_display expirationNameList =
   let s = sprintf "%O" expirationNameList.product.exchange

src/client/spark/AW2/AnalyticsParameters/AnalyticsExpirationParametersSettings.cs

@@ -1,4 +1,6 @@
 namespace mbts.spark {
+   using bts.api;
+
    using ProtoBuf;
 
    [ProtoContract]
@@ -10,6 +12,12 @@ namespace mbts.spark {
       [ProtoMember(10)]
       public string[] selected_symbols {get;set;}
 
+      [ProtoMember(11)]
+      public Venue[] selected_venues {get;set;}
+
+      [ProtoMember(12)]
+      public SecurityType[] selected_types {get;set;}
+
       [ProtoMember(20)]
       public bool auto_copy_rates_is_checked {get;set;}
 

src/client/spark/AW2/AnalyticsParameters/AnalyticsExpirationParametersVM.cs

@@ -44,6 +44,8 @@ namespace mbts.spark {
       readonly DeferrableAction _raise_selected_rows_changed;
       readonly IRaiseCanExecuteChangedCommand _set_driver_to_nearest_exp_cmd;
       IList<string>? _pending_selected_symbols;
+      IList<Venue>? _pending_selected_venues;
+      IList<SecurityType>? _pending_selected_types;
 
       AnalyticsExpirationParametersMap? _parameter_map;
       (ICommand cmd, IDisposable can_exe_sub)? _smart_copy_roll_target_to_selection_cmd;
@@ -56,9 +58,13 @@ namespace mbts.spark {
          _auto_copy_rolls_is_checked = backing(nameof(auto_copy_rolls_is_checked), false);
          _raise_selected_rows_changed = new(() => selected_rows_changed?.Invoke(_selected_rows));
          var selected_symbol_ref_count = new RefCountMap<string>();
+         var selected_venue_ref_count = new RefCountMap<Venue>();
+         var selected_type_ref_count = new RefCountMap<SecurityType>();
          _sync_visible_rows = new(() =>
             visible_rows.sync_with(
-               rows.Where(x => selected_symbol_ref_count.ContainsKey(x.security.symbol))));
+               rows.Where(x => selected_symbol_ref_count.ContainsKey(x.security.symbol)
+                  && selected_venue_ref_count.ContainsKey(x.security.exchange)
+                  && selected_type_ref_count.ContainsKey(x.security.security_type))));
          non_options_with_empty = new ObservableAggregateCollection<SecurityExpiration>(
             new() { SecurityExpiration.empty }, non_options);
 
@@ -67,6 +73,8 @@ namespace mbts.spark {
 
          if (startup_data is not null) {
             _pending_selected_symbols = startup_data.selected_symbols;
+            _pending_selected_venues = startup_data.selected_venues;
+            _pending_selected_types = startup_data.selected_types;
             auto_copy_rolls_is_checked = startup_data.auto_copy_rolls_is_checked;
             auto_copy_rates_is_checked = startup_data.auto_copy_rates_is_checked;
          }
@@ -103,6 +111,26 @@ namespace mbts.spark {
                   }
                });
             }),
+            ObservableAutoWrapper.connect_item_hooks(selected_venues, sym => {
+               if (selected_venue_ref_count.increment_adds_key(sym)) {
+                  _sync_visible_rows.Invoke();
+               }
+               return Disposable.Create(() => {
+                  if (selected_venue_ref_count.decrement_removes_key(sym)) {
+                     _sync_visible_rows.Invoke();
+                  }
+               });
+            }),
+            ObservableAutoWrapper.connect_item_hooks(selected_types, sym => {
+               if (selected_type_ref_count.increment_adds_key(sym)) {
+                  _sync_visible_rows.Invoke();
+               }
+               return Disposable.Create(() => {
+                  if (selected_type_ref_count.decrement_removes_key(sym)) {
+                     _sync_visible_rows.Invoke();
+                  }
+               });
+            }),
             ObservableAutoWrapper.connect_item_hooks(
                rows, r => r.observe_is_dirty().Subscribe(_ => _is_dirty.refresh())),
             _is_dirty.observe_current(true).OnDispatcher().Subscribe(can_revert_observer),
@@ -128,6 +156,10 @@ namespace mbts.spark {
 
       public ObservableCollection<string> symbols { get; } = new();
       public ObservableCollection<string> selected_symbols { get; } = new();
+      public ObservableCollection<Venue> venues { get; } = new();
+      public ObservableCollection<Venue> selected_venues { get; } = new();
+      public ObservableCollection<SecurityType> types { get; } = new() { SecurityType.Future, SecurityType.Option };
+      public ObservableCollection<SecurityType> selected_types { get; } = new();
 
       public bool is_dirty => _is_dirty.Value;
 
@@ -191,6 +223,8 @@ namespace mbts.spark {
       public AnalyticsExpirationParametersSettings get_state() =>
          new() {
             selected_symbols = selected_symbols.Cast<string>().ToArray(),
+            selected_venues = selected_venues.ToArray(),
+            selected_types = selected_types.ToArray(),
             auto_copy_rates_is_checked = auto_copy_rates_is_checked,
             auto_copy_rolls_is_checked = auto_copy_rolls_is_checked
          };
@@ -206,7 +240,16 @@ namespace mbts.spark {
                .Where(d => d.HasSecurity && d.Security.Symbol == r.security.symbol)
                .Where(d => is_unexpired(d))
                .TryGetMinElement(
-                  d => (r.security.expiration - d.Security.Expiration.date_time()).Ticks.abs(),
+                  d => {
+                     var expDiff = (r.security.expiration - d.Security.Expiration.date_time()).Ticks.abs();
+                     if (r.security.exchange != d.Security.Exchange.to_venue()) {
+                        // This is a hack to favor the security from the same venue as the row's
+                        // security in the event of a tie (for the nearest exp). As long as exp
+                        // dates are separated by more than 1 tick (100ns), the hack will work.
+                        expDiff += 1;
+                     }
+                     return expDiff;
+                  },
                   out var nearest_exp)) {
                r.driver = nearest_exp;
             }
@@ -294,10 +337,16 @@ namespace mbts.spark {
          }
 
          symbols.sync_with(_parameters.Select(p => p.security.symbol).Distinct().Ordered());
+         venues.sync_with(_parameters.Select(p => p.security.exchange).Distinct().OrderBy(v => v.ToString()));
+         types.sync_with(_parameters.Select(p => p.security.security_type).Distinct().OrderBy(v => v.ToString()));
          using var _ = _sync_visible_rows.defer();
          if (is_first_update) {
             selected_symbols.sync_with(_pending_selected_symbols?.Intersect(symbols) ?? symbols);
             _pending_selected_symbols = null;
+            selected_venues.sync_with(_pending_selected_venues?.Intersect(venues) ?? venues);
+            _pending_selected_venues = null;
+            selected_types.sync_with(_pending_selected_types?.Intersect(types) ?? types);
+            _pending_selected_types = null;
          }
       }