Cleaning market data streams

.dotnet-cli/.nuget/packages/microsoft.net.test.sdk/17.10.0/build/netcoreapp3.1/Microsoft.NET.Test.Sdk.Program.cs

@@ -0,0 +1,4 @@
+// <auto-generated> This file has been auto generated. </auto-generated>
+using System;
+[Microsoft.VisualStudio.TestPlatform.TestSDKAutoGeneratedCode]
+class AutoGeneratedProgram {static void Main(string[] args){}}

.gitignore

@@ -64,3 +64,9 @@ trace_data/
 *.lst
 .env.local
 .secrets
+.dotnet-cli/
+.dotnet-cli/
+.dotnet-cli /
+.dotnet/
+.codex_restore*.log
+src/cs/Bts.Spark.Core/.codexobj/

.vscode/launch.json

@@ -203,9 +203,71 @@
         "program": "PubSubServer.exe",
       },
       "cwd": "build/Debug/bin/net8.0",
-      "env": {
-        "DOTNET_ENVIRONMENT": "${input:pubSubServerEnvironment}",
+      // "env": {
+      //   "DOTNET_ENVIRONMENT": "${input:pubSubServerEnvironment}",
+      // },
+      "stopAtEntry": false,
+      "console": "integratedTerminal"
+    },
+    {
+      "name": "DSPublisher",
+      "type": "coreclr",
+      "request": "launch",
+      "preLaunchTask": "Build DataStream (Debug)",
+      "program": "",
+      "linux": {
+        "program": "DSPublisher",
       },
+      "windows": {
+        "program": "DSPublisher.exe",
+      },
+      "cwd": "build/Debug/bin/net8.0",
+      "stopAtEntry": false,
+      "console": "integratedTerminal"
+    },
+    {
+      "name": "DSPublisher (Skip Build)",
+      "type": "coreclr",
+      "request": "launch",
+      "program": "",
+      "linux": {
+        "program": "DSPublisher",
+      },
+      "windows": {
+        "program": "DSPublisher.exe",
+      },
+      "cwd": "build/Debug/bin/net8.0",
+      "stopAtEntry": false,
+      "console": "integratedTerminal"
+    },
+    {
+      "name": "PersistenceServer",
+      "type": "coreclr",
+      "request": "launch",
+      "preLaunchTask": "Build DataStream (Debug)",
+      "program": "",
+      "linux": {
+        "program": "PersistenceServer",
+      },
+      "windows": {
+        "program": "PersistenceServer.exe",
+      },
+      "cwd": "build/Debug/bin/net8.0",
+      "stopAtEntry": false,
+      "console": "integratedTerminal"
+    },
+    {
+      "name": "PersistenceServer (Skip Build)",
+      "type": "coreclr",
+      "request": "launch",
+      "program": "",
+      "linux": {
+        "program": "PersistenceServer",
+      },
+      "windows": {
+        "program": "PersistenceServer.exe",
+      },
+      "cwd": "build/Debug/bin/net8.0",
       "stopAtEntry": false,
       "console": "integratedTerminal"
     },

src/DataStreamSuite/.vscode/settings.json

@@ -0,0 +1,3 @@
+{
+   "dotnet.preferCSharpExtension": true
+}

src/DataStreamSuite/DSPublisher/Configurations/.gitkeep

@@ -0,0 +1 @@
+

src/DataStreamSuite/DSPublisher/DSLWSClient/dsl_websocket_client.py

@@ -0,0 +1,168 @@
+"""
+Simple DSPublisher DSL websocket client.
+
+Example:
+   python dsl_websocket_client.py --dsl "HeikinAshiStream(...)"
+   python dsl_websocket_client.py --dsl-file query.dsl
+"""
+
+import argparse
+import asyncio
+import json
+import os
+import subprocess
+import sys
+from pathlib import Path
+
+try:
+   import websockets
+except ImportError:
+   print("Installing missing dependency: websockets")
+   subprocess.check_call([sys.executable, "-m", "pip", "install", "websockets"])
+   import websockets
+
+
+DEFAULT_DSL = 'RSIUIStream("ES", "TIME", 5, 7)'
+DEFAULT_URL = "ws://localhost:19093/ws/"
+
+
+def load_dsl(dsl: str | None, dsl_file: str | None) -> str:
+   if dsl and dsl_file:
+      raise ValueError("Pass either --dsl or --dsl-file, not both.")
+
+   if dsl_file:
+      return Path(dsl_file).read_text(encoding="utf-8").strip()
+
+   if dsl:
+      return dsl.strip()
+
+   if DEFAULT_DSL.startswith("#"):
+      raise ValueError("No DSL provided. Fill in DEFAULT_DSL, or pass --dsl / --dsl-file.")
+
+   return DEFAULT_DSL
+
+
+def load_config_path(config_path: str | None) -> Path | None:
+   if config_path:
+      return Path(config_path)
+
+   environment = os.environ.get("DOTNET_ENVIRONMENT")
+   config_file_name = "appsettings.json" if not environment else f"appsettings.{environment}.json"
+   script_dir = Path(__file__).resolve().parent
+   candidate_dirs = [
+      script_dir,
+      script_dir.parent,
+      Path.cwd(),
+   ]
+
+   for candidate_dir in candidate_dirs:
+      candidate_path = candidate_dir / config_file_name
+      if candidate_path.exists():
+         return candidate_path
+
+   return None
+
+
+def load_url(url: str | None, config_path: str | None) -> str:
+   if url:
+      return url
+
+   resolved_config_path = load_config_path(config_path)
+   if not resolved_config_path:
+      print(f"Config file not found; using default url: {DEFAULT_URL}")
+      return DEFAULT_URL
+
+   config = json.loads(resolved_config_path.read_text(encoding="utf-8"))
+   dsl_api_server_config = config.get("DSLAPIServer", {})
+   ip = dsl_api_server_config.get("Ip", "localhost")
+   port = int(dsl_api_server_config.get("Port", 19093))
+   ws_url = f"ws://{ip}:{port}/ws/"
+   print(f"Loaded DSPublisher DSL websocket url from {resolved_config_path}: {ws_url}")
+   return ws_url
+
+
+async def send_subscription(websocket, dsl: str) -> None:
+   payload = {
+      "type": "subscribe",
+      "dsl": dsl,
+   }
+   await websocket.send(json.dumps(payload))
+   print("Sent subscribe request:")
+   print(json.dumps(payload, indent=2))
+
+
+async def send_unsubscribe(websocket, subscription_id: str) -> None:
+   payload = {
+      "type": "unsubscribe",
+      "subscriptionId": subscription_id,
+   }
+   await websocket.send(json.dumps(payload))
+   print("Sent unsubscribe request:")
+   print(json.dumps(payload, indent=2))
+
+
+def print_server_message(raw_message: str) -> str | None:
+   try:
+      message = json.loads(raw_message)
+   except json.JSONDecodeError:
+      print(raw_message)
+      return None
+
+   print("Received server message:")
+   print(json.dumps(message, indent=2))
+   return message.get("subscriptionId")
+
+
+async def run_client(url: str, dsl: str) -> None:
+   subscription_id: str | None = None
+
+   async with websockets.connect(url) as websocket:
+      print(f"Connected to DSPublisher DSL websocket: {url}")
+      await send_subscription(websocket, dsl)
+
+      try:
+         async for raw_message in websocket:
+            latest_subscription_id = print_server_message(raw_message)
+            subscription_id = subscription_id or latest_subscription_id
+      finally:
+         if subscription_id:
+            await send_unsubscribe(websocket, subscription_id)
+
+
+def main() -> int:
+   parser = argparse.ArgumentParser(
+      description="Subscribe to a DSPublisher DSL websocket query and print responses."
+   )
+   parser.add_argument(
+      "--url",
+      help="DSPublisher DSL websocket URL. Overrides appsettings DSLAPIServer config.",
+   )
+   parser.add_argument(
+      "--config",
+      help="Path to DSPublisher appsettings JSON. Defaults to appsettings[.$DOTNET_ENVIRONMENT].json near this script.",
+   )
+   parser.add_argument(
+      "--dsl",
+      help="DSL expression to subscribe to.",
+   )
+   parser.add_argument(
+      "--dsl-file",
+      help="Path to a text file containing the DSL expression.",
+   )
+   args = parser.parse_args()
+
+   try:
+      dsl = load_dsl(args.dsl, args.dsl_file)
+      url = load_url(args.url, args.config)
+      asyncio.run(run_client(url, dsl))
+      return 0
+   except KeyboardInterrupt:
+      print("stopped")
+      return 0
+   except Exception as ex:
+      print(f"error: {ex}")
+      return 1
+
+
+if __name__ == "__main__":
+   raise SystemExit(main())

src/DataStreamSuite/DSPublisher/DSLWSServer/DSLAPIWSListener.cs

@@ -0,0 +1,264 @@
+namespace DSPublisher.DSLWSServer;
+
+using System;
+using System.Collections.Concurrent;
+using System.Collections.Generic;
+using System.Net;
+using System.Net.WebSockets;
+using System.Text;
+using System.Text.Json;
+using System.Threading;
+using System.Threading.Tasks;
+
+using BCG;
+using Bts.DataStreamCore;
+
+/// <summary>
+/// Websocket-facing API boundary for DSL clients. It opens the listener port,
+/// accepts subscribe/unsubscribe messages, and sends DataPacket JSON back to clients.
+/// Instantiated by DSPAPIServer.
+/// </summary>
+public sealed class DSLAPIWSListener : IDisposable {
+   readonly HttpListener listener = new();
+   readonly IDSLBookkeeper bookkeeper;
+   readonly string prefix;
+   readonly ConcurrentDictionary<string, ClientConnection> clients_by_id = new();
+   CancellationTokenSource? cancellation_token_source;
+   Task? listener_task;
+   bool disposed;
+
+   sealed class ClientConnection {
+      public required string client_id { get; init; }
+      public required WebSocket web_socket { get; init; }
+      public HashSet<Guid> subscription_ids { get; } = new();
+      public SemaphoreSlim send_lock { get; } = new(1, 1);
+      public DataPacketJsonSerializer serializer { get; } = new();
+   }
+
+   public DSLAPIWSListener(string prefix, IDSLBookkeeper bookkeeper) {
+      ArgumentException.ThrowIfNullOrWhiteSpace(prefix);
+      ArgumentNullException.ThrowIfNull(bookkeeper);
+      this.prefix = prefix;
+      this.bookkeeper = bookkeeper;
+   }
+
+   public Task StartAsync(CancellationToken cancellation_token = default) {
+      _throw_if_disposed();
+      if (listener_task is not null) {
+         return Task.CompletedTask;
+      }
+
+      cancellation_token_source = CancellationTokenSource.CreateLinkedTokenSource(cancellation_token);
+      listener.Prefixes.Add(prefix);
+      listener.Start();
+      listener_task = Task.Run(() => _listen_loop(cancellation_token_source.Token), CancellationToken.None);
+      return Task.CompletedTask;
+   }
+
+   public async Task StopAsync() {
+      if (cancellation_token_source is null) {
+         return;
+      }
+
+      cancellation_token_source.Cancel();
+      if (listener.IsListening) {
+         listener.Stop();
+      }
+
+      foreach (var client_connection in clients_by_id.Values) {
+         await _cleanup_client(client_connection);
+      }
+
+      if (listener_task is not null) {
+         await listener_task;
+         listener_task = null;
+      }
+
+      cancellation_token_source.Dispose();
+      cancellation_token_source = null;
+   }
+
+   public void Dispose() {
+      if (disposed) {
+         return;
+      }
+
+      StopAsync().GetAwaiter().GetResult();
+      listener.Close();
+      disposed = true;
+   }
+
+   async Task _listen_loop(CancellationToken cancellation_token) {
+      while (!cancellation_token.IsCancellationRequested) {
+         HttpListenerContext context;
+         try {
+            context = await listener.GetContextAsync();
+         } catch (HttpListenerException) when (cancellation_token.IsCancellationRequested || !listener.IsListening) {
+            break;
+         } catch (ObjectDisposedException) {
+            break;
+         }
+
+         _ = Task.Run(() => _handle_request(context, cancellation_token), cancellation_token);
+      }
+   }
+
+   async Task _handle_request(HttpListenerContext context, CancellationToken cancellation_token) {
+      if (!context.Request.IsWebSocketRequest) {
+         context.Response.StatusCode = (int)HttpStatusCode.BadRequest;
+         context.Response.Close();
+         return;
+      }
+
+      HttpListenerWebSocketContext web_socket_context = await context.AcceptWebSocketAsync(null);
+      var client_connection = new ClientConnection {
+         client_id = Guid.NewGuid().ToString("N"),
+         web_socket = web_socket_context.WebSocket,
+      };
+
+      clients_by_id[client_connection.client_id] = client_connection;
+      try {
+         await _read_messages(client_connection, cancellation_token);
+      } finally {
+         await _cleanup_client(client_connection);
+      }
+   }
+
+   async Task _read_messages(ClientConnection client_connection, CancellationToken cancellation_token) {
+      var buffer = new byte[64 * 1024];
+      while (client_connection.web_socket.State == WebSocketState.Open && !cancellation_token.IsCancellationRequested) {
+         var result = await client_connection.web_socket.ReceiveAsync(buffer, cancellation_token);
+         if (result.MessageType == WebSocketMessageType.Close) {
+            await client_connection.web_socket.CloseAsync(WebSocketCloseStatus.NormalClosure, "ok", cancellation_token);
+            break;
+         }
+
+         var json = Encoding.UTF8.GetString(buffer, 0, result.Count);
+         await _handle_message(client_connection, json, cancellation_token);
+      }
+   }
+
+   async Task _handle_message(ClientConnection client_connection, string json, CancellationToken cancellation_token) {
+      DSLWebSocketRequest? request;
+      try {
+         request = JsonSerializer.Deserialize<DSLWebSocketRequest>(json);
+      } catch (JsonException ex) {
+         await _send_error(client_connection, $"Invalid request JSON: {ex.Message}", cancellation_token);
+         return;
+      }
+
+      if (request?.type is null) {
+         await _send_error(client_connection, "Request type is required.", cancellation_token);
+         return;
+      }
+
+      if (string.Equals(request.type, "subscribe", StringComparison.OrdinalIgnoreCase)) {
+         await _handle_subscribe(client_connection, request, cancellation_token);
+         return;
+      }
+
+      if (string.Equals(request.type, "unsubscribe", StringComparison.OrdinalIgnoreCase)) {
+         await _handle_unsubscribe(client_connection, request, cancellation_token);
+         return;
+      }
+
+      await _send_error(client_connection, $"Unknown request type: {request.type}", cancellation_token);
+   }
+
+   async Task _handle_subscribe(ClientConnection client_connection, DSLWebSocketRequest request, CancellationToken cancellation_token) {
+      if (string.IsNullOrWhiteSpace(request.dsl)) {
+         await _send_error(client_connection, "Subscribe request requires dsl.", cancellation_token);
+         return;
+      }
+
+      Guid subscription_id = Guid.Empty;
+      subscription_id = bookkeeper.Subscribe(request.dsl, packet => {
+         if (subscription_id == Guid.Empty) {
+            return;
+         }
+         _ = _send_packet(client_connection, subscription_id, packet, CancellationToken.None);
+      }, client_connection.client_id);
+
+      client_connection.subscription_ids.Add(subscription_id);
+      await _send_response(client_connection, new DSLWebSocketResponse {
+         type = "subscribed",
+         subscription_id = subscription_id,
+      }, cancellation_token);
+   }
+
+   async Task _handle_unsubscribe(ClientConnection client_connection, DSLWebSocketRequest request, CancellationToken cancellation_token) {
+      if (request.subscription_id is null) {
+         await _send_error(client_connection, "Unsubscribe request requires subscriptionId.", cancellation_token);
+         return;
+      }
+
+      var subscription_id = request.subscription_id.Value;
+      if (!client_connection.subscription_ids.Remove(subscription_id)) {
+         await _send_error(client_connection, $"Subscription does not belong to this client: {subscription_id}", cancellation_token);
+         return;
+      }
+
+      bookkeeper.Unsubscribe(subscription_id);
+      await _send_response(client_connection, new DSLWebSocketResponse {
+         type = "unsubscribed",
+         subscription_id = subscription_id,
+      }, cancellation_token);
+   }
+
+   async Task _send_packet(ClientConnection client_connection, Guid subscription_id, DataPacket packet, CancellationToken cancellation_token) {
+      var packet_json = client_connection.serializer.Serialize(packet);
+      using var packet_document = JsonDocument.Parse(packet_json ?? "{}");
+      await _send_response(client_connection, new DSLWebSocketResponse {
+         type = "data",
+         subscription_id = subscription_id,
+         packet = packet_document.RootElement.Clone(),
+      }, cancellation_token);
+   }
+
+   Task _send_error(ClientConnection client_connection, string message, CancellationToken cancellation_token) =>
+      _send_response(client_connection, new DSLWebSocketResponse {
+         type = "error",
+         message = message,
+      }, cancellation_token);
+
+   async Task _send_response(ClientConnection client_connection, DSLWebSocketResponse response, CancellationToken cancellation_token) {
+      if (client_connection.web_socket.State != WebSocketState.Open) {
+         bookkeeper.DisposeClient(client_connection.client_id);
+         return;
+      }
+
+      var json = JsonSerializer.Serialize(response);
+      var bytes = Encoding.UTF8.GetBytes(json);
+      await client_connection.send_lock.WaitAsync(cancellation_token);
+      try {
+         await client_connection.web_socket.SendAsync(bytes, WebSocketMessageType.Text, true, cancellation_token);
+      } catch {
+         bookkeeper.DisposeClient(client_connection.client_id);
+         throw;
+      } finally {
+         client_connection.send_lock.Release();
+      }
+   }
+
+   async Task _cleanup_client(ClientConnection client_connection) {
+      if (clients_by_id.TryRemove(client_connection.client_id, out _)) {
+         bookkeeper.DisposeClient(client_connection.client_id);
+      }
+
+      if (client_connection.web_socket.State == WebSocketState.Open || client_connection.web_socket.State == WebSocketState.CloseReceived) {
+         try {
+            await client_connection.web_socket.CloseAsync(WebSocketCloseStatus.NormalClosure, "cleanup", CancellationToken.None);
+         } catch {
+         }
+      }
+
+      client_connection.send_lock.Dispose();
+      client_connection.web_socket.Dispose();
+   }
+
+   void _throw_if_disposed() {
+      if (disposed) {
+         throw new ObjectDisposedException(nameof(DSLAPIWSListener));
+      }
+   }
+}

src/DataStreamSuite/DSPublisher/DSLWSServer/DSLBookkeeper.cs

@@ -0,0 +1,216 @@
+namespace DSPublisher.DSLWSServer;
+
+using System;
+using System.Collections.Generic;
+
+using BCG;
+using Bts.DataStreamCore;
+using bts.BlueOrb;
+using bts;
+
+/// <summary>
+/// Tracks active DSL subscriptions and shared DSLStream instances. Multiple clients
+/// subscribing to the same DSL reuse one stream, and DoWork dispatches queued packets.
+/// Instantiated by DSPAPIServer.
+/// </summary>
+public sealed class DSLBookkeeper : IDSLBookkeeper {
+   sealed class QueryState {
+      public required string dsl { get; init; }
+      public required DSLStream stream { get; init; }
+      public DataStream.ChangeDelegate? stream_handler { get; set; }
+      public HashSet<Guid> subscription_ids { get; } = new();
+      public Queue<DataPacket> pending_packets { get; } = new();
+   }
+
+   sealed class SubscriptionState {
+      public required Guid subscription_id { get; init; }
+      public required string client_id { get; init; }
+      public required string dsl_key { get; init; }
+      public required Action<DataPacket> on_data { get; init; }
+   }
+
+   readonly object lock_obj = new();
+   readonly Dictionary<string, QueryState> queries_by_dsl = new(StringComparer.Ordinal);
+   readonly Dictionary<Guid, SubscriptionState> subscriptions_by_id = new();
+   readonly Dictionary<string, HashSet<Guid>> subscriptions_by_client = new(StringComparer.Ordinal);
+   bool disposed;
+
+   public Guid Subscribe(string dsl, Action<DataPacket> on_data, string client_id) {
+      ArgumentException.ThrowIfNullOrWhiteSpace(dsl);
+      ArgumentNullException.ThrowIfNull(on_data);
+      ArgumentException.ThrowIfNullOrWhiteSpace(client_id);
+
+      lock (lock_obj) {
+         _throw_if_disposed();
+
+         var dsl_key = _normalize_dsl(dsl);
+         if (!queries_by_dsl.TryGetValue(dsl_key, out var query_state)) {
+            _log_lifecycle($"DSLBookkeeper.Subscribe client_id: {client_id} creating new DSLStream dsl: {dsl_key}");
+            query_state = _create_query_state(dsl_key);
+            queries_by_dsl.Add(dsl_key, query_state);
+         } else {
+            _log_lifecycle($"DSLBookkeeper.Subscribe client_id: {client_id} reusing existing DSLStream subscription_count: {query_state.subscription_ids.Count} dsl: {dsl_key}");
+         }
+
+         var subscription_id = Guid.NewGuid();
+         var subscription_state = new SubscriptionState {
+            subscription_id = subscription_id,
+            client_id = client_id,
+            dsl_key = dsl_key,
+            on_data = on_data,
+         };
+
+         subscriptions_by_id.Add(subscription_id, subscription_state);
+         query_state.subscription_ids.Add(subscription_id);
+
+         if (!subscriptions_by_client.TryGetValue(client_id, out var client_subscriptions)) {
+            client_subscriptions = new();
+            subscriptions_by_client.Add(client_id, client_subscriptions);
+         }
+         client_subscriptions.Add(subscription_id);
+
+         _log_lifecycle($"DSLBookkeeper.Subscribe client_id: {client_id} subscription_id: {subscription_id} active_subscriptions_for_dsl: {query_state.subscription_ids.Count} dsl: {dsl_key}");
+         return subscription_id;
+      }
+   }
+
+   public void Unsubscribe(Guid subscription_id) {
+      lock (lock_obj) {
+         if (disposed) {
+            return;
+         }
+         _unsubscribe_locked(subscription_id);
+      }
+   }
+
+   public void DisposeClient(string client_id) {
+      ArgumentException.ThrowIfNullOrWhiteSpace(client_id);
+
+      lock (lock_obj) {
+         if (disposed) {
+            return;
+         }
+
+         if (!subscriptions_by_client.TryGetValue(client_id, out var subscription_ids)) {
+            return;
+         }
+
+         foreach (var subscription_id in subscription_ids.ToArray()) {
+            _unsubscribe_locked(subscription_id);
+         }
+      }
+   }
+
+   public void DoWork() {
+      List<(Action<DataPacket> on_data, DataPacket packet)> callbacks = new();
+
+      lock (lock_obj) {
+         if (disposed) {
+            return;
+         }
+
+         foreach (var query_state in queries_by_dsl.Values) {
+            while (query_state.pending_packets.TryDequeue(out var packet)) {
+               foreach (var subscription_id in query_state.subscription_ids) {
+                  if (subscriptions_by_id.TryGetValue(subscription_id, out var subscription_state)) {
+                     callbacks.Add((subscription_state.on_data, packet));
+                  }
+               }
+            }
+         }
+      }
+
+      foreach (var callback in callbacks) {
+         callback.on_data(callback.packet);
+      }
+   }
+
+   public void Dispose() {
+      lock (lock_obj) {
+         if (disposed) {
+            return;
+         }
+
+         foreach (var query_state in queries_by_dsl.Values) {
+            _dispose_query_state(query_state);
+         }
+
+         queries_by_dsl.Clear();
+         subscriptions_by_id.Clear();
+         subscriptions_by_client.Clear();
+         disposed = true;
+      }
+   }
+
+   QueryState _create_query_state(string dsl_key) {
+      _log_lifecycle($"DSLBookkeeper._create_query_state creating DSLStream dsl: {dsl_key}");
+      var stream = new DSLStream(dsl_key);
+      var query_state = new QueryState {
+         dsl = dsl_key,
+         stream = stream,
+      };
+
+      DataStream.ChangeDelegate stream_handler = new_value => {
+         if (new_value is not DataPacket packet) {
+            return;
+         }
+
+         lock (lock_obj) {
+            if (!disposed) {
+               query_state.pending_packets.Enqueue(packet);
+            }
+         }
+      };
+
+      query_state.stream_handler = stream_handler;
+      stream.onChange += stream_handler;
+      return query_state;
+   }
+
+   void _unsubscribe_locked(Guid subscription_id) {
+      if (!subscriptions_by_id.Remove(subscription_id, out var subscription_state)) {
+         return;
+      }
+
+      if (subscriptions_by_client.TryGetValue(subscription_state.client_id, out var client_subscriptions)) {
+         client_subscriptions.Remove(subscription_id);
+         if (client_subscriptions.Count == 0) {
+            subscriptions_by_client.Remove(subscription_state.client_id);
+         }
+      }
+
+      if (queries_by_dsl.TryGetValue(subscription_state.dsl_key, out var query_state)) {
+         query_state.subscription_ids.Remove(subscription_id);
+         _log_lifecycle($"DSLBookkeeper.Unsubscribe subscription_id: {subscription_id} client_id: {subscription_state.client_id} remaining_subscriptions_for_dsl: {query_state.subscription_ids.Count} dsl: {subscription_state.dsl_key}");
+         if (query_state.subscription_ids.Count == 0) {
+            _log_lifecycle($"DSLBookkeeper.Unsubscribe garbage collecting DSLStream; last subscription removed dsl: {subscription_state.dsl_key}");
+            _dispose_query_state(query_state);
+            queries_by_dsl.Remove(subscription_state.dsl_key);
+         }
+      }
+   }
+
+   static string _normalize_dsl(string dsl) =>
+      dsl.Trim();
+
+   static void _dispose_query_state(QueryState query_state) {
+      _log_lifecycle($"DSLBookkeeper._dispose_query_state disposing DSLStream dsl: {query_state.dsl} pending_packets: {query_state.pending_packets.Count} subscription_count: {query_state.subscription_ids.Count}");
+      if (query_state.stream_handler is not null) {
+         query_state.stream.onChange -= query_state.stream_handler;
+         query_state.stream_handler = null;
+      }
+      query_state.subscription_ids.Clear();
+      query_state.pending_packets.Clear();
+   }
+
+   static void _log_lifecycle(string message) {
+      Console.WriteLine(message);
+      Log.Gen.info(message);
+   }
+
+   void _throw_if_disposed() {
+      if (disposed) {
+         throw new ObjectDisposedException(nameof(DSLBookkeeper));
+      }
+   }
+}

src/DataStreamSuite/DSPublisher/DSLWSServer/DSLWebSocketMessages.cs

@@ -0,0 +1,39 @@
+namespace DSPublisher.DSLWSServer;
+
+using System;
+using System.Text.Json;
+using System.Text.Json.Serialization;
+
+/// <summary>
+/// Client-to-server websocket protocol message for subscribing and unsubscribing.
+/// Instantiated by System.Text.Json when DSLAPIWSListener deserializes client messages.
+/// </summary>
+public sealed class DSLWebSocketRequest {
+   [JsonPropertyName("type")]
+   public string? type { get; set; }
+
+   [JsonPropertyName("dsl")]
+   public string? dsl { get; set; }
+
+   [JsonPropertyName("subscriptionId")]
+   public Guid? subscription_id { get; set; }
+}
+
+/// <summary>
+/// Server-to-client websocket protocol message. Data responses carry serialized
+/// DataPacket JSON in the packet field.
+/// Instantiated by DSLAPIWSListener before serializing responses to clients.
+/// </summary>
+public sealed class DSLWebSocketResponse {
+   [JsonPropertyName("type")]
+   public string type { get; set; } = "";
+
+   [JsonPropertyName("subscriptionId")]
+   public Guid? subscription_id { get; set; }
+
+   [JsonPropertyName("message")]
+   public string? message { get; set; }
+
+   [JsonPropertyName("packet")]
+   public JsonElement? packet { get; set; }
+}

src/DataStreamSuite/DSPublisher/DSLWSServer/DSPAPIServer.cs

@@ -0,0 +1,37 @@
+namespace DSPublisher.DSLWSServer;
+
+using System;
+using System.Threading;
+using System.Threading.Tasks;
+
+/// <summary>
+/// Top-level DSPublisher API server. Owns the websocket listener and the DSL
+/// subscription bookkeeper, and exposes the small lifecycle surface Program.cs uses.
+/// Instantiated by Program.cs.
+/// </summary>
+public sealed class DSPAPIServer : IDisposable {
+   readonly DSLBookkeeper bookkeeper;
+   readonly DSLAPIWSListener web_socket_listener;
+
+   public DSPAPIServer(global::DSLAPIServerConfig config) {
+      ArgumentNullException.ThrowIfNull(config);
+      bookkeeper = new DSLBookkeeper();
+      var web_socket_prefix = $"http://{config.Ip}:{config.Port}/ws/";
+      web_socket_listener = new DSLAPIWSListener(web_socket_prefix, bookkeeper);
+      Console.WriteLine($"DSPAPIServer listening for websocket DSL API requests on {web_socket_prefix}");
+   }
+
+   public Task StartAsync(CancellationToken cancellation_token = default) =>
+      web_socket_listener.StartAsync(cancellation_token);
+
+   public Task StopAsync() =>
+      web_socket_listener.StopAsync();
+
+   public void DoWork() =>
+      bookkeeper.DoWork();
+
+   public void Dispose() {
+      web_socket_listener.Dispose();
+      bookkeeper.Dispose();
+   }
+}

src/DataStreamSuite/DSPublisher/DSLWSServer/IDSLBookkeeper.cs

@@ -0,0 +1,18 @@
+namespace DSPublisher.DSLWSServer;
+
+using System;
+
+using BCG;
+using Bts.DataStreamCore;
+
+/// <summary>
+/// Internal contract between the websocket listener and the DSL subscription owner.
+/// The listener handles transport; the bookkeeper owns stream lifetime and callbacks.
+/// Implemented by DSLBookkeeper and consumed by DSLAPIWSListener.
+/// </summary>
+public interface IDSLBookkeeper : IDisposable {
+   Guid Subscribe(string dsl, Action<DataPacket> on_data, string client_id);
+   void Unsubscribe(Guid subscription_id);
+   void DisposeClient(string client_id);
+   void DoWork();
+}

src/DataStreamSuite/DSPublisher/DSPublisher.csproj

@@ -0,0 +1,31 @@
+<Project Sdk="Microsoft.NET.Sdk">
+
+  <PropertyGroup>
+    <OutputType>Exe</OutputType>
+    <TargetFramework>net8.0</TargetFramework>
+    <Nullable>enable</Nullable>
+    <ImplicitUsings>enable</ImplicitUsings>
+  </PropertyGroup>
+
+  <ItemGroup>
+    <ProjectReference Include="..\DataStream\DataStream.csproj" />
+    <ProjectReference Include="..\PubSub\PubSub.csproj" />
+    <ProjectReference Include="..\SharedStreams\SharedStreams.csproj" />
+  </ItemGroup>
+
+  <ItemGroup>
+    <ConfigurationProfile Include="appsettings*.json" />
+
+    <None Update="appsettings*.json">
+      <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
+      <TargetPath>DSPublisher.%(Filename)%(Extension)</TargetPath>
+    </None>
+    <None Update="WSPublishers\**\*.py">
+      <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
+    </None>
+    <None Update="DSLWSClient\**\*.py">
+      <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
+    </None>
+  </ItemGroup>
+
+</Project>

src/DataStreamSuite/DSPublisher/DSPublisherConfig.cs

@@ -0,0 +1,21 @@
+using System.Collections.Generic;
+
+public sealed class DSPublisherConfig {
+   public List<StreamPublishingNode> StreamPublishingNodes { get; init; } = new();
+   public List<PythonPublishingNode> PythonPublishingNodes { get; init; } = new();
+}
+
+public sealed class DSLAPIServerConfig {
+   public string Ip { get; init; } = "localhost";
+   public int Port { get; init; } = 19093;
+}
+
+public sealed class StreamPublishingNode {
+   public string Name { get; init; } = string.Empty;
+   public string Expression { get; init; } = string.Empty;
+}
+
+public sealed class PythonPublishingNode {
+   public string PythonScriptLocation { get; init; } = string.Empty;
+   public string Args { get; init; } = string.Empty;
+}

src/DataStreamSuite/DSPublisher/Program.cs

@@ -0,0 +1,125 @@
+using System;
+using System.IO;
+using System.Text.Json;
+using System.Threading;
+
+using BCG;
+using bts;
+using Bts.PubSub;
+using bts.BlueOrb;
+using DSPublisher.DSLWSServer;
+
+Console.WriteLine("Starting DSPublisher");
+
+var configPath = load_config_path();
+Console.WriteLine($"Loading config from: {configPath}");
+if (!File.Exists(configPath)) {
+   Console.WriteLine($"Config file not found: {configPath}");
+   return;
+}
+
+var json = File.ReadAllText(configPath);
+using var doc = JsonDocument.Parse(json);
+
+if (!doc.RootElement.TryGetProperty("DSPublisher", out var dspPublisherElement)) {
+   Console.WriteLine("DSPublisher config section not found");
+   return;
+}
+
+var json_options = new JsonSerializerOptions {
+   PropertyNameCaseInsensitive = true,
+};
+
+var config = dspPublisherElement.Deserialize<DSPublisherConfig>(json_options) ?? new DSPublisherConfig();
+var dsl_api_server_config = load_dsl_api_server_config(doc, json_options);
+
+Console.WriteLine("StreamPublishingNodes:");
+foreach (var node in config.StreamPublishingNodes) {
+   Console.WriteLine($"  Name: {node.Name}");
+   Console.WriteLine($"  Expression: {node.Expression}");
+}
+
+Console.WriteLine("PythonPublishingNodes:");
+foreach (var node in config.PythonPublishingNodes) {
+   Console.WriteLine($"  PythonScriptLocation: {node.PythonScriptLocation}");
+   Console.WriteLine($"  Args: {node.Args}");
+}
+
+var pubsub_config = load_pubsub_config(doc, json_options);
+PubSubConfig.SetGlobalConfig(pubsub_config);
+using var view_stream_database = new ViewStreamDatabase(new SubscriberCoreFactory());
+ViewStreamDatabase.SetGlobalInstance(view_stream_database);
+
+//var pub_table_broker = new PubTableBroker();
+//var stream_publishers = _process_publishing_nodes(config.StreamPublishingNodes, pub_table_broker);
+
+using var dsp_api_server = new DSPAPIServer(dsl_api_server_config);
+await dsp_api_server.StartAsync();
+
+while (true) {
+   view_stream_database.doWork();
+   dsp_api_server.DoWork();
+   Thread.Sleep(TimeSpan.FromMilliseconds(10));
+}
+
+string load_config_path() {
+   var environment = Environment.GetEnvironmentVariable("DOTNET_ENVIRONMENT");
+   var baseFileName = string.IsNullOrWhiteSpace(environment)
+      ? "appsettings.json"
+      : $"appsettings.{environment}.json";
+   var dspPublisherFileName = $"DSPublisher.{baseFileName}";
+   var dspPublisherConfigPath = Path.Combine(AppContext.BaseDirectory, dspPublisherFileName);
+   if (File.Exists(dspPublisherConfigPath)) {
+      return dspPublisherConfigPath;
+   }
+
+   return Path.Combine(AppContext.BaseDirectory, baseFileName);
+}
+
+PubSubConfig load_pubsub_config(JsonDocument doc, JsonSerializerOptions json_options) {
+   if (!doc.RootElement.TryGetProperty("PubSub", out var pubsub_element)) {
+      Log.Gen.warn("PubSub config section not found; using PubSubConfig.Default().");
+      return PubSubConfig.Default();
+   }
+
+   var pubsub_config = pubsub_element.Deserialize<PubSubConfig>(json_options);
+   if (pubsub_config is null) {
+      Log.Gen.error("PubSub config section could not be deserialized; using PubSubConfig.Default().");
+      return PubSubConfig.Default();
+   }
+
+   Log.Gen.info("Loaded PubSub config from DSPublisher config file.");
+   return pubsub_config;
+}
+
+DSLAPIServerConfig load_dsl_api_server_config(JsonDocument doc, JsonSerializerOptions json_options) {
+   if (!doc.RootElement.TryGetProperty("DSLAPIServer", out var dsl_api_server_element)) {
+      Log.Gen.warn("DSLAPIServer config section not found; using DSLAPIServerConfig defaults.");
+      return new DSLAPIServerConfig();
+   }
+
+   var dsl_api_server_config = dsl_api_server_element.Deserialize<DSLAPIServerConfig>(json_options);
+   if (dsl_api_server_config is null) {
+      Log.Gen.error("DSLAPIServer config section could not be deserialized; using DSLAPIServerConfig defaults.");
+      return new DSLAPIServerConfig();
+   }
+
+   Log.Gen.info($"Loaded DSLAPIServer config ip: {dsl_api_server_config.Ip} port: {dsl_api_server_config.Port}");
+   return dsl_api_server_config;
+}
+
+List<IDisposable> _process_publishing_nodes(IEnumerable<StreamPublishingNode> publishing_nodes, PubTableBroker pub_table_broker) {
+   var stream_publishers = new List<IDisposable>();
+   foreach (var node in publishing_nodes) {
+      if (string.IsNullOrWhiteSpace(node.Name) || string.IsNullOrWhiteSpace(node.Expression)) {
+         Log.Gen.error("Skipping StreamPublishingNode because name or expression is empty.");
+         continue;
+      }
+
+      Log.Gen.info($"Creating ViewPublisher name: {node.Name} expression: {node.Expression}");
+      var stream = new DSLStream(node.Expression);
+      stream_publishers.Add(new ViewPublisher(node.Name, pub_table_broker, stream));
+   }
+
+   return stream_publishers;
+}

src/DataStreamSuite/DSPublisher/Services/.gitkeep

@@ -0,0 +1 @@
+

src/DataStreamSuite/DSPublisher/WSPublishers/DatabentoPublisher.py

@@ -0,0 +1,146 @@
+"""
+Requires:
+   pip install -U databento
+   pip install websockets
+
+Sources:
+   https://databento.com/docs/quickstart/set-up
+   https://databento.com/docs/schemas-and-data-formats/bbo
+"""
+
+import datetime as dt
+import subprocess
+import sys
+import time
+
+try:
+   import databento as db
+except ImportError:
+   print("Installing missing dependency: databento")
+   subprocess.check_call([sys.executable, "-m", "pip", "install", "-U", "databento"])
+   import databento as db
+
+from databento.common.error import BentoError
+
+try:
+   import websockets
+except ImportError:
+   print("Installing missing dependency: websockets")
+   subprocess.check_call([sys.executable, "-m", "pip", "install", "websockets"])
+   import websockets
+
+from databento_transformer import DatabentoTransformer
+from pubsub_client import PubSubClient
+
+
+API_KEY = "db-VksTJvhMmbLjbJu9Kg6GPS99B8FDN"
+DATASET = "GLBX.MDP3"
+SYMBOL = "ES.FUT"
+PUBSUB_WS_URL = "ws://localhost:19092/ws/"
+BBO_TABLE_NAME = "livebbo_glbx_future_es_continuouscontracts"
+TRADES_TABLE_NAME = "livetrades_glbx_future_es_continuouscontracts"
+DEFINITIONS_TABLE_NAME = "livecontracts_glbx_future_es"
+
+_pubsub_client = PubSubClient(PUBSUB_WS_URL)
+_transformer = DatabentoTransformer(
+   symbol=SYMBOL,
+   bbo_table_name=BBO_TABLE_NAME,
+   trades_table_name=TRADES_TABLE_NAME,
+   definitions_table_name=DEFINITIONS_TABLE_NAME,
+)
+
+
+def _publish_contract_definitions() -> None:
+   historical_client = db.Historical(API_KEY)
+   dataset_range = historical_client.metadata.get_dataset_range(DATASET)
+   schema_range = dataset_range.get("schema", {}).get("definition", {})
+   available_end = schema_range.get("end", dataset_range["end"])
+
+   if isinstance(available_end, str):
+      available_end_dt = dt.datetime.fromisoformat(available_end.replace("Z", "+00:00"))
+   else:
+      available_end_dt = available_end
+
+   # Databento's range end is exclusive. Back off slightly so the request stays
+   # strictly inside the available window.
+   end_dt = available_end_dt - dt.timedelta(seconds=1)
+   start_dt = max(end_dt - dt.timedelta(days=1), end_dt.replace(hour=0, minute=0, second=0, microsecond=0))
+   start = start_dt.isoformat()
+   end = end_dt.isoformat()
+
+   definition_data = historical_client.timeseries.get_range(
+      dataset=DATASET,
+      schema="definition",
+      stype_in="parent",
+      symbols=SYMBOL,
+      start=start,
+      end=end,
+   )
+
+   df = definition_data.to_df()
+   payload = _transformer.build_definitions_payload(df)
+   if payload is None:
+      print("No Databento contract definitions returned")
+      return
+
+   print(f"Publishing {len(payload['rows'])} Databento contract definition rows")
+   _pubsub_client.publish(payload)
+
+
+def on_record(record) -> None:
+   # Print every callback first so we can see whether Databento is flowing
+   # even if our row extraction logic misses a field shape.
+   print(record)
+
+   payloads = _transformer.transform_record(record)
+   if payloads:
+      for payload in payloads:
+         _pubsub_client.publish(payload)
+      return
+
+   print("Skipping record with no BBO/trade fields")
+
+
+def main() -> None:
+   _pubsub_client.start()
+   print(f"Starting Databento live client for {SYMBOL} on {DATASET}")
+   try:
+      _publish_contract_definitions()
+   except Exception as ex:
+      print(f"Contract definition publish failed: {ex}", flush=True)
+
+   while True:
+      try:
+         client = db.Live(key=API_KEY)
+
+         client.subscribe(
+            dataset=DATASET,
+            schema="bbo-1s",
+            stype_in="parent",
+            symbols=SYMBOL,
+         )
+         client.subscribe(
+            dataset=DATASET,
+            schema="trades",
+            stype_in="parent",
+            symbols=SYMBOL,
+         )
+
+         client.add_callback(on_record)
+         client.start()
+         print("Databento client started, waiting for BBO and trade records...", flush=True)
+         client.block_for_close()
+         print("Databento client closed cleanly", flush=True)
+         break
+      except BentoError as ex:
+         print(f"Databento live connection error: {ex}", flush=True)
+         print("Retrying Databento live connection in 5 seconds...", flush=True)
+         time.sleep(5)
+      except Exception as ex:
+         print(f"Databento publisher unexpected error: {ex}", flush=True)
+         print("Retrying Databento live connection in 5 seconds...", flush=True)
+         time.sleep(5)
+
+
+if __name__ == "__main__":
+   main()

src/DataStreamSuite/DSPublisher/WSPublishers/databento_transformer.py

@@ -0,0 +1,244 @@
+import datetime as dt
+
+
+class DatabentoTransformer:
+   def __init__(
+      self,
+      symbol: str,
+      bbo_table_name: str,
+      trades_table_name: str,
+      definitions_table_name: str,
+   ):
+      self._symbol = symbol
+      self._bbo_table_name = bbo_table_name
+      self._trades_table_name = trades_table_name
+      self._definitions_table_name = definitions_table_name
+      self._instrument_id_to_symbol: dict[int, str] = {}
+      self._last_bbo_row: dict | None = None
+      self._trade_row_id = 0
+      self._trade_table_created = False
+
+   def transform_record(self, record) -> list[dict]:
+      payload = self._handle_bbo_record(record)
+      if payload is not None:
+         return [payload]
+
+      payload = self._handle_trade_record(record)
+      if payload is not None:
+         return [payload]
+
+      return []
+
+   def build_definitions_payload(self, definition_df) -> dict | None:
+      if definition_df is None or definition_df.empty:
+         return None
+
+      df = definition_df.reset_index()
+      if "instrument_class" in df.columns:
+         df = df[df["instrument_class"].astype(str) == "F"]
+      if "security_type" in df.columns:
+         df = df[df["security_type"].astype(str) == "FUT"]
+      if "raw_symbol" in df.columns:
+         df = df.sort_values("ts_recv" if "ts_recv" in df.columns else "ts_event")
+         df = df.drop_duplicates(subset=["raw_symbol"], keep="last")
+
+      rows: list[dict] = []
+      for idx, row in enumerate(df.itertuples(index=False), start=1):
+         instrument_id = int(getattr(row, "instrument_id", 0) or 0)
+         security_name = str(getattr(row, "raw_symbol", ""))
+         if instrument_id > 0 and security_name:
+            self._instrument_id_to_symbol[instrument_id] = security_name
+
+         rows.append({
+            "id": idx,
+            "security_name": security_name,
+            "instrument_id": instrument_id,
+            "instrument_class": str(getattr(row, "instrument_class", "")),
+            "security_type": str(getattr(row, "security_type", "")),
+            "asset": str(getattr(row, "asset", "")),
+            "exchange": str(getattr(row, "exchange", "")),
+            "currency": str(getattr(row, "currency", "")),
+            "expiration": self._ns_to_iso(getattr(row, "expiration", 0)),
+            "activation": self._ns_to_iso(getattr(row, "activation", 0)),
+            "min_price_increment": self._nano_to_float(getattr(row, "min_price_increment", 0)),
+            "display_factor": self._nano_to_float(getattr(row, "display_factor", 0)),
+            "price_display_multiplier": self._nano_to_float(getattr(row, "display_factor", 0)),
+            "contract_multiplier": int(getattr(row, "contract_multiplier", 0) or 0),
+            "contract_multiplier_unit": int(getattr(row, "contract_multiplier_unit", 0) or 0),
+            "unit_of_measure": str(getattr(row, "unit_of_measure", "")),
+            "unit_of_measure_qty": self._nano_to_float(getattr(row, "unit_of_measure_qty", 0)),
+            "min_lot_size": int(getattr(row, "min_lot_size", 0) or 0),
+            "min_trade_vol": int(getattr(row, "min_trade_vol", 0) or 0),
+            "max_trade_vol": int(getattr(row, "max_trade_vol", 0) or 0),
+            "maturity_year": int(getattr(row, "maturity_year", 0) or 0),
+            "maturity_month": int(getattr(row, "maturity_month", 0) or 0),
+            "maturity_day": int(getattr(row, "maturity_day", 0) or 0),
+            "underlying": str(getattr(row, "underlying", "")),
+         })
+
+      if not rows:
+         return None
+
+      return {
+         "op": "create_table",
+         "table_name": self._definitions_table_name,
+         "rows": rows,
+      }
+
+   def _handle_bbo_record(self, record) -> dict | None:
+      row = self._to_bbo_row(record)
+      if row is None:
+         return None
+
+      if self._last_bbo_row is None:
+         payload = {
+            "op": "create_table",
+            "table_name": self._bbo_table_name,
+            "rows": [row],
+         }
+      else:
+         payload = {
+            "op": "multiple_remove_add",
+            "table_name": self._bbo_table_name,
+            "removed_rows": [self._last_bbo_row],
+            "added_rows": [row],
+         }
+
+      self._last_bbo_row = row
+      return payload
+
+   def _handle_trade_record(self, record) -> dict | None:
+      row = self._to_trade_row(record)
+      if row is None:
+         return None
+
+      row["id"] = self._trade_row_id
+      self._trade_row_id += 1
+
+      if not self._trade_table_created:
+         payload = {
+            "op": "create_table",
+            "table_name": self._trades_table_name,
+            "rows": [row],
+         }
+         self._trade_table_created = True
+         return payload
+
+      return {
+         "op": "multiple_remove_add",
+         "table_name": self._trades_table_name,
+         "removed_rows": [],
+         "added_rows": [row],
+      }
+
+   def _to_bbo_row(self, record) -> dict | None:
+      bid_px = self._get_level_attr(record, "bid_px", default=self._get_attr(record, "bid_px_00"))
+      ask_px = self._get_level_attr(record, "ask_px", default=self._get_attr(record, "ask_px_00"))
+      if bid_px is None and ask_px is None:
+         return None
+
+      return {
+         "id": 0,
+         "security_name": self._get_security_name(record),
+         "timestamp": self._get_timestamp(record),
+         "bid_px": float(bid_px or 0.0),
+         "ask_px": float(ask_px or 0.0),
+         "bid_sz": int(self._get_level_attr(record, "bid_sz", default=self._get_attr(record, "bid_sz_00", default=0)) or 0),
+         "ask_sz": int(self._get_level_attr(record, "ask_sz", default=self._get_attr(record, "ask_sz_00", default=0)) or 0),
+      }
+
+   def _to_trade_row(self, record) -> dict | None:
+      price = self._get_attr(record, "price", "px")
+      size = self._get_attr(record, "size")
+      if price is None and size is None:
+         return None
+
+      side = self._get_attr(record, "side", "aggressor_side", default="")
+
+      return {
+         "id": 0,
+         "security_name": self._get_security_name(record),
+         "timestamp": self._get_timestamp(record),
+         "price": float(price or 0.0),
+         "size": int(size or 0),
+         "side": str(side),
+      }
+
+   def _get_security_name(self, record) -> str:
+      instrument_id = self._get_attr(record, "instrument_id", default=0)
+      try:
+         instrument_id_int = int(instrument_id or 0)
+      except (TypeError, ValueError):
+         instrument_id_int = 0
+
+      if instrument_id_int > 0 and instrument_id_int in self._instrument_id_to_symbol:
+         return self._instrument_id_to_symbol[instrument_id_int]
+
+      value = self._get_attr(
+         record,
+         "symbol",
+         "raw_symbol",
+         "stype_out_symbol",
+         "stype_in_symbol",
+         default=self._symbol,
+      )
+      return str(value)
+
+   def _get_timestamp(self, record) -> int:
+      value = self._get_attr(record, "ts_event", "ts_recv", default=0)
+      try:
+         return int(value)
+      except (TypeError, ValueError):
+         return 0
+
+   def _get_attr(self, record, *names, default=None):
+      for name in names:
+         if hasattr(record, name):
+            value = getattr(record, name)
+            if value is not None:
+               return value
+      return default
+
+   def _get_level_attr(self, record, name, default=None):
+      levels = self._get_attr(record, "levels", default=None)
+      if levels:
+         level0 = levels[0]
+         if hasattr(level0, name):
+            value = getattr(level0, name)
+            if value is not None:
+               return value
+      return default
+
+   def _ns_to_iso(self, timestamp_ns) -> str:
+      if timestamp_ns is None:
+         return ""
+
+      if hasattr(timestamp_ns, "isoformat"):
+         try:
+            if hasattr(timestamp_ns, "to_pydatetime"):
+               timestamp_dt = timestamp_ns.to_pydatetime()
+            else:
+               timestamp_dt = timestamp_ns
+
+            if timestamp_dt is None:
+               return ""
+
+            if getattr(timestamp_dt, "tzinfo", None) is None:
+               timestamp_dt = timestamp_dt.replace(tzinfo=dt.timezone.utc)
+            return timestamp_dt.isoformat()
+         except (TypeError, ValueError, AttributeError):
+            pass
+
+      try:
+         ts_int = int(timestamp_ns)
+         if ts_int <= 0:
+            return ""
+         return dt.datetime.fromtimestamp(ts_int / 1_000_000_000, tz=dt.timezone.utc).isoformat()
+      except (TypeError, ValueError, OSError, OverflowError):
+         return ""
+
+   def _nano_to_float(self, value) -> float:
+      try:
+         return float(value) / 1_000_000_000.0
+      except (TypeError, ValueError):
+         return 0.0

src/DataStreamSuite/DSPublisher/WSPublishers/pubsub_client.py

@@ -0,0 +1,58 @@
+import asyncio
+import json
+import queue
+import threading
+
+import websockets
+
+
+class PubSubClient:
+   def __init__(self, ws_url: str):
+      self._ws_url = ws_url
+      self._outgoing_queue: "queue.Queue[dict]" = queue.Queue()
+      self._sender_thread: threading.Thread | None = None
+
+   def start(self) -> threading.Thread:
+      if self._sender_thread is None or not self._sender_thread.is_alive():
+         self._sender_thread = self._start_sender_thread()
+      return self._sender_thread
+
+   def publish(self, payload: dict) -> None:
+      self._outgoing_queue.put(payload)
+
+   def publish_create_table(self, table_name: str, rows: list[dict]) -> None:
+      self.publish({
+         "op": "create_table",
+         "table_name": table_name,
+         "rows": rows,
+      })
+
+   def publish_multiple_remove_add(
+      self,
+      table_name: str,
+      removed_rows: list[dict],
+      added_rows: list[dict],
+   ) -> None:
+      self.publish({
+         "op": "multiple_remove_add",
+         "table_name": table_name,
+         "removed_rows": removed_rows,
+         "added_rows": added_rows,
+      })
+
+   def _start_sender_thread(self) -> threading.Thread:
+      thread = threading.Thread(target=self._run_send_loop, daemon=True)
+      thread.start()
+      return thread
+
+   def _run_send_loop(self) -> None:
+      asyncio.run(self._ws_send_loop())
+
+   async def _ws_send_loop(self) -> None:
+      async with websockets.connect(self._ws_url) as ws:
+         print(f"Connected to PubSubServer websocket: {self._ws_url}")
+         while True:
+            payload = await asyncio.to_thread(self._outgoing_queue.get)
+            body = json.dumps(payload)
+            print(body)
+            await ws.send(body)

src/DataStreamSuite/DSPublisher/appsettings.Development.json

@@ -0,0 +1,69 @@
+{
+  "PubSub": {
+    "HeartbeatInterval": 15000,
+    "MissingHeartbeatsBeforeGc": 50,
+    "DisposedConnectionCheckPeriod": 1000,
+    "ServerPort": 22019,
+    "Routing": {
+      "PublisherRoutes": [
+        {
+          "Name": "default",
+          "ServerAddress": "bts-cloud://127.0.0.1:22019",
+          "TagRegExp": ".*"
+        }
+      ],
+      "SubscriberRoutes": [
+        {
+          "Name": "default",
+          "ServerAddress": "bts-cloud://127.0.0.1:22019",
+          "TagRegExp": ".*"
+        }
+      ]
+    },
+    "TagTypeToAssemblyMap": {
+      "Entries": [
+        {
+          "HandlerClass": "BCG.PubSubServerTagHandler",
+          "Assembly": "DataStreamCore.dll"
+        }
+      ]
+    },
+    "TagHandlerFactory": {
+      "Handlers": [
+        {
+          "TagType": "DataStream",
+          "Assembly": "DataStreamCore",
+          "Class": "bts.PubsubServerStreamTagHandler"
+        },
+        {
+          "TagType": "DS_.*",
+          "Assembly": "DataStreamCore",
+          "Class": "bts.PubsubServerStreamTagHandler"
+        },
+        {
+          "TagType": ".*",
+          "Assembly": "DataStreamCore",
+          "Class": "bts.PubsubServerGenericDataHandler"
+        }
+      ]
+    }
+  },
+  "DSLAPIServer": {
+    "Ip": "localhost",
+    "Port": 19093
+  },
+  "DSPublisher": {
+    "StreamPublishingNodes": [
+      {
+        "Name": "sample_stream",
+        "Expression": "MillisecondStream(1000,5)"
+      }
+    ],
+    "PythonPublishingNodes": [
+      {
+        "PythonScriptLocation": "WSPublishers/DatabentoPublisher.py",
+        "Args": ""
+      }
+    ]
+  }
+}

src/DataStreamSuite/DSPublisher/appsettings.json

@@ -0,0 +1,69 @@
+{
+  "PubSub": {
+    "HeartbeatInterval": 15000,
+    "MissingHeartbeatsBeforeGc": 50,
+    "DisposedConnectionCheckPeriod": 1000,
+    "ServerPort": 22019,
+    "Routing": {
+      "PublisherRoutes": [
+        {
+          "Name": "default",
+          "ServerAddress": "bts-cloud://127.0.0.1:22019",
+          "TagRegExp": ".*"
+        }
+      ],
+      "SubscriberRoutes": [
+        {
+          "Name": "default",
+          "ServerAddress": "bts-cloud://127.0.0.1:22019",
+          "TagRegExp": ".*"
+        }
+      ]
+    },
+    "TagTypeToAssemblyMap": {
+      "Entries": [
+        {
+          "HandlerClass": "BCG.PubSubServerTagHandler",
+          "Assembly": "DataStreamCore.dll"
+        }
+      ]
+    },
+    "TagHandlerFactory": {
+      "Handlers": [
+        {
+          "TagType": "DataStream",
+          "Assembly": "DataStreamCore",
+          "Class": "bts.PubsubServerStreamTagHandler"
+        },
+        {
+          "TagType": "DS_.*",
+          "Assembly": "DataStreamCore",
+          "Class": "bts.PubsubServerStreamTagHandler"
+        },
+        {
+          "TagType": ".*",
+          "Assembly": "DataStreamCore",
+          "Class": "bts.PubsubServerGenericDataHandler"
+        }
+      ]
+    }
+  },
+  "DSLAPIServer": {
+    "Ip": "localhost",
+    "Port": 19093
+  },
+  "DSPublisher": {
+    "StreamPublishingNodes": [
+      {
+        "Name": "sample_stream",
+        "Expression": "MillisecondStream(1000,5)"
+      }
+    ],
+    "PythonPublishingNodes": [
+      {
+        "PythonScriptLocation": "WSPublishers/DatabentoPublisher.py",
+        "Args": ""
+      }
+    ]
+  }
+}

src/DataStreamSuite/DataStream/BlueOrb/AddColumnsStream.cs

@@ -2,7 +2,8 @@
    using BCG;
 
    [DataStreamUserRespresentation("AddColumnsStream")]
-   public class AddColumnsStream: ViewStream {
+   // example: AddColumnsStream(l,"jc1:001")
+   public class AddColumnsStream : ViewStream {
       public AddColumnsStream(ViewStream parent_str, string add_col_expr) {
          ViewStream rs = new BCG.AddColumnsStream(parent_str, add_col_expr);
          mySetRealSource(rs);

src/DataStreamSuite/DataStream/BlueOrb/HideColumnsStream.cs

@@ -0,0 +1,126 @@
+namespace BlueOrb.Operators.Core {
+   using BCG;
+
+   using System;
+   using System.Collections;
+   using System.Collections.Generic;
+
+   [DataStreamUserRespresentation("HideColumnsStream")]
+   public class HideColumnsStream : ViewStream {
+      // Wrapper row to hide the columns
+      public class MyRow : IBlueRow {
+         #region IBlueRow members
+         private IBlueTableSchema _schema;
+         private IBlueRow _row;
+         public MyRow(IBlueTableSchema schema, IBlueRow row) {
+            _schema = schema;
+            _row = row;
+         }
+         public IBlueTableSchema schema { get { return _schema; } }
+         #endregion
+
+         #region ICollection members
+         public int Count { get { return schema.columnNames.Length; } }
+
+         public bool IsSynchronized { get { return false; } }
+
+         public object SyncRoot { get { return this; } }
+
+         public void CopyTo(Array array, int index) {
+            int i = index;
+            foreach (object o in this as IEnumerable)
+               array.SetValue(o, i++);
+         }
+         #endregion
+
+         #region IDictionary members
+         public bool IsFixedSize { get { return true; } }
+
+         public bool IsReadOnly { get { return true; } }
+
+         public object this[object key] {
+            get {
+               if (key is string)
+                  return this[(string)key];
+               return this[(int)key];
+            }
+            set { throw new NotSupportedException("The IBlueRow is readonly"); }
+         }
+
+         public object this[int index] {
+            get {
+               return this[schema.columnInfo[index].colName];
+            }
+         }
+
+         public object this[string col_name] {
+            get {
+               if (_schema.contains_column(col_name))
+                  return _row[col_name];
+               else
+                  throw new ApplicationException($"{col_name} not found");
+            }
+         }
+
+         public ICollection Keys { get { return schema.columnNames; } }
+
+         public ICollection Values {
+            get {
+               object[] retVal = new object[schema.columnNames.Length];
+               CopyTo(retVal, 0);
+               return retVal;
+            }
+         }
+
+         public void Add(object key, object value) { throw new NotSupportedException("The IBlueRow is readonly"); }
+
+         public void Clear() { throw new NotSupportedException("The IBlueRow is readonly"); }
+
+         public bool Contains(object key) { return schema[(string)key] != null; }
+
+         public IDictionaryEnumerator GetEnumerator() { return new BlueRowDictionaryEnumerator(this); }
+
+         IEnumerator IEnumerable.GetEnumerator() { return new BlueRowEnumerator(this); }
+
+         public void Remove(object key) { throw new NotSupportedException("The IBlueRow is readonly"); }
+         #endregion
+
+      }
+
+      private readonly ViewStream _parent_str;
+      private readonly HashSet<string> _removed_cols;
+      private BlueTable _state;
+      public HideColumnsStream(ViewStream parent_str, string hide_cols_expr) {
+         _parent_str = parent_str;
+         _removed_cols = new HashSet<string>(hide_cols_expr.Split(',', ' '));
+         if (_removed_cols.Contains("id"))
+            _removed_cols.Remove("id");
+      }
+
+      protected override void myOnFirstSubscriber() {
+         _parent_str.onChange += _handle_parent_str_cb;
+      }
+
+      private void _handle_parent_str_cb(object dp_obj) {
+         // this could be made incremental and hence faster
+         DataPacket dp_in = dp_obj as DataPacket;
+         if (_state == null) {
+            List<ColumnInfo> cis = new List<ColumnInfo>();
+            foreach (ColumnInfo ci in dp_in.schema.columnInfo) {
+               if (!_removed_cols.Contains(ci.colName))
+                  cis.Add(new ColumnInfo(ci.colName, cis.Count, ci.colType));
+            }
+            _state = new BlueTable(new BlueTableSchema(dp_in.schema.tableName, cis.ToArray()));
+         }
+         _state.clear();
+         foreach (IBlueRow r in dp_in.newRows) {
+            _state.addRow(new MyRow(_state.schema, r));
+         }
+         var dp_out = BlueDataUtility.build_dp_inc(_state,
+            myLastValue == null ? new BlueTable(_state.schema) : (myLastValue as DataPacket).newRows);
+         myFireEvent(dp_out);
+      }
+   }
+
+
+}

src/DataStreamSuite/DataStream/BlueOrb/Join/CartesianProductStream.cs

@@ -0,0 +1,14 @@
+namespace bts.BlueOrb {
+   using BCG;
+
+   [DataStreamUserRespresentation("CartesianProductStream")]
+   public class CartesianProductStream : ViewStream {
+      public CartesianProductStream(ViewStream ls, ViewStream rs) {
+         ls = new AddColumnsStream(ls, "ljc: 001");
+         rs = new AddColumnsStream(rs, "rjc: 001");
+         ViewStream res = new SimpleJoinStream(ls, "ljc", rs, "rjc");
+         res = new global::BlueOrb.Operators.Core.HideColumnsStream(res, "ljc, rjc");
+         mySetRealSource(res);
+      }
+   }
+}

src/DataStreamSuite/DataStream/BlueOrb/Join/FullJoinStream.cs

@@ -0,0 +1,15 @@
+namespace BlueOrb.Operators.Core {
+   using System;
+
+   using BCG;
+
+   [DataStreamUserRespresentation("FullJoinStream")]
+   // JoinStream is technically a LeftJoin. FullJoin only results in rows that have bpth left and right side
+   public class FullJoinStream : ViewStream {
+      public FullJoinStream(ViewStream l_str, string ljcs, ViewStream r_str, string rjcs) {
+         ViewStream js = new global::bts.BlueOrb.SimpleJoinStream(l_str, ljcs, r_str, rjcs);
+         ViewStream rs = new bts.BlueOrb.FilterStream(js, r => !DBNull.Value.Equals(r["right_id"]));
+         mySetRealSource(rs);
+      }
+   }
+}

src/DataStreamSuite/DataStream/BlueOrb/Join/SimpleJoinStream.cs

@@ -1,4 +1,5 @@
-namespace bts.BlueOrb {
+namespace bts.BlueOrb
+{
    using BCG;
 
    using Bts.DataStream;
@@ -13,8 +14,10 @@
    using RowHash = System.Int32;
 
    [DataStreamUserRespresentation("JoinStream")]
-   public class SimpleJoinStream : ViewStream {
-      public class JoinBookkeepingData {
+   public class SimpleJoinStream : ViewStream
+   {
+      public class JoinBookkeepingData
+      {
          // initialized in cons
          readonly string[] _join_col_names;
          private readonly BlueRowHasher _hasher;
@@ -23,7 +26,8 @@
          int[]? _join_cols;
          private IDictionary<RowHash, List<IBlueRow>> _hash_row_map = new Dictionary<RowHash, List<IBlueRow>>();
 
-         public JoinBookkeepingData(ViewStream vs, List<string> join_cols, BlueRowHasher row_hasher) {
+         public JoinBookkeepingData(ViewStream vs, List<string> join_cols, BlueRowHasher row_hasher)
+         {
             _join_col_names = join_cols.Where(c => !string.IsNullOrEmpty(c)).ToArray();
             _hasher = row_hasher;
          }
@@ -40,22 +44,28 @@
          public List<IBlueRow>? get_matching_rows(int hash) =>
             _hash_row_map.GetValueOrDefault(hash);
 
-         private RowHash _get_hash(IBlueRow r) {
-            if (_join_cols == null || _join_cols.Length == 0) {
+         private RowHash _get_hash(IBlueRow r)
+         {
+            if (_join_cols == null || _join_cols.Length == 0)
+            {
                return 0;
             }
             return _hasher.get_hash(r, _join_cols);
          }
 
-         public void update_data(IDataPacket dp) {
-            if (!ready) {
+         public void update_data(IDataPacket dp)
+         {
+            if (!ready)
+            {
                _schema = dp.newRows.schema;
                _join_cols = _schema.ResolveIndices(_join_col_names);
             }
             _hash_row_map.Clear();
-            foreach (IBlueRow r in dp.newRows) {
+            foreach (IBlueRow r in dp.newRows)
+            {
                int hash = _get_hash(r);
-               if (!_hash_row_map.TryGetValue(hash, out var hash_rows)) {
+               if (!_hash_row_map.TryGetValue(hash, out var hash_rows))
+               {
                   hash_rows = new List<IBlueRow>();
                   _hash_row_map.Add(hash, hash_rows);
                }
@@ -64,8 +74,10 @@
          }
 
          [MemberNotNullWhen(true, nameof(_schema))]
-         public bool ready {
-            get {
+         public bool ready
+         {
+            get
+            {
                return _schema != null;
             }
          }
@@ -77,9 +89,11 @@
       IBlueTableSchema? _schema;
 
       public SimpleJoinStream(ViewStream ls, string ljc, ViewStream rs, string rjc) :
-         this(ViewStreamDatabase.instance, ls, ljc, rs, rjc) { }
+         this(ViewStreamDatabase.instance, ls, ljc, rs, rjc)
+      { }
 
-      public SimpleJoinStream(ViewStreamDatabase vsd, ViewStream ls, string ljc, ViewStream rs, string rjc) {
+      public SimpleJoinStream(ViewStreamDatabase vsd, ViewStream ls, string ljc, ViewStream rs, string rjc)
+      {
          _ls = ls;
          _rs = rs;
          List<string> lj_col_list = ljc.Split(',').Where(x => !String.IsNullOrEmpty(x)).ToList();
@@ -91,30 +105,35 @@
          _rdata = new JoinBookkeepingData(rs, rj_col_list, hasher);
       }
 
-      protected override void myOnFirstSubscriber() {
+      protected override void myOnFirstSubscriber()
+      {
          _ls.onChange += _ls_onchange;
          _rs.onChange += _rs_onchange;
       }
 
-      private void _ls_onchange(object dp_obj) {
+      private void _ls_onchange(object dp_obj)
+      {
          var dp = (IDataPacket)dp_obj;
          _ldata.update_data(dp);
          _form_and_fire();
       }
 
-      private void _rs_onchange(object dp_obj) {
+      private void _rs_onchange(object dp_obj)
+      {
          var dp = (IDataPacket)dp_obj;
          _rdata.update_data(dp);
          _form_and_fire();
       }
 
-      protected override void myOnLastSubscriber() {
+      protected override void myOnLastSubscriber()
+      {
          _ls.onChange -= _ls_onchange;
          _rs.onChange -= _rs_onchange;
       }
 
       [MemberNotNull(nameof(_schema))]
-      private void _build_schema() {
+      private void _build_schema()
+      {
          if (_schema != null)
             return;
          var lschema = Log.notnull(_ldata.schema);
@@ -122,52 +141,64 @@
          var cis = new List<ColumnInfo> {
             new("id", 0, TypeCode.Int32)
          };
-         foreach (ColumnInfo ci in lschema.columnInfo) {
+         foreach (ColumnInfo ci in lschema.columnInfo)
+         {
             string col_name = rschema.contains_column(ci.colName) ? $"left_{ci.colName}" : ci.colName;
             cis.Add(new ColumnInfo(col_name, cis.Count, ci.colType));
          }
-         foreach (ColumnInfo ci in rschema.columnInfo) {
+         foreach (ColumnInfo ci in rschema.columnInfo)
+         {
             string col_name = lschema.contains_column(ci.colName) ? $"right_{ci.colName}" : ci.colName;
             cis.Add(new ColumnInfo(col_name, cis.Count, ci.colType));
          }
          _schema = new BlueTableSchema($"{lschema.tableName}_join_{rschema.tableName}", cis.ToArray());
       }
 
-      private void _form_and_fire() {
+      private void _form_and_fire()
+      {
          if (!_ldata.ready || !_rdata.ready)
             return;
          _build_schema();
          List<object[]> res_rows = new List<object[]>();
 
          // this is where the join happens
-         foreach (var kv in _ldata.rows) {
+         foreach (var kv in _ldata.rows)
+         {
             RowHash hash = kv.Key;
             List<IBlueRow> l_rows = kv.Value;
             List<IBlueRow>? r_rows = _rdata.get_matching_rows(hash);
 
-            foreach (IBlueRow lr in l_rows) {
-               if (r_rows != null) {
-                  foreach (IBlueRow r in r_rows) {
+            foreach (IBlueRow lr in l_rows)
+            {
+               if (r_rows != null)
+               {
+                  foreach (IBlueRow r in r_rows)
+                  {
                      object[] rowvals = build_row_array(res_rows.Count, lr);
                      copy_row(rowvals, r, false);
                      res_rows.Add(rowvals);
                   }
-               } else {
+               }
+               else
+               {
                   object[] rowvals = build_row_array(res_rows.Count, lr);
                   res_rows.Add(rowvals);
                }
             }
          }
          BlueTable rv = new BlueTable(_schema);
-         foreach (object[] rval in res_rows) {
+         foreach (object[] rval in res_rows)
+         {
             rv.addRow(new BlueRow(_schema, rval));
          }
          var dp = BlueDataUtility.build_dp(rv, myLastValue as IDataPacket);
          myFireEvent(dp);
 
-         object[] build_row_array(int id, IBlueRow l_row) {
+         object[] build_row_array(int id, IBlueRow l_row)
+         {
             object[] rv = new object[_schema.columnNames.Length];
-            for (int i = 0; i < _schema.columnNames.Length; i++) {
+            for (int i = 0; i < _schema.columnNames.Length; i++)
+            {
                rv[i] = DBNull.Value;
             }
             rv[0] = id;
@@ -175,30 +206,26 @@
             return rv;
          }
 
-         void copy_row(object?[] rval, IBlueRow r, bool is_left) {
+         void copy_row(object?[] rval, IBlueRow r, bool is_left)
+         {
             int offset = 1;
             JoinBookkeepingData sdata = _ldata;
-            if (is_left) {
+            if (is_left)
+            {
                offset = 1;
                sdata = _ldata;
-            } else {
+            }
+            else
+            {
                sdata = _rdata;
                offset = 1 + _ldata.RequireSchema().columnInfo.Length;
             }
-            for (int i = 0; i < sdata.RequireSchema().columnInfo.Length; i++) {
+            for (int i = 0; i < sdata.RequireSchema().columnInfo.Length; i++)
+            {
                rval[offset + i] = r[i];
             }
          }
       }
    }
 
-   [DataStreamUserRespresentation("FullJoinStream")]
-   public class FullJoinStream : ViewStream {
-      public FullJoinStream(ViewStream l_str, string ljcs, ViewStream r_str, string rjcs) {
-         ViewStream js = new SimpleJoinStream(l_str, ljcs, r_str, rjcs);
-         ViewStream rs = new bts.BlueOrb.FilterStream(js, r => !DBNull.Value.Equals(r["right_id"]));
-         mySetRealSource(rs);
-      }
-   }
-
 }

src/DataStreamSuite/DataStream/BlueOrb/TimeSeriesStream.cs

@@ -29,22 +29,26 @@ namespace bts.BlueOrb {
          _stream.onChange -= new ChangeDelegate(_onstream_change);
       }
 
-
+      private bool _orig_schema_has_timestamp = false;
       private void _onstream_change(object o) {
          DataPacket dp = o as DataPacket;
          DateTime now = DateTime.Now;
          if (_table == null) {
             IBlueTableSchema schema = new BlueTableSchema(dp.newRows.schema.columnInfo, dp.newRows.schema.tableName);
-            schema = schema.add_columns(Tuple.Create(typeof(DateTime), "timestamp"));
+            if (!schema.contains_column("timestamp")) {
+               _orig_schema_has_timestamp = true;
+               schema = schema.add_columns(Tuple.Create(typeof(DateTime), "timestamp"));
+            }
             _table = new CircularBufferTable(schema, _size);
          }
          foreach (IBlueRow r in dp.addedRows) {
             MutableBlueRow ar = new MutableBlueRow(_table.schema, r);
-            ar.setValue("timestamp", now);
+            if (!_orig_schema_has_timestamp)
+               ar.setValue("timestamp", now);
             ar.setValue("id", _id++);
             _table.add_row(ar);
          }
-         var dp_out =  BlueDataUtility.build_dp(_table, myLastValue as DataPacket);
+         var dp_out = BlueDataUtility.build_dp_inc(_table, myLastValue as DataPacket);
          myFireEvent(dp_out);
       }
    }

src/DataStreamSuite/DataStream/Primitives/CartesianProduct/CartesianProductStream.cs

@@ -1,22 +0,0 @@
-// using System;
-// using BCG;
-// using BCG.Utilities;
-
-// namespace BCG
-// {
-//    public class CartesianProductStream : ViewStream
-//    {
-//       public CartesianProductStream(ViewStream v1, ViewStream v2)
-//       {
-//          mySetRealSource(
-//          new JoinStream (new AddColumnsStream (v1,
-//             "hiddenCartesianColumn=1"),
-//             "hiddenCartesianColumn",
-//             "!id hiddenCartesianColumn dateTime:dateBoundary",
-//             new AddColumnsStream (v2,
-//             "hiddenCartesianColumn=1"),
-//             "hiddenCartesianColumn",
-//             "!id hiddenCartesianColumn"));
-//       }
-//    }
-// }

src/DataStreamSuite/DataStream/Primitives/Misc/PubsubEchoTestStream.cs

@@ -8,48 +8,34 @@ using System;
 using System.Collections.Generic;
 using System.IO;
 using System.Linq;
+using bts;
+using BCG;
 
-namespace bts
-{
+namespace BlueOrb.Pubsub.Diagnostics {
+   // This publishes data to the pubsub server, subscribes, gets it back and records the time it took for the round trip
    [DataStreamUserRespresentation("pss_echo")]
-   public class PubsubEchoTestStream : ViewStream, IWorker
-   {
-      public class PayloadMessage
-      {
+   public class PubsubEchoTestStream : ViewStream, IWorker {
+      public class PayloadMessage {
          public readonly string? msg_id;
          public readonly Int64 timestamp = 0;
          public readonly byte[]? payload = null;
 
-         public static byte[] create_data(int payload_size, byte payload_byte)
-         {
+         public static byte[] create_data(int payload_size, byte payload_byte) {
             var rv = new byte[payload_size];
-            for (int i = 0; i < payload_size; i++)
-            {
+            for (int i = 0; i < payload_size; i++) {
                rv[i] = payload_byte;
             }
             return rv;
          }
 
-         // public PayloadMessage(string guid_, Int64 timestamp_, int payload_size, byte payload_byte)
-         // {
-         //    guid = guid_;
-         //    timestamp = timestamp_;
-         //    payload = new byte[payload_size];
-         //    for (int i = 0; i < payload_size; i++)
-         //    {
-         //       payload[i] = payload_byte;
-         //    }
-         // }
-         public PayloadMessage(string guid_, Int64 timestamp_, byte[] payload_)
-         {
+         public PayloadMessage(string guid_, Int64 timestamp_, byte[] payload_) {
             msg_id = guid_;
             timestamp = timestamp_;
             payload = payload_;
          }
 
          private PayloadMessage() { }
-         public byte[] serialize()
-         {
+         public byte[] serialize() {
             var str = new MemoryStream();
             BinaryWriter bw = new BinaryWriter(str);
             bw.Write(msg_id ?? string.Empty);
@@ -61,8 +47,7 @@ namespace bts
             return bytes;
          }
 
-         public static PayloadMessage deserialize(byte[] ser_bytes)
-         {
+         public static PayloadMessage deserialize(byte[] ser_bytes) {
             var str = new MemoryStream(ser_bytes) {
                Position = 0
             };
@@ -87,8 +72,7 @@ namespace bts
       private readonly string _tag_name;
       private readonly int _payload_sz;
       private readonly IPubSubRouter _router;
-      private enum IOState
-      {
+      private enum IOState {
          SEND = 0,
          RECV = 1
       };
@@ -123,8 +107,7 @@ namespace bts
       public PubsubEchoTestStream(string server_url, int publish_period_msecs, int payload_size, string tag_name)
       : this(server_url, TimeSpan.FromMilliseconds(publish_period_msecs), payload_size, tag_name, ViewStreamDatabase.instance) { }
 
-      protected override void myOnFirstSubscriber()
-      {
+      protected override void myOnFirstSubscriber() {
          _pub_core = PublisherCore.Create(_router);
          _sub_core = new SubscriberCore(_router);
          _sub_core.subscribe(_tag_name);
@@ -132,8 +115,7 @@ namespace bts
 
       }
 
-      protected override void myOnLastSubscriber()
-      {
+      protected override void myOnLastSubscriber() {
          _vsd.removeWorker(this);
          Log.notnull(_sub_core).unsubscribe(_tag_name);
       }
@@ -144,8 +126,7 @@ namespace bts
             SubscriberRoutes = new() { new(serverUrl, ".*") },
          });
 
-      private void _fire_dp_after_send()
-      {
+      private void _fire_dp_after_send() {
          Log.notnull(_last_sent_msg);
          // as soon as the data is published, its reflected in the stream state
          BlueRow r = new BlueRow(_table.schema, _table.Count, _last_sent_msg.msg_id, _last_sent_msg.timestamp, DBNull.Value, DBNull.Value,
@@ -155,8 +136,7 @@ namespace bts
          myFireEvent(dp_out);
       }
 
-      private void _fire_dp_after_recv()
-      {
+      private void _fire_dp_after_recv() {
          Log.notnull(_last_recvd_msg);
          Log.notnull(_last_sent_msg);
          // update the last publish row
@@ -173,13 +153,11 @@ namespace bts
          }
       }
 
-      private string _get_next_id()
-      {
+      private string _get_next_id() {
          return _table.Count.ToString();
       }
 
-      private void _publish_next_packet()
-      {
+      private void _publish_next_packet() {
          Log.notnull(_pub_core);
          var bytes = PayloadMessage.create_data(_payload_sz, _payload_byte);
          _last_sent_msg = new PayloadMessage(_get_next_id(), epoch_now(), bytes);
@@ -188,13 +166,10 @@ namespace bts
          _last_publish_time = DateTime.Now;
       }
 
-      public void doWork()
-      {
-         switch (_state)
-         {
+      public void doWork() {
+         switch (_state) {
             case IOState.SEND:
-               if (DateTime.Now - _last_publish_time >= _publish_period)
-               {
+               if (DateTime.Now - _last_publish_time >= _publish_period) {
                   _publish_next_packet();
                   _last_publish_time = DateTime.Now;
                   _fire_dp_after_send();
@@ -206,8 +181,7 @@ namespace bts
                var updates = Log.notnull(_sub_core).get_updates(TimeSpan.Zero);
                updates = updates.Where(u => u.tag == _tag_name);
                var recv_msg = find_packet(updates);
-               if (recv_msg != null)
-               {
+               if (recv_msg != null) {
                   _last_recvd_msg = recv_msg;
                   _fire_dp_after_recv();
                   _state = IOState.SEND;
@@ -230,8 +204,7 @@ namespace bts
          }
       }
 
-      public static long epoch_now()
-      {
+      public static long epoch_now() {
          return DateTimeOffset.UtcNow.ToUnixTimeMilliseconds();
       }
 

src/DataStreamSuite/DataStream/Primitives/Project/New/NewProjectStream.cs

@@ -1,12 +1,17 @@
 namespace bts {
    using BCG;
 
+   using bts.collections;
+
+   using IronPython.Runtime.Operations;
+
    using ProtoBuf;
 
    using System;
    using System.Collections;
    using System.Collections.Generic;
    using System.Linq;
+   using System.Runtime.Intrinsics.Arm;
 
    public class ProjectData {
       // schema from the base stream

src/DataStreamSuite/DataStream/Publish/PublisherStream.cs

@@ -1,4 +1,5 @@
-namespace BCG {
+namespace BlueOrb.PubSub {
+   using BCG;
    // this for testing purposes mainly, its wraps a ViewPublisher around a stream, 
    // and hence epublishes it out to the PS
    public class PublisherStream : ViewStream {

src/DataStreamSuite/DataStreamCore/BlueData/BlueDataUtility.cs

@@ -1847,7 +1847,7 @@ namespace BCG {
          return sb.ToString();
       }
 
-      public static string id_string(IBlueRowCollection table) {
+      public static string rows_id_string(this IBlueRowCollection table) {
          string rv = String.Empty;
          foreach (IBlueRow r in table)
             rv += $"{r["id"]} ";
@@ -2246,7 +2246,7 @@ namespace BCG {
 
       public static T get_row_value<T>(IBlueRow r, string col_name, T error_val) {
          try {
-            object rv = (r != null && r.Contains(col_name)) ? r[col_name] : DBNull.Value;
+            object rv = (r != null && r.schema.contains_column(col_name)) ? r[col_name] : DBNull.Value;
             if (rv == null || rv == DBNull.Value)
                return error_val;
             else

src/DataStreamSuite/DataStreamCore/BlueData/ExtensionFunctions/Comparisons.cs

@@ -0,0 +1,142 @@
+namespace BlueOrb.Data {
+   using BCG;
+
+   using bts;
+
+
+   using System;
+   using System.Linq;
+
+   public static class Comparisons {
+      public static int compare_row(this IBlueRow row, IBlueRow other_row, params string[] col_names) {
+         int comp_res = 0;
+         foreach (string col_name in col_names) {
+            object val1 = row[col_name];
+            object val2 = other_row[col_name];
+            comp_res = compare_row_values(val1, val2);
+            if (comp_res != 0)
+               break;
+         }
+         return comp_res;
+      }
+
+      public static void compare_tables_id(this IBlueRowCollection table1,
+                                           IBlueRowCollection table2,
+                                           out IBlueRowCollection rows_only_in_table1,
+                                           out IBlueRowCollection rows_only_in_table2) {
+         Log.assert(table1 != null);
+         table2 = table2 ?? new BlueTable(table1.schema);
+         var schema = get_output_schema(table1, table2);
+         var table1_only = new BlueTable(schema);
+         var table2_only = new BlueTable(schema);
+
+         if (table1 is not null) {
+            foreach (IBlueRow row in table1)
+               if (table2 is null || !table2.containsID(row.get<int>("id")))
+                  table1_only.addRow(row);
+         }
+
+         if (table2 is not null) {
+            foreach (IBlueRow row in table2)
+               if (table1 is null || !table1.containsID(row.get<int>("id")))
+                  table2_only.addRow(row);
+         }
+
+         rows_only_in_table1 = table1_only;
+         rows_only_in_table2 = table2_only;
+      }
+
+      public static void compare_tables_deep(this IBlueRowCollection table1,
+                                             IBlueRowCollection table2,
+                                             out IBlueRowCollection rows_only_in_table1,
+                                             out IBlueRowCollection rows_only_in_table2) {
+         Log.assert(table1 != null);
+         table2 = table2 ?? new BlueTable(table1.schema);
+         var schema = get_output_schema(table1, table2);
+         validate_matching_schemas(table1, table2);
+
+         var table1_only = new BlueTable(schema);
+         var table2_only = new BlueTable(schema);
+         var compare_cols = schema.columnNames.Where(c => c != "id").ToArray();
+
+         if (table1 is not null) {
+            foreach (IBlueRow row in table1) {
+               if (table2 is null || !table2.containsID(row.get<int>("id"))) {
+                  table1_only.addRow(row);
+                  continue;
+               }
+
+               var other_row = table2[row.get<int>("id")];
+               if (row.compare_row(other_row, compare_cols) != 0) {
+                  table1_only.addRow(row);
+                  table2_only.addRow(other_row);
+               }
+            }
+         }
+
+         if (table2 is not null) {
+            foreach (IBlueRow row in table2)
+               if (table1 is null || !table1.containsID(row.get<int>("id")))
+                  table2_only.addRow(row);
+         }
+
+         rows_only_in_table1 = table1_only;
+         rows_only_in_table2 = table2_only;
+      }
+
+      static int compare_row_values(object v1, object v2) {
+         if (is_null(v1))
+            return is_null(v2) ? 0 : -1;
+         if (is_null(v2))
+            return 1;
+         try {
+            var comparable1 = (IComparable)v1;
+            return comparable1.CompareTo(v2);
+         } catch (InvalidCastException) {
+            if (v1 is byte[] bytes1 && v2 is byte[] bytes2)
+               return compare_bytes(bytes1, bytes2) ? 0 : 1;
+            throw;
+         }
+      }
+
+      static bool compare_bytes(byte[] v1, byte[] v2) {
+         if (v1.Length != v2.Length)
+            return false;
+
+         for (int i = 0; i < v1.Length; i++)
+            if (v1[i] != v2[i])
+               return false;
+
+         return true;
+      }
+
+      static bool is_null(object o) {
+         return o == null || o == DBNull.Value;
+      }
+
+      static IBlueTableSchema get_output_schema(IBlueRowCollection table1, IBlueRowCollection table2) {
+         if (table1 is not null)
+            return table1.schema;
+         if (table2 is not null)
+            return table2.schema;
+         throw new ArgumentNullException(nameof(table1), "At least one table must be non-null.");
+      }
+
+      static void validate_matching_schemas(IBlueRowCollection table1, IBlueRowCollection table2) {
+         if (table1 is null || table2 is null)
+            return;
+
+         var schema1 = table1.schema;
+         var schema2 = table2.schema;
+         if (schema1.columnInfo.Length != schema2.columnInfo.Length)
+            throw new ArgumentException("Tables must have matching schemas.");
+
+         for (int i = 0; i < schema1.columnInfo.Length; i++) {
+            var c1 = schema1.columnInfo[i];
+            var c2 = schema2.columnInfo[i];
+            if (c1.colName != c2.colName || c1.colType != c2.colType)
+               throw new ArgumentException("Tables must have matching schemas.");
+         }
+      }
+   }
+}

src/DataStreamSuite/DataStreamCore/BlueData/ExtensionFunctions/ToStrings.cs

@@ -0,0 +1,35 @@
+namespace BlueOrb.Data {
+   using BCG;
+
+   using System;
+   using System.Text;
+
+public static class ToStrings {
+      public static string ToDebugString(this IBlueRow row) {
+         StringBuilder sb = new StringBuilder();
+         foreach (ColumnInfo c in row.schema.columnInfo) {
+            if (sb.Length != 0)
+               sb.Append(", ");
+            sb.Append(c.colName);
+            sb.Append(": ");
+            object val = row[c.colName];
+            sb.Append(is_null(val) ? "NULL" : val.ToString());
+         }
+         return sb.ToString();
+      }
+
+      public static string ToDebugString(this IBlueRowCollection rows) {
+         StringBuilder sb = new StringBuilder();
+         foreach (IBlueRow row in rows) {
+            if (sb.Length != 0)
+               sb.AppendLine();
+            sb.Append(row.ToDebugString());
+         }
+         return sb.ToString();
+      }
+
+      static bool is_null(object o) {
+         return o == null || o == DBNull.Value;
+      }
+   }
+}

src/DataStreamSuite/DataStreamCore/DynamicLoading/PrimitiveManager.cs

@@ -6,89 +6,89 @@ namespace BCG {
    using System.Reflection;
    using System.IO;
    using Bts.DataStreamCore;
+   using System.Collections.Generic;
+   using System.Linq;
 
    public class PrimitiveManager {
       static MarketDataStreamFactory mdStreamFactory;
-      private IDictionary myTypeLookupTable = new Hashtable();
-      private static PrimitiveManager mySelf;
-      private ArrayList myAssemblyNames = new ArrayList();
+      private Dictionary<string, Type> _dslopname_to_type_map = new Dictionary<string, Type>();
+      private static PrimitiveManager _self;
+      private HashSet<string> _loaded_assemblies = new HashSet<string>();
+      private static string[] _allowed_namespaces_matches = new string[] {
+         "blueorb"
+      };
 
       private PrimitiveManager() {
-         myLoadAllTypes();
+         _load_all_streams();
       }
 
-      private void myLoadAllTypes() {
-         AppDomain currentDomain = AppDomain.CurrentDomain;
-         currentDomain.AssemblyResolve += new ResolveEventHandler(myAssemblyResolver);
-         loadAssembly(this.GetType().Assembly.FullName);
-         var assemblies = new string[] { "DataStream.dll", "SharedStreams.dll" };
-         foreach (string assemblyName in assemblies) {
-            bts.Log.Gen.info($"PrimitiveManager.myLoadAllTypes assemblyName: {assemblyName}");
-            string resolvedAssemblyPath = assemblyName;
-            if (!System.IO.Path.IsPathRooted(assemblyName)) {
-               resolvedAssemblyPath = Path.GetFullPath(Path.Combine(AppDomain.CurrentDomain.BaseDirectory, assemblyName));
+      private static bool _is_loadable_stream(Type type) {
+         Func<bool> namespace_match = () => {
+            foreach (string ns in _allowed_namespaces_matches) {
+               if (type.Namespace.ToLower().Contains(ns.ToLower()))
+                  return true;
             }
-            loadAssembly(resolvedAssemblyPath);
+            return false;
+         };
+         bool rv = type.IsSubclassOf(typeof(ViewStream))
+                   && type.IsPublic;
+         //rv = rv && (!type.IsInterface && !type.IsAbstract);
+         rv = rv && namespace_match();
+         return rv;
+      }
+
+      private void _load_all_streams() {
+         load_streams_from_assembly(this.GetType().Assembly.FullName);
+         var assemblies = new string[] { "DataStream.dll", "SharedStreams.dll" };
+         foreach (string assembly_name in assemblies) {
+            bts.Log.Gen.info($"PrimitiveManager._load_all_streams assemblyName: {assembly_name}");
+            string resolved_path_name = assembly_name;
+            if (!System.IO.Path.IsPathRooted(assembly_name)) {
+               resolved_path_name = Path.GetFullPath(Path.Combine(AppDomain.CurrentDomain.BaseDirectory, assembly_name));
+            }
+            load_streams_from_assembly(resolved_path_name);
          }
       }
 
-      private static Assembly myAssemblyResolver(object sender, ResolveEventArgs args) {
-         foreach (Assembly a in AppDomain.CurrentDomain.GetAssemblies())
-            if (args.Name.Equals(a.FullName))
-               return a;
-         return null;
+      public Type get_dsl_op_type(string dsl_op_name) {
+         Type rv;
+         if (!_dslopname_to_type_map.TryGetValue(dsl_op_name, out rv))
+            return null;
+         return rv;
       }
 
-      /// <summary>
-      /// Loads all the Stream types associated with the assembly
-      /// </summary>
-      /// <param name="assemblyName"></param>
-      public void loadAssembly(string assemblyName) {
-         bts.Log.Gen.info($"PrimitiveManager.loadAssembly assemblyName: {assemblyName}");
-         if (myAssemblyNames.Contains(assemblyName))
+      public void load_streams_from_assembly(string assembly_name) {
+         bts.Log.Gen.info($"PrimitiveManager.loadAssembly assemblyName: {assembly_name}");
+         if (_loaded_assemblies.Contains(assembly_name))
             return;
-         myAssemblyNames.Add(assemblyName);
-         Assembly loadedAssembly = GetType().Assembly.FullName.Equals(assemblyName) ?
+         _loaded_assemblies.Add(assembly_name);
+         Assembly loaded_assembly = GetType().Assembly.FullName.Equals(assembly_name) ?
             GetType().Assembly :
-            Assembly.LoadFrom(assemblyName);
+            Assembly.LoadFrom(assembly_name);
 
-         foreach (Type type in loadedAssembly.GetTypes())
-            if (!type.IsInterface
-               && !type.IsAbstract
-               && type.IsSubclassOf(typeof(ViewStream))
-               && type.IsPublic) {
-               bool isVisible = true;
-               string userRep = createDefaultName(type.Name);
+         foreach (Type type in loaded_assembly.GetTypes())
+            if (_is_loadable_stream(type)) {
+               bool is_visible = true;
+               string dsl_name = type.Name; //createDefaultName(type.Name);
                foreach (Attribute att in type.GetCustomAttributes(false))
                   if (att is DataStreamUserRespresentation) {
                      DataStreamUserRespresentation dsatt = att as DataStreamUserRespresentation;
-                     isVisible = dsatt.isVisible;
-                     userRep = isVisible ?
-                        dsatt.friendlyName == null ? userRep : dsatt.friendlyName
+                     is_visible = dsatt.isVisible;
+                     dsl_name = is_visible ?
+                        dsatt.friendlyName == null ? dsl_name : dsatt.friendlyName
                         : null;
                      break;
                   }
-               if (isVisible) {
-                 // Log.assert_d(myTypeLookupTable[userRep] == null, type.FullName);
-                  myTypeLookupTable[userRep] = type;
+               if (is_visible) {
+                  // Log.assert_d(myTypeLookupTable[userRep] == null, type.FullName);
+                  _dslopname_to_type_map[dsl_name] = type;
                }
             }
       }
 
-      public static string createDefaultName(string typeName) {
-         if (!typeName.EndsWith("Stream"))
-            return typeName;
-         string retVal = typeName.Remove(typeName.Length - "Stream".Length, "Stream".Length);
-         string startString = new String(new char[] { retVal[0] }).ToLower();
-         retVal = retVal.Remove(0, 1);
-         retVal = retVal.Insert(0, startString);
-         return retVal;
-      }
-
-
 
       public static PrimitiveManager instance {
-         get { return mySelf = mySelf == null ? new PrimitiveManager() : mySelf; }
+         get { return _self = _self == null ? new PrimitiveManager() : _self; }
       }
 
       public static void SetMarketDataClient(object value) {
@@ -96,31 +96,20 @@ namespace BCG {
          mdStreamFactory = new(value);
       }
 
-      /// <summary>
-      /// Returns the Type name corresponding to the primitive name
-      /// </summary>
-      /// <param name="friendlyName"></param>
-      /// <returns></returns>
-      public string find(string friendlyName) {
-         if (myTypeLookupTable[friendlyName] == null)
-            throw new ApplicationException(friendlyName + " is not the friendly name of any of the loaded types");
-         return (myTypeLookupTable[friendlyName] as Type).Name;
+      public string dsl_to_type_name(string dsl_op_name) {
+         Type rv;
+         if (!_dslopname_to_type_map.TryGetValue(dsl_op_name, out rv))
+            throw new ApplicationException(dsl_op_name + " is not a loaded dsl op");
+         return rv.Name;
       }
 
-      /// <summary>
-      /// Constructs and returns a ViewStream of type corresponding to the friendly name
-      /// </summary>
-      /// <param name="friendlyName">friendly primitive name </param>
-      /// <param name="args">args can be null or an empty ICollection if the paramterless constructor
-      /// is the desired one</param>
-      /// <returns></returns>
       public ViewStream build(string friendlyName, ICollection args) {
          object[] constructorArgs = null;
          if (args != null && args.Count > 0) {
             constructorArgs = new object[args.Count];
             args.CopyTo(constructorArgs, 0);
          }
-         var type = myTypeLookupTable[friendlyName] as Type
+         var type = _dslopname_to_type_map[friendlyName] as Type
             ?? throw new ApplicationException(friendlyName + " is not the friendly name of any of the loaded types");
          if (type.HasCustomAttribute<LiveMarketDataStreamAttribute>()) {
             if (mdStreamFactory is null) {
@@ -135,30 +124,14 @@ namespace BCG {
             Activator.CreateInstance(type, constructorArgs) as ViewStream;
       }
 
-      /// <summary>
-      /// Returns the name of all the primitives 
-      /// </summary>
-      /// <returns></returns>
-      public string[] getPrimitiveNames() {
-         string[] retVal = new string[myTypeLookupTable.Count];
-         myTypeLookupTable.Keys.CopyTo(retVal, 0);
-         return retVal;
+      public string[] get_dsl_op_names() {
+         return _dslopname_to_type_map.Keys.ToArray();
       }
 
-      /// <summary>
-      /// Return the types of all the streams
-      /// </summary>
-      /// <returns></returns>
-      public Type[] getStreamTypes() {
-         Type[] retVal = new Type[myTypeLookupTable.Count];
-         myTypeLookupTable.Values.CopyTo(retVal, 0);
-         return retVal;
-      }
 
-      public ICollection getConstructorSigns(string primitiveName) {
+      public ICollection get_dsl_op_cons_args(string dsl_op_name) {
          ArrayList retVal = new ArrayList();
-         Log.assert(myTypeLookupTable[primitiveName] != null);
-         Type t = myTypeLookupTable[primitiveName] as Type;
+         Type t = get_dsl_op_type(dsl_op_name);
          foreach (ConstructorInfo ci in t.GetConstructors()) {
             string sign = "";
             sign = t.Name + " (";
@@ -172,9 +145,18 @@ namespace BCG {
          return retVal;
       }
 
-      public string getAssemblyName(string primitiveName) {
-         Log.assert(myTypeLookupTable[primitiveName] != null);
-         return (myTypeLookupTable[primitiveName] as Type).Assembly.GetName().Name;
+      public string get_dslop_assembly_name(string dsl_op_name) {
+         Type rv;
+         if (!_dslopname_to_type_map.TryGetValue(dsl_op_name, out rv))
+            throw new ApplicationException(dsl_op_name + " is not a loaded dsl op");
+         return rv.Assembly.FullName;
       }
+
+      public string get_qualified_op_name(string dsl_op_name) {
+         Type dsl_op_type = get_dsl_op_type(dsl_op_name);
+         Log.assert(dsl_op_type != null);
+         return $"{dsl_op_type.Namespace}.{dsl_op_name}";
+      }
+
    }
 }

src/DataStreamSuite/DataStreamCore/Messaging/Subscription/TableAccumulator2.cs

@@ -3,23 +3,23 @@ using System.IO;
 using System.Collections.Generic;
 using System.Linq;
 
+using BlueOrb.Data;
 using bts;
+using System.Threading;
 
-namespace BCG
-{
-   // This class is responsible for mantaining the state fo a table. It gets the
-   // bytes from the pubsubserver via the handle_updates call (called by VSD), deserializes
+namespace BCG {
+   // This class is responsible for mantaining the state for a table. It gets the
+   // bytes from the pubsubserver via the handle_updates call 
+   // (called by VSD, - VSD is polling the SubscriberCOre for udpates), deserializes,
    // updates the state of the table, and fires the TableUpdateDelegate.
-   public class TableAccumulator2 : ITableAccumulator
-   {
+   public class TableAccumulator2 : ITableAccumulator {
       private readonly string _tag;
       private readonly ISubscriberCore _subscriber_core;
       IStreamMsgDeserilizer _deser = new StreamIODeserializationController();
       private BlueTable _table;
       private TableUpdateDelegate _on_update_delegate;
 
-      public TableAccumulator2(string tag, ISubscriberCore sub_core)
-      {
+      public TableAccumulator2(string tag, ISubscriberCore sub_core) {
          _tag = tag;
          _subscriber_core = sub_core;
       }
@@ -28,61 +28,54 @@ namespace BCG
 
       //public bool is_active() { return _on_update_delegate != null; }
 
-      public event TableUpdateDelegate on_update
-      {
-         add
-         {
+      public event TableUpdateDelegate on_update {
+         add {
             Log.Gen.info("TableAccumulator2.on_update.add {_tag} subscribed");
-            if (_on_update_delegate == null)
-            {
+            if (_on_update_delegate == null) {
                //ie first subscription
                Log.Gen.info("TableAccumulator2.on_update {_tag} subscribed for the first time");
                _subscriber_core.subscribe(_tag);
             }
             _on_update_delegate =
                Delegate.Combine(_on_update_delegate, value) as TableUpdateDelegate;
+            // a new subsriber should get a full callback immediatly (new rows, 0 deleted, all added)
             if (_table != null)
                value(_table, _table, new BlueTable(_table.schema));
          }
-         remove
-         {
+         remove {
             Log.Gen.info("TableAccumulator2.on_update.remove {_tag} unsubscribed");
-            if (_on_update_delegate != null)
-            {
+            if (_on_update_delegate != null) {
                _on_update_delegate =
                   Delegate.Remove(_on_update_delegate, value) as TableUpdateDelegate;
                int n_subs = _on_update_delegate == null ? 0 : _on_update_delegate.GetInvocationList().Length;
                Log.Gen.info("TableAccumulator2.on_update.remove {_tag} unsubscribed n_subs: {n_subs}");
-               if (_on_update_delegate == null)
-               {
+               if (_on_update_delegate == null) {
                   _subscriber_core.unsubscribe(_tag);
-                  // TODO
-                  //_table = null;
+                  _table = null;
                }
             }
 
          }
       }
       private int _update_no = 0;
-      public void handle_updates(IEnumerable<byte[]> msgs)
-      {
+      public void handle_updates(IEnumerable<byte[]> msgs) {
          if (msgs == null || msgs.Count() == 0)
             return;
          Log.Gen.info($"TableAccumulator2.handle_updates _update_no: {_update_no++} msgs: {msgs.Count()} msg_0: {msgs.ElementAt(0).Length}");
          IBlueRowCollection table_0 = _table == null ? null : new BlueTable(_table);
-         foreach (var msg in msgs)
-         {
-            bts.StreamIOMessage str_msg = _deser.deserialize(msg);
+         int i = 0;
+         foreach (var msg in msgs) {
+            bts.StreamIOMessage new_msg_txt = _deser.deserialize(msg);
             var state_txt = _table == null ? "NULL" : _table.Count.ToString();// BlueDataUtility.id_string(_table);
-            Log.Gen.info($"TableAccumulator2.handle_update tag: {_tag} sz: {msg.Length} str_msg: {str_msg} _state: {state_txt}");
-            switch (str_msg.op)
-            {
+            Log.Gen.info($"TableAccumulator2.handle_update tag: {_tag} object_hash: {GetHashCode()} thread_id: {Thread.CurrentThread.ManagedThreadId} msg: {i++ / msgs.Count()} msg_sz: {msg.Length} new_msg_txt: {new_msg_txt} pre_update_count: {state_txt}");
+
+            switch (new_msg_txt.op) {
                case bts.StreamIOMessage.StreamOp.CREATE:
-                  _table = new BlueTable(str_msg.create_message.rows);
+                  _table = new BlueTable(new_msg_txt.create_message.rows);
                   break;
                case bts.StreamIOMessage.StreamOp.MULTIPLE_REMOVE_ADD:
-                  _table.deleteRows(str_msg.multiple_remove_add_message.removed_rows);
-                  _table.addRows(str_msg.multiple_remove_add_message.added_rows);
+                  _table.deleteRows(new_msg_txt.multiple_remove_add_message.removed_rows);
+                  _table.addRows(new_msg_txt.multiple_remove_add_message.added_rows);
                   break;
             }
          }
@@ -91,11 +84,16 @@ namespace BCG
          IBlueRowCollection d_rows, a_rows;
          //BlueDataUtility.compute_differences_id_based(_table, table_0, out d_rows, out a_rows);
          BlueDataUtility.compute_differences_fast(_table, table_0, out d_rows, out a_rows);
+         Log.Gen.info($"TableAccumulator2.handle_updates firing update, _table: {_table.rows_id_string()} d_rows: {d_rows.rows_id_string()} a_rows: {a_rows.rows_id_string()}");
+         if (a_rows.Count == 0 && d_rows.Count == 0 && table_0 != null) {
+            Log.Gen.info($"TableAccmulator2.handle_updates ex! _table: {_table.ToDebugString()} table_0: {table_0.ToDebugString()} a_rows: {a_rows.ToDebugString()} d_rows: {d_rows.ToDebugString()}");
+            //same table was pubslished (maybe surpress it on the publisher or pubsubs)
+            return;
+         }
          _on_update_delegate?.Invoke(_table, a_rows, d_rows);
       }
 
-      public void update(Stream[] streams)
-      {
+      public void update(Stream[] streams) {
       }
 
       #endregion

src/DataStreamSuite/DataStreamCore/Messaging/v2/Receive/PubSubServer/PubsubServerStreamTagHandler.cs

@@ -30,7 +30,7 @@ namespace bts {
       }
    }
 
-   // This class is created by the SingleTagData for stream tag_type. Every pub
+   // This class is created by the SingleTagData for stream tag_type. Every DS tag has a PubsubServerSingleStreamTagHandler
 
    public class PubsubServerSingleStreamTagHandler : IPubSubServerTagHandler {
       // this is kept around to deserielize the incoming messages and keep the state of the table

src/DataStreamSuite/DataStreamExpressions/PrimitiveExpression.cs

@@ -20,7 +20,7 @@ public class PrimitiveExpression : StatementExpression {
    }
 
    public override string print(int startCol, IList<string> activeBindings) {
-      string prelude = "new " + PrimitiveManager.instance.find(myName) + " (";
+      string prelude = "new " + PrimitiveManager.instance.dsl_to_type_name(myName) + " (";
 
       return prelude + myPrintArgs(startCol + prelude.Length, activeBindings) + ")";
    }

src/DataStreamSuite/DataStreamPublisher/WSPublishers/DatabentoPublisher.py

@@ -0,0 +1,146 @@
+"""
+Requires:
+   pip install -U databento
+   pip install websockets
+
+Sources:
+   https://databento.com/docs/quickstart/set-up
+   https://databento.com/docs/schemas-and-data-formats/bbo
+"""
+
+import datetime as dt
+import subprocess
+import sys
+import time
+
+try:
+   import databento as db
+except ImportError:
+   print("Installing missing dependency: databento")
+   subprocess.check_call([sys.executable, "-m", "pip", "install", "-U", "databento"])
+   import databento as db
+
+from databento.common.error import BentoError
+
+try:
+   import websockets
+except ImportError:
+   print("Installing missing dependency: websockets")
+   subprocess.check_call([sys.executable, "-m", "pip", "install", "websockets"])
+   import websockets
+
+from databento_transformer import DatabentoTransformer
+from pubsub_client import PubSubClient
+
+
+API_KEY = "db-VksTJvhMmbLjbJu9Kg6GPS99B8FDN"
+DATASET = "GLBX.MDP3"
+SYMBOL = "ES.FUT"
+PUBSUB_WS_URL = "ws://localhost:19092/ws/"
+BBO_TABLE_NAME = "livebbo_glbx_future_es_continuouscontracts"
+TRADES_TABLE_NAME = "livetrades_glbx_future_es_continuouscontracts"
+DEFINITIONS_TABLE_NAME = "livecontracts_glbx_future_es"
+
+_pubsub_client = PubSubClient(PUBSUB_WS_URL)
+_transformer = DatabentoTransformer(
+   symbol=SYMBOL,
+   bbo_table_name=BBO_TABLE_NAME,
+   trades_table_name=TRADES_TABLE_NAME,
+   definitions_table_name=DEFINITIONS_TABLE_NAME,
+)
+
+
+def _publish_contract_definitions() -> None:
+   historical_client = db.Historical(API_KEY)
+   dataset_range = historical_client.metadata.get_dataset_range(DATASET)
+   schema_range = dataset_range.get("schema", {}).get("definition", {})
+   available_end = schema_range.get("end", dataset_range["end"])
+
+   if isinstance(available_end, str):
+      available_end_dt = dt.datetime.fromisoformat(available_end.replace("Z", "+00:00"))
+   else:
+      available_end_dt = available_end
+
+   # Databento's range end is exclusive. Back off slightly so the request stays
+   # strictly inside the available window.
+   end_dt = available_end_dt - dt.timedelta(seconds=1)
+   start_dt = max(end_dt - dt.timedelta(days=1), end_dt.replace(hour=0, minute=0, second=0, microsecond=0))
+   start = start_dt.isoformat()
+   end = end_dt.isoformat()
+
+   definition_data = historical_client.timeseries.get_range(
+      dataset=DATASET,
+      schema="definition",
+      stype_in="parent",
+      symbols=SYMBOL,
+      start=start,
+      end=end,
+   )
+
+   df = definition_data.to_df()
+   payload = _transformer.build_definitions_payload(df)
+   if payload is None:
+      print("No Databento contract definitions returned")
+      return
+
+   print(f"Publishing {len(payload['rows'])} Databento contract definition rows")
+   _pubsub_client.publish(payload)
+
+
+def on_record(record) -> None:
+   # Print every callback first so we can see whether Databento is flowing
+   # even if our row extraction logic misses a field shape.
+   print(record)
+
+   payloads = _transformer.transform_record(record)
+   if payloads:
+      for payload in payloads:
+         _pubsub_client.publish(payload)
+      return
+
+   print("Skipping record with no BBO/trade fields")
+
+
+def main() -> None:
+   _pubsub_client.start()
+   print(f"Starting Databento live client for {SYMBOL} on {DATASET}")
+   try:
+      _publish_contract_definitions()
+   except Exception as ex:
+      print(f"Contract definition publish failed: {ex}", flush=True)
+
+   while True:
+      try:
+         client = db.Live(key=API_KEY)
+
+         client.subscribe(
+            dataset=DATASET,
+            schema="bbo-1s",
+            stype_in="parent",
+            symbols=SYMBOL,
+         )
+         client.subscribe(
+            dataset=DATASET,
+            schema="trades",
+            stype_in="parent",
+            symbols=SYMBOL,
+         )
+
+         client.add_callback(on_record)
+         client.start()
+         print("Databento client started, waiting for BBO and trade records...", flush=True)
+         client.block_for_close()
+         print("Databento client closed cleanly", flush=True)
+         break
+      except BentoError as ex:
+         print(f"Databento live connection error: {ex}", flush=True)
+         print("Retrying Databento live connection in 5 seconds...", flush=True)
+         time.sleep(5)
+      except Exception as ex:
+         print(f"Databento publisher unexpected error: {ex}", flush=True)
+         print("Retrying Databento live connection in 5 seconds...", flush=True)
+         time.sleep(5)
+
+
+if __name__ == "__main__":
+   main()

src/DataStreamSuite/DataStreamPublisher/WSPublishers/databento_transformer.py

@@ -0,0 +1,244 @@
+import datetime as dt
+
+
+class DatabentoTransformer:
+   def __init__(
+      self,
+      symbol: str,
+      bbo_table_name: str,
+      trades_table_name: str,
+      definitions_table_name: str,
+   ):
+      self._symbol = symbol
+      self._bbo_table_name = bbo_table_name
+      self._trades_table_name = trades_table_name
+      self._definitions_table_name = definitions_table_name
+      self._instrument_id_to_symbol: dict[int, str] = {}
+      self._last_bbo_row: dict | None = None
+      self._trade_row_id = 0
+      self._trade_table_created = False
+
+   def transform_record(self, record) -> list[dict]:
+      payload = self._handle_bbo_record(record)
+      if payload is not None:
+         return [payload]
+
+      payload = self._handle_trade_record(record)
+      if payload is not None:
+         return [payload]
+
+      return []
+
+   def build_definitions_payload(self, definition_df) -> dict | None:
+      if definition_df is None or definition_df.empty:
+         return None
+
+      df = definition_df.reset_index()
+      if "instrument_class" in df.columns:
+         df = df[df["instrument_class"].astype(str) == "F"]
+      if "security_type" in df.columns:
+         df = df[df["security_type"].astype(str) == "FUT"]
+      if "raw_symbol" in df.columns:
+         df = df.sort_values("ts_recv" if "ts_recv" in df.columns else "ts_event")
+         df = df.drop_duplicates(subset=["raw_symbol"], keep="last")
+
+      rows: list[dict] = []
+      for idx, row in enumerate(df.itertuples(index=False), start=1):
+         instrument_id = int(getattr(row, "instrument_id", 0) or 0)
+         security_name = str(getattr(row, "raw_symbol", ""))
+         if instrument_id > 0 and security_name:
+            self._instrument_id_to_symbol[instrument_id] = security_name
+
+         rows.append({
+            "id": idx,
+            "security_name": security_name,
+            "instrument_id": instrument_id,
+            "instrument_class": str(getattr(row, "instrument_class", "")),
+            "security_type": str(getattr(row, "security_type", "")),
+            "asset": str(getattr(row, "asset", "")),
+            "exchange": str(getattr(row, "exchange", "")),
+            "currency": str(getattr(row, "currency", "")),
+            "expiration": self._ns_to_iso(getattr(row, "expiration", 0)),
+            "activation": self._ns_to_iso(getattr(row, "activation", 0)),
+            "min_price_increment": self._nano_to_float(getattr(row, "min_price_increment", 0)),
+            "display_factor": self._nano_to_float(getattr(row, "display_factor", 0)),
+            "price_display_multiplier": self._nano_to_float(getattr(row, "display_factor", 0)),
+            "contract_multiplier": int(getattr(row, "contract_multiplier", 0) or 0),
+            "contract_multiplier_unit": int(getattr(row, "contract_multiplier_unit", 0) or 0),
+            "unit_of_measure": str(getattr(row, "unit_of_measure", "")),
+            "unit_of_measure_qty": self._nano_to_float(getattr(row, "unit_of_measure_qty", 0)),
+            "min_lot_size": int(getattr(row, "min_lot_size", 0) or 0),
+            "min_trade_vol": int(getattr(row, "min_trade_vol", 0) or 0),
+            "max_trade_vol": int(getattr(row, "max_trade_vol", 0) or 0),
+            "maturity_year": int(getattr(row, "maturity_year", 0) or 0),
+            "maturity_month": int(getattr(row, "maturity_month", 0) or 0),
+            "maturity_day": int(getattr(row, "maturity_day", 0) or 0),
+            "underlying": str(getattr(row, "underlying", "")),
+         })
+
+      if not rows:
+         return None
+
+      return {
+         "op": "create_table",
+         "table_name": self._definitions_table_name,
+         "rows": rows,
+      }
+
+   def _handle_bbo_record(self, record) -> dict | None:
+      row = self._to_bbo_row(record)
+      if row is None:
+         return None
+
+      if self._last_bbo_row is None:
+         payload = {
+            "op": "create_table",
+            "table_name": self._bbo_table_name,
+            "rows": [row],
+         }
+      else:
+         payload = {
+            "op": "multiple_remove_add",
+            "table_name": self._bbo_table_name,
+            "removed_rows": [self._last_bbo_row],
+            "added_rows": [row],
+         }
+
+      self._last_bbo_row = row
+      return payload
+
+   def _handle_trade_record(self, record) -> dict | None:
+      row = self._to_trade_row(record)
+      if row is None:
+         return None
+
+      row["id"] = self._trade_row_id
+      self._trade_row_id += 1
+
+      if not self._trade_table_created:
+         payload = {
+            "op": "create_table",
+            "table_name": self._trades_table_name,
+            "rows": [row],
+         }
+         self._trade_table_created = True
+         return payload
+
+      return {
+         "op": "multiple_remove_add",
+         "table_name": self._trades_table_name,
+         "removed_rows": [],
+         "added_rows": [row],
+      }
+
+   def _to_bbo_row(self, record) -> dict | None:
+      bid_px = self._get_level_attr(record, "bid_px", default=self._get_attr(record, "bid_px_00"))
+      ask_px = self._get_level_attr(record, "ask_px", default=self._get_attr(record, "ask_px_00"))
+      if bid_px is None and ask_px is None:
+         return None
+
+      return {
+         "id": 0,
+         "security_name": self._get_security_name(record),
+         "timestamp": self._get_timestamp(record),
+         "bid_px": float(bid_px or 0.0),
+         "ask_px": float(ask_px or 0.0),
+         "bid_sz": int(self._get_level_attr(record, "bid_sz", default=self._get_attr(record, "bid_sz_00", default=0)) or 0),
+         "ask_sz": int(self._get_level_attr(record, "ask_sz", default=self._get_attr(record, "ask_sz_00", default=0)) or 0),
+      }
+
+   def _to_trade_row(self, record) -> dict | None:
+      price = self._get_attr(record, "price", "px")
+      size = self._get_attr(record, "size")
+      if price is None and size is None:
+         return None
+
+      side = self._get_attr(record, "side", "aggressor_side", default="")
+
+      return {
+         "id": 0,
+         "security_name": self._get_security_name(record),
+         "timestamp": self._get_timestamp(record),
+         "price": float(price or 0.0),
+         "size": int(size or 0),
+         "side": str(side),
+      }
+
+   def _get_security_name(self, record) -> str:
+      instrument_id = self._get_attr(record, "instrument_id", default=0)
+      try:
+         instrument_id_int = int(instrument_id or 0)
+      except (TypeError, ValueError):
+         instrument_id_int = 0
+
+      if instrument_id_int > 0 and instrument_id_int in self._instrument_id_to_symbol:
+         return self._instrument_id_to_symbol[instrument_id_int]
+
+      value = self._get_attr(
+         record,
+         "symbol",
+         "raw_symbol",
+         "stype_out_symbol",
+         "stype_in_symbol",
+         default=self._symbol,
+      )
+      return str(value)
+
+   def _get_timestamp(self, record) -> int:
+      value = self._get_attr(record, "ts_event", "ts_recv", default=0)
+      try:
+         return int(value)
+      except (TypeError, ValueError):
+         return 0
+
+   def _get_attr(self, record, *names, default=None):
+      for name in names:
+         if hasattr(record, name):
+            value = getattr(record, name)
+            if value is not None:
+               return value
+      return default
+
+   def _get_level_attr(self, record, name, default=None):
+      levels = self._get_attr(record, "levels", default=None)
+      if levels:
+         level0 = levels[0]
+         if hasattr(level0, name):
+            value = getattr(level0, name)
+            if value is not None:
+               return value
+      return default
+
+   def _ns_to_iso(self, timestamp_ns) -> str:
+      if timestamp_ns is None:
+         return ""
+
+      if hasattr(timestamp_ns, "isoformat"):
+         try:
+            if hasattr(timestamp_ns, "to_pydatetime"):
+               timestamp_dt = timestamp_ns.to_pydatetime()
+            else:
+               timestamp_dt = timestamp_ns
+
+            if timestamp_dt is None:
+               return ""
+
+            if getattr(timestamp_dt, "tzinfo", None) is None:
+               timestamp_dt = timestamp_dt.replace(tzinfo=dt.timezone.utc)
+            return timestamp_dt.isoformat()
+         except (TypeError, ValueError, AttributeError):
+            pass
+
+      try:
+         ts_int = int(timestamp_ns)
+         if ts_int <= 0:
+            return ""
+         return dt.datetime.fromtimestamp(ts_int / 1_000_000_000, tz=dt.timezone.utc).isoformat()
+      except (TypeError, ValueError, OSError, OverflowError):
+         return ""
+
+   def _nano_to_float(self, value) -> float:
+      try:
+         return float(value) / 1_000_000_000.0
+      except (TypeError, ValueError):
+         return 0.0

src/DataStreamSuite/DataStreamPublisher/WSPublishers/pubsub_client.py

@@ -0,0 +1,58 @@
+import asyncio
+import json
+import queue
+import threading
+
+import websockets
+
+
+class PubSubClient:
+   def __init__(self, ws_url: str):
+      self._ws_url = ws_url
+      self._outgoing_queue: "queue.Queue[dict]" = queue.Queue()
+      self._sender_thread: threading.Thread | None = None
+
+   def start(self) -> threading.Thread:
+      if self._sender_thread is None or not self._sender_thread.is_alive():
+         self._sender_thread = self._start_sender_thread()
+      return self._sender_thread
+
+   def publish(self, payload: dict) -> None:
+      self._outgoing_queue.put(payload)
+
+   def publish_create_table(self, table_name: str, rows: list[dict]) -> None:
+      self.publish({
+         "op": "create_table",
+         "table_name": table_name,
+         "rows": rows,
+      })
+
+   def publish_multiple_remove_add(
+      self,
+      table_name: str,
+      removed_rows: list[dict],
+      added_rows: list[dict],
+   ) -> None:
+      self.publish({
+         "op": "multiple_remove_add",
+         "table_name": table_name,
+         "removed_rows": removed_rows,
+         "added_rows": added_rows,
+      })
+
+   def _start_sender_thread(self) -> threading.Thread:
+      thread = threading.Thread(target=self._run_send_loop, daemon=True)
+      thread.start()
+      return thread
+
+   def _run_send_loop(self) -> None:
+      asyncio.run(self._ws_send_loop())
+
+   async def _ws_send_loop(self) -> None:
+      async with websockets.connect(self._ws_url) as ws:
+         print(f"Connected to PubSubServer websocket: {self._ws_url}")
+         while True:
+            payload = await asyncio.to_thread(self._outgoing_queue.get)
+            body = json.dumps(payload)
+            print(body)
+            await ws.send(body)

src/DataStreamSuite/DataStreamPublisher/appsettings.json

@@ -59,14 +59,9 @@
           "Class": "bts.PubsubServerStreamTagHandler"
         },
         {
-          "TagType": "System_Sub",
-          "Assembly": "PubSub",
-          "Class": "BCG.SystemSubscriptionMapPublisher"
-        },
-        {
-          "TagType": "System_Stats",
-          "Assembly": "PubSub",
-          "Class": "BCG.SystemStatsPublisher"
+          "TagType": "DS_.*",
+          "Assembly": "DataStreamCore",
+          "Class": "bts.PubsubServerStreamTagHandler"
         },
         {
           "TagType": ".*",
@@ -90,4 +85,4 @@
       }
     ]
   }
-}
+}

src/DataStreamSuite/DataStreamQueryAnalyzer/MainForm.cs

@@ -21,11 +21,55 @@ using System.Reactive.Subjects;
 using System.Text.RegularExpressions;
 using System.Threading;
 using System.Windows.Forms;
+using BlueOrb.Pubsub.Diagnostics;
 
 namespace BCG
 {
    public partial class MainForm : System.Windows.Forms.Form
    {
+      private const string LastQueryRegistryKey = "LastQuery";
+      private const string RecentSavedFilesRegistryKey = "RecentSavedFiles";
+      private const string DateTimeDisplayFormat = "M/d/yyyy h:mm:ss.fff tt";
+      private const int MaxRecentSavedFiles = 10;
+      private static readonly Color QueryEditorDefaultColor = Color.White;
+      private static readonly Color QueryEditorCommentColor = Color.FromArgb(106, 115, 125);
+      private static readonly Color QueryEditorOperatorColor = Color.FromArgb(86, 156, 214);
+      private static readonly Color QueryEditorAssignmentNameColor = Color.FromArgb(78, 201, 176);
+      private static readonly Color GridBackgroundColor = Color.FromArgb(37, 37, 38);
+      private static readonly Color GridAltRowColor = Color.FromArgb(43, 43, 48);
+      private static readonly Color GridHeaderColor = Color.FromArgb(62, 62, 66);
+      private static readonly Color GridGridLineColor = Color.FromArgb(58, 58, 61);
+      private static readonly Color GridSelectionColor = Color.FromArgb(9, 71, 113);
+      private static readonly Color GridSelectionForeColor = Color.White;
+      private static readonly Color GridTimestampColor = Color.FromArgb(206, 215, 222);
+      private static readonly Color GridIdentifierColor = Color.FromArgb(156, 163, 175);
+      private static readonly Color GridIndicatorColor = Color.FromArgb(156, 220, 254);
+
+      class PrimitiveTreeNodeComparer : IComparer
+      {
+         public int Compare(object? x, object? y)
+         {
+            var left = x as TreeNode;
+            var right = y as TreeNode;
+            if (left is null || right is null)
+               return 0;
+
+            string left_key = _sort_key(left);
+            string right_key = _sort_key(right);
+            return StringComparer.OrdinalIgnoreCase.Compare(left_key, right_key);
+         }
+
+         static string _sort_key(TreeNode node)
+         {
+            if (node.Parent is null)
+               return node.Text;
+
+            if (node.Tag is string primitive_name)
+               return primitive_name;
+
+            return node.Text;
+         }
+      }
       // static async Task _run() {
       //    var login = "bts";
       //    var passwd = String.Empty;
@@ -40,6 +84,8 @@ namespace BCG
       private string? myLastFileName;
       private ViewStream? myCurrentStream;
       private List<ToolStripMenuItem> mySiteItems = new List<ToolStripMenuItem>();
+      private readonly List<string> _recent_saved_files = new List<string>();
+      private ToolStripSeparator? _recent_saved_files_separator;
       // private string mySite = SitesCore.getMySiteName ();
       private string mySite = "TODO";
       private string myLastClassName = "MyView";
@@ -49,11 +95,20 @@ namespace BCG
       private TextWriter? myConsoleOutput;
       IDataPacket? pendingData;
       IDisposable? titleSub;
+      readonly ToolTip _primitive_tool_tip = new ToolTip();
+      readonly Dictionary<string, string?> _primitive_source_path_cache = new Dictionary<string, string?>();
+      Dictionary<string, string?>? _source_file_index;
+      string? _repo_root_path;
+      char? _last_primitive_search_char;
+      TreeNode? _last_primitive_search_assembly_node;
+      bool _is_applying_query_syntax_highlighting;
+      HashSet<string>? _dsl_operator_names;
 
       public MainForm()
       {
          //Task.Run(async () => await _run());
          InitializeComponent();
+         _apply_grid_theme();
          // myKQ = ConfigurationRequester.getApplicationQualifier();         
          myKQ = "TODO";
          var handleCreatedSubject = new Subject<bool>();
@@ -72,13 +127,12 @@ namespace BCG
          myWindowState = new PersistWindowState();
          myWindowState.parent = this;
          myWindowState.saveState = false;
+         myWindowState.LoadState += myLoadPersistedState;
+         myWindowState.SaveState += mySavePersistedState;
 
          this.Load += new EventHandler(myLoad);
          this.Closing += new CancelEventHandler(myClosing);
-         myQueryTextBox.TextChanged += (o, e) =>
-         {
-            myQueryTextBox.ForeColor = Color.White;
-         };
+         myQueryTextBox.TextChanged += (o, e) => _apply_query_syntax_highlighting();
       }
 
       event Action? titleInvalidated;
@@ -86,14 +140,20 @@ namespace BCG
       public string query_text
       {
          get { return myQueryTextBox.Text; }
-         set { myQueryTextBox.Text = value; }
+         set
+         {
+            myQueryTextBox.Text = value;
+            _apply_query_syntax_highlighting();
+         }
       }
 
       /// <summary>
       /// Clean up any resources being used.
       /// </summary>
-      protected override void Dispose(bool disposing) {
-         if (disposing) {
+      protected override void Dispose(bool disposing)
+      {
+         if (disposing)
+         {
             Disposable.Dispose(ref titleSub);
             components?.Dispose();
          }
@@ -104,7 +164,7 @@ namespace BCG
       {
          // TODO: Port PublishedTagsStream from edge (then we can uncomment impl below)
          //ViewStream vs = new PubSubServerStateStream();
-         ViewStream vs = new PubSubServerStateStream(new SubscriberCoreFactory(), ViewStreamDatabase.instance);
+         ViewStream vs = new PubSubServerStateStream(ViewStreamDatabase.instance);
          return new FilterStream(vs, "tag~='DS_'");
          //return new LiteralStream("tag/String", "DS_table1;DS_table2");
       }
@@ -114,7 +174,8 @@ namespace BCG
          titleInvalidated?.Invoke();
       }
 
-      void refreshTitle(IBlueRowCollection rows) {
+      void refreshTitle(IBlueRowCollection rows)
+      {
          string route_txt = String.Empty;
          foreach (IBlueRow r in rows)
          {
@@ -135,11 +196,11 @@ namespace BCG
       private IDictionary myGetLibraryData()
       {
          SortedList retVal = new SortedList();
-         string[] primitiveNames = PrimitiveManager.instance.getPrimitiveNames();
+         string[] primitiveNames = PrimitiveManager.instance.get_dsl_op_names();
          foreach (string pName in primitiveNames)
          {
-            string aName = PrimitiveManager.instance.getAssemblyName(pName);
-            ICollection cNames = PrimitiveManager.instance.getConstructorSigns(pName);
+            string aName = PrimitiveManager.instance.get_dslop_assembly_name(pName);
+            ICollection cNames = PrimitiveManager.instance.get_dsl_op_cons_args(pName);
             if (!retVal.Contains(aName))
                retVal.Add(aName, new SortedList());
             (retVal[aName] as IDictionary)[pName] = new ArrayList(cNames);
@@ -151,6 +212,65 @@ namespace BCG
          return retVal;
       }
 
+      HashSet<string> _get_dsl_operator_names()
+      {
+         if (_dsl_operator_names != null)
+            return _dsl_operator_names;
+
+         _dsl_operator_names = new HashSet<string>(
+            PrimitiveManager.instance.get_dsl_op_names().Cast<string>(),
+            StringComparer.OrdinalIgnoreCase);
+         return _dsl_operator_names;
+      }
+
+      void _apply_query_syntax_highlighting()
+      {
+         if (_is_applying_query_syntax_highlighting || myQueryTextBox.IsDisposed)
+            return;
+
+         _is_applying_query_syntax_highlighting = true;
+         try
+         {
+            int selection_start = myQueryTextBox.SelectionStart;
+            int selection_length = myQueryTextBox.SelectionLength;
+
+            myQueryTextBox.SuspendLayout();
+            myQueryTextBox.SelectAll();
+            myQueryTextBox.SelectionColor = QueryEditorDefaultColor;
+
+            foreach (Match match in Regex.Matches(myQueryTextBox.Text, @"^\s*([A-Za-z_]\w*)\s*=", RegexOptions.Multiline))
+            {
+               Group group = match.Groups[1];
+               myQueryTextBox.Select(group.Index, group.Length);
+               myQueryTextBox.SelectionColor = QueryEditorAssignmentNameColor;
+            }
+
+            foreach (Match match in Regex.Matches(myQueryTextBox.Text, @"\b([A-Za-z_]\w*)\s*\("))
+            {
+               Group group = match.Groups[1];
+               if (!_get_dsl_operator_names().Contains(group.Value))
+                  continue;
+               myQueryTextBox.Select(group.Index, group.Length);
+               myQueryTextBox.SelectionColor = QueryEditorOperatorColor;
+            }
+
+            foreach (Match match in Regex.Matches(myQueryTextBox.Text, @"//.*$", RegexOptions.Multiline))
+            {
+               myQueryTextBox.Select(match.Index, match.Length);
+               myQueryTextBox.SelectionColor = QueryEditorCommentColor;
+            }
+
+            myQueryTextBox.Select(selection_start, selection_length);
+            if (selection_length == 0)
+               myQueryTextBox.SelectionColor = QueryEditorDefaultColor;
+            myQueryTextBox.ResumeLayout();
+         }
+         finally
+         {
+            _is_applying_query_syntax_highlighting = false;
+         }
+      }
+
 
       private void myLoadAssemblies()
       {
@@ -161,13 +281,18 @@ namespace BCG
             TreeNode assemblyNode = new TreeNode(de.Key as String);
             foreach (DictionaryEntry innerDe in (de.Value as IDictionary))
             {
-               TreeNode primitiveNode = new TreeNode(innerDe.Key as string);
+               string primitive_name = innerDe.Key as string;
+               TreeNode primitiveNode = new TreeNode(primitive_name)
+               {
+                  Tag = primitive_name
+               };
                foreach (string con in (innerDe.Value as ICollection))
                   primitiveNode.Nodes.Add(con);
                assemblyNode.Nodes.Add(primitiveNode);
             }
             myTreeView.Nodes.Add(assemblyNode);
          }
+         myTreeView.Sort();
          /*
          myPrimitivesTextBox.Clear ();
          ArrayList al = new ArrayList(PrimitiveManager.instance.getPrimitiveNames());
@@ -182,7 +307,8 @@ namespace BCG
 
       private void myHandleTableListChange(object o)
       {
-         BeginInvoke(() => {
+         BeginInvoke(() =>
+         {
             tagRows = ((IDataPacket)o).newRows;
             myUpdateTableList();
          });
@@ -195,6 +321,19 @@ namespace BCG
          get { return myQueryTextBox.Height; }
          set { myQueryTextBox.Height = value; }
       }
+
+      private void myLoadPersistedState(object sender, BlueRegistry blueReg)
+      {
+         myQueryTextBox.Text = blueReg.getValue(RegistryRoot.HKCU, LastQueryRegistryKey, string.Empty)?.ToString() ?? string.Empty;
+         _load_recent_saved_files(blueReg.getValue(RegistryRoot.HKCU, RecentSavedFilesRegistryKey, string.Empty)?.ToString());
+         _refresh_recent_saved_files_menu();
+      }
+
+      private void mySavePersistedState(object sender, BlueRegistry blueReg)
+      {
+         blueReg.setValue(RegistryRoot.HKCU, LastQueryRegistryKey, myQueryTextBox.Text ?? string.Empty);
+         blueReg.setValue(RegistryRoot.HKCU, RecentSavedFilesRegistryKey, string.Join("|", _recent_saved_files));
+      }
       #endregion
 
       private Stopwatch _stopwatch = new Stopwatch();
@@ -287,6 +426,7 @@ namespace BCG
             }
             myLastFileName = fileName;
             this.Text = fileName;
+            _track_recent_saved_file(fileName);
          }
          catch (Exception exc)
          {
@@ -371,6 +511,122 @@ namespace BCG
          }
       }
 
+      private void myApplyDateTimeFormatting(IBlueTableSchema schema)
+      {
+         var dateTimeColumns = new HashSet<string>(
+            schema.columnInfo
+               .Where(ci => ci.colType == typeof(DateTime))
+               .Select(ci => ci.colName));
+
+         foreach (DataGridViewColumn column in myDataGrid.Columns)
+         {
+            if (!dateTimeColumns.Contains(column.DataPropertyName) && !dateTimeColumns.Contains(column.Name))
+               continue;
+
+            column.DefaultCellStyle.Format = DateTimeDisplayFormat;
+            column.Width = Math.Max(column.Width, 220);
+         }
+      }
+
+      void _apply_grid_theme()
+      {
+         myDataGrid.BackgroundColor = GridBackgroundColor;
+         myDataGrid.BorderStyle = BorderStyle.None;
+         myDataGrid.EnableHeadersVisualStyles = false;
+         myDataGrid.GridColor = GridGridLineColor;
+         myDataGrid.CellBorderStyle = DataGridViewCellBorderStyle.SingleHorizontal;
+         myDataGrid.SelectionMode = DataGridViewSelectionMode.FullRowSelect;
+         myDataGrid.MultiSelect = false;
+         myDataGrid.RowHeadersVisible = false;
+         myDataGrid.ColumnHeadersBorderStyle = DataGridViewHeaderBorderStyle.Single;
+         myDataGrid.RowHeadersBorderStyle = DataGridViewHeaderBorderStyle.None;
+         myDataGrid.ColumnHeadersHeight = 36;
+         myDataGrid.RowTemplate.Height = 24;
+         myDataGrid.Font = new Font("Consolas", 9F, FontStyle.Regular, GraphicsUnit.Point);
+
+         myDataGrid.DefaultCellStyle.BackColor = GridBackgroundColor;
+         myDataGrid.DefaultCellStyle.ForeColor = Color.White;
+         myDataGrid.DefaultCellStyle.SelectionBackColor = GridSelectionColor;
+         myDataGrid.DefaultCellStyle.SelectionForeColor = GridSelectionForeColor;
+         myDataGrid.DefaultCellStyle.Padding = new Padding(4, 1, 4, 1);
+
+         myDataGrid.AlternatingRowsDefaultCellStyle.BackColor = GridAltRowColor;
+         myDataGrid.AlternatingRowsDefaultCellStyle.ForeColor = Color.White;
+         myDataGrid.AlternatingRowsDefaultCellStyle.SelectionBackColor = GridSelectionColor;
+         myDataGrid.AlternatingRowsDefaultCellStyle.SelectionForeColor = GridSelectionForeColor;
+
+         myDataGrid.ColumnHeadersDefaultCellStyle.BackColor = GridHeaderColor;
+         myDataGrid.ColumnHeadersDefaultCellStyle.ForeColor = Color.White;
+         myDataGrid.ColumnHeadersDefaultCellStyle.SelectionBackColor = GridHeaderColor;
+         myDataGrid.ColumnHeadersDefaultCellStyle.SelectionForeColor = Color.White;
+         myDataGrid.ColumnHeadersDefaultCellStyle.Font = new Font("Consolas", 9F, FontStyle.Bold, GraphicsUnit.Point);
+         myDataGrid.ColumnHeadersDefaultCellStyle.Alignment = DataGridViewContentAlignment.MiddleLeft;
+         myDataGrid.ColumnHeadersDefaultCellStyle.Padding = new Padding(6, 2, 6, 2);
+      }
+
+      void _apply_grid_column_styling(IBlueTableSchema schema)
+      {
+         var column_types = schema.columnInfo.ToDictionary(ci => ci.colName, ci => ci.colType, StringComparer.OrdinalIgnoreCase);
+         foreach (DataGridViewColumn column in myDataGrid.Columns)
+         {
+            if (!column_types.TryGetValue(column.DataPropertyName, out var col_type) &&
+                !column_types.TryGetValue(column.Name, out col_type))
+            {
+               continue;
+            }
+
+            column.SortMode = DataGridViewColumnSortMode.NotSortable;
+
+            if (_is_numeric_type(col_type))
+            {
+               column.DefaultCellStyle.Alignment = DataGridViewContentAlignment.MiddleRight;
+            }
+            else
+            {
+               column.DefaultCellStyle.Alignment = DataGridViewContentAlignment.MiddleLeft;
+            }
+
+            if (col_type == typeof(DateTime))
+            {
+               column.DefaultCellStyle.ForeColor = GridTimestampColor;
+            }
+            else if (string.Equals(column.HeaderText, "id", StringComparison.OrdinalIgnoreCase))
+            {
+               column.DefaultCellStyle.ForeColor = GridIdentifierColor;
+            }
+            else if (_looks_like_indicator(column.HeaderText))
+            {
+               column.DefaultCellStyle.ForeColor = GridIndicatorColor;
+            }
+            else
+            {
+               column.DefaultCellStyle.ForeColor = Color.White;
+            }
+         }
+
+         static bool _is_numeric_type(Type type)
+         {
+            return type == typeof(int) ||
+                   type == typeof(long) ||
+                   type == typeof(float) ||
+                   type == typeof(double) ||
+                   type == typeof(decimal) ||
+                   type == typeof(short) ||
+                   type == typeof(uint) ||
+                   type == typeof(ulong);
+         }
+
+         static bool _looks_like_indicator(string column_name)
+         {
+            return column_name.Contains("rsi", StringComparison.OrdinalIgnoreCase) ||
+                   column_name.Contains("macd", StringComparison.OrdinalIgnoreCase) ||
+                   column_name.Contains("atr", StringComparison.OrdinalIgnoreCase) ||
+                   column_name.Contains("moving_average", StringComparison.OrdinalIgnoreCase) ||
+                   column_name.Contains("signal", StringComparison.OrdinalIgnoreCase) ||
+                   column_name.Contains("histogram", StringComparison.OrdinalIgnoreCase);
+         }
+      }
+
       // this is the stream callback
       private void myCallback(object o)
       {
@@ -386,7 +642,8 @@ namespace BCG
       private void myTimer_Tick(object sender, System.EventArgs e)
       {
          var dp = Interlocked.Exchange(ref pendingData, null);
-         if (dp is null) {
+         if (dp is null)
+         {
             return;
          }
          _stopwatch.Stop();
@@ -403,6 +660,10 @@ namespace BCG
          int hscrollval = myChangeStyle ? 0 : myDataGrid.HorizScrollBarValue;
 
          myDataGrid.DataSource = new SimpleBindableCollection(dp.newRows.schema, dp.newRows);
+         _apply_grid_theme();
+         myApplyDateTimeFormatting(dp.newRows.schema);
+         _apply_grid_column_styling(dp.newRows.schema);
+         myDataGrid.ClearSelection();
          var force = false;
          if (myChangeStyle && force)
          {
@@ -475,7 +736,7 @@ namespace BCG
       private void myMenuItemOpenClick(object sender, System.EventArgs e)
       {
          OpenFileDialog od = new OpenFileDialog();
-         od.Filter = "C# class (*.cs)|*.cs";
+         od.Filter = "All files (*.*)|*.*|C# class (*.cs)|*.cs";
 
          if (od.ShowDialog() == DialogResult.OK)
             loadFile(od.FileName);
@@ -487,6 +748,7 @@ namespace BCG
             w.Write(myCSharpText.Text);
          myLastFileName = fileName;
          this.Text = fileName;
+         _track_recent_saved_file(fileName);
       }
 
       private void myMenuItemSaveClick(object sender, System.EventArgs e)
@@ -509,6 +771,75 @@ namespace BCG
             mySave(sd.FileName);
       }
 
+      private void _load_recent_saved_files(string? serialized_paths)
+      {
+         _recent_saved_files.Clear();
+         if (string.IsNullOrWhiteSpace(serialized_paths))
+            return;
+
+         foreach (string path in serialized_paths.Split('|'))
+         {
+            if (string.IsNullOrWhiteSpace(path))
+               continue;
+            if (_recent_saved_files.Contains(path, StringComparer.OrdinalIgnoreCase))
+               continue;
+            _recent_saved_files.Add(path);
+            if (_recent_saved_files.Count >= MaxRecentSavedFiles)
+               break;
+         }
+      }
+
+      private void _track_recent_saved_file(string file_name)
+      {
+         _recent_saved_files.RemoveAll(path => string.Equals(path, file_name, StringComparison.OrdinalIgnoreCase));
+         _recent_saved_files.Insert(0, file_name);
+         if (_recent_saved_files.Count > MaxRecentSavedFiles)
+            _recent_saved_files.RemoveRange(MaxRecentSavedFiles, _recent_saved_files.Count - MaxRecentSavedFiles);
+         _refresh_recent_saved_files_menu();
+      }
+
+      private void _refresh_recent_saved_files_menu()
+      {
+         var file_menu = myToolStripMenuItemFile.DropDownItems;
+         for (int i = file_menu.Count - 1; i >= 0; i--)
+         {
+            if (file_menu[i] is ToolStripMenuItem menu_item && Equals(menu_item.Tag, "recent-saved-file"))
+               file_menu.RemoveAt(i);
+         }
+
+         if (_recent_saved_files_separator != null)
+         {
+            file_menu.Remove(_recent_saved_files_separator);
+            _recent_saved_files_separator.Dispose();
+            _recent_saved_files_separator = null;
+         }
+
+         if (_recent_saved_files.Count == 0)
+            return;
+
+         int insert_index = file_menu.IndexOf(myToolStripMenuItemExit);
+         if (insert_index < 0)
+            insert_index = file_menu.Count;
+
+         _recent_saved_files_separator = new ToolStripSeparator();
+         file_menu.Insert(insert_index, _recent_saved_files_separator);
+         insert_index++;
+
+         foreach (string path in _recent_saved_files)
+         {
+            var menu_item = new ToolStripMenuItem(Path.GetFileName(path))
+            {
+               Tag = "recent-saved-file",
+               ToolTipText = path,
+               BackColor = Color.FromArgb(36, 36, 49),
+               ForeColor = Color.Lime
+            };
+            menu_item.Click += (_, __) => loadFile(path);
+            file_menu.Insert(insert_index, menu_item);
+            insert_index++;
+         }
+      }
+
       private void myTableListBox_DoubleClick(object sender, System.EventArgs e)
       {
          if (myTableListBox.SelectedItem != null)
@@ -599,6 +930,7 @@ namespace BCG
          }
          Console.WriteLine("Redirecting console ouput to this window...");
 
+         myTreeView.TreeViewNodeSorter = new PrimitiveTreeNodeComparer();
          myLoadAssemblies();
 
          // ViewStreamDatabase.instance = new ViewStreamDatabase();
@@ -606,6 +938,9 @@ namespace BCG
          myAllTagsStream.onChange += new BCG.DataStream.ChangeDelegate(myHandleTableListChange);
          myTabControl.SelectedIndex = 3;
          myTreeView.AfterSelect += new TreeViewEventHandler(myHandleTreeNodeSelect);
+         myTreeView.KeyPress += myHandleTreeViewKeyPress;
+         myTreeView.NodeMouseHover += myHandleTreeNodeMouseHover;
+         myTreeView.MouseLeave += myHandleTreeViewMouseLeave;
          //do this lazily so we don't have to have bps, etc up to run dsqa
          //_library = new LibraryForm(this);
       }
@@ -646,12 +981,138 @@ namespace BCG
          {
             if (myTreeView.SelectedNode.LastNode == null)
             {
-               string sn = myTreeView.SelectedNode.Parent.Text + "()";
+               string primitive_name = myTreeView.SelectedNode.Parent.Tag as string ?? myTreeView.SelectedNode.Parent.Text;
+               string sn = primitive_name + "()";
                myQueryTextBox.Text = myQueryTextBox.Text == "" ? sn : myQueryTextBox.Text + "\n" + sn;
             }
          }
       }
 
+      private void myHandleTreeViewKeyPress(object? sender, KeyPressEventArgs e)
+      {
+         if (!Char.IsLetterOrDigit(e.KeyChar))
+            return;
+
+         char search_char = Char.ToLowerInvariant(e.KeyChar);
+         TreeNode? assembly_node = _get_selected_assembly_node();
+         if (assembly_node is null)
+            return;
+
+         int start_idx = 0;
+         if (_last_primitive_search_char == search_char &&
+             ReferenceEquals(_last_primitive_search_assembly_node, assembly_node) &&
+             myTreeView.SelectedNode?.Parent == assembly_node)
+         {
+            start_idx = myTreeView.SelectedNode.Index + 1;
+         }
+
+         TreeNode? match_node = _find_matching_primitive_node(assembly_node, search_char, start_idx) ??
+                                _find_matching_primitive_node(assembly_node, search_char, 0);
+
+         if (match_node is not null)
+         {
+            myTreeView.SelectedNode = match_node;
+            match_node.EnsureVisible();
+            _last_primitive_search_char = search_char;
+            _last_primitive_search_assembly_node = assembly_node;
+            e.Handled = true;
+         }
+      }
+
+      private TreeNode? _get_selected_assembly_node()
+      {
+         if (myTreeView.SelectedNode is null)
+            return myTreeView.Nodes.Count > 0 ? myTreeView.Nodes[0] : null;
+
+         return myTreeView.SelectedNode.Parent ?? myTreeView.SelectedNode;
+      }
+
+      private TreeNode? _find_matching_primitive_node(TreeNode assembly_node, char search_char, int start_idx)
+      {
+         for (int i = start_idx; i < assembly_node.Nodes.Count; i++)
+         {
+            TreeNode primitive_node = assembly_node.Nodes[i];
+            string primitive_name = primitive_node.Tag as string ?? primitive_node.Text;
+            if (primitive_name.Length > 0 && Char.ToLowerInvariant(primitive_name[0]) == search_char)
+               return primitive_node;
+         }
+         return null;
+      }
+
+      private void myHandleTreeNodeMouseHover(object? sender, TreeNodeMouseHoverEventArgs e)
+      {
+         if (e.Node?.Tag is not string primitive_name)
+         {
+            _primitive_tool_tip.Hide(myTreeView);
+            return;
+         }
+
+         Type? primitive_type = PrimitiveManager.instance.get_dsl_op_type(primitive_name);
+         if (primitive_type is null)
+         {
+            _primitive_tool_tip.Hide(myTreeView);
+            return;
+         }
+
+         string tooltip_text = $"Namespace: {primitive_type.Namespace ?? "(none)"}";
+         string? source_path = _get_source_file_path(primitive_type);
+         if (!String.IsNullOrWhiteSpace(source_path))
+            tooltip_text += $"\nSource: {source_path}";
+
+         Point mouse_point = myTreeView.PointToClient(Cursor.Position);
+         _primitive_tool_tip.Show(tooltip_text, myTreeView, mouse_point.X + 12, mouse_point.Y + 12, 8000);
+      }
+
+      private void myHandleTreeViewMouseLeave(object? sender, EventArgs e)
+      {
+         _primitive_tool_tip.Hide(myTreeView);
+      }
+
+      private string? _get_source_file_path(Type primitive_type)
+      {
+         string cache_key = primitive_type.FullName ?? primitive_type.Name;
+         if (_primitive_source_path_cache.TryGetValue(cache_key, out string? cached_path))
+            return cached_path;
+
+         Dictionary<string, string?> source_file_index = _source_file_index ??= _build_source_file_index();
+         if (!source_file_index.TryGetValue(primitive_type.Name, out string? source_path))
+            return null;
+         _primitive_source_path_cache[cache_key] = source_path;
+         return source_path;
+      }
+
+      private Dictionary<string, string?> _build_source_file_index()
+      {
+         Dictionary<string, string?> source_file_index = new Dictionary<string, string?>(StringComparer.OrdinalIgnoreCase);
+         string? repo_root = _repo_root_path ??= _find_repo_root();
+         if (String.IsNullOrWhiteSpace(repo_root))
+            return source_file_index;
+
+         string source_root = Path.Combine(repo_root, "src", "DataStreamSuite");
+         if (!Directory.Exists(source_root))
+            return source_file_index;
+
+         foreach (string file_path in Directory.EnumerateFiles(source_root, "*.cs", SearchOption.AllDirectories))
+         {
+            string type_name = Path.GetFileNameWithoutExtension(file_path);
+            if (!source_file_index.ContainsKey(type_name))
+               source_file_index[type_name] = file_path;
+         }
+         return source_file_index;
+      }
+
+      private string? _find_repo_root()
+      {
+         string? curr_path = AppDomain.CurrentDomain.BaseDirectory;
+         while (!String.IsNullOrWhiteSpace(curr_path))
+         {
+            if (Directory.Exists(Path.Combine(curr_path, "src", "DataStreamSuite")))
+               return curr_path;
+            curr_path = Directory.GetParent(curr_path)?.FullName;
+         }
+         return null;
+      }
+
       private void myToolStripMenuItemExit_Click(object sender, EventArgs e)
       {
          Close();
@@ -681,7 +1142,7 @@ namespace BCG
          DialogResult dr = ofd.ShowDialog(this);
          if (ofd.CheckFileExists && ofd.CheckPathExists && dr == DialogResult.OK)
          {
-            PrimitiveManager.instance.loadAssembly(ofd.FileName);
+            PrimitiveManager.instance.load_streams_from_assembly(ofd.FileName);
             myLoadAssemblies();
          }
       }
@@ -768,7 +1229,8 @@ namespace BCG
          s = new MutableAppendStream(s, s);
          s.onChange += OnChange;
 
-         void OnChange(object? obj) {
+         void OnChange(object? obj)
+         {
             s.onChange -= OnChange;
             //s = new MutableAppendStream(s, s);
             var rowCollection = ((IDataPacket)obj!).newRows;

src/DataStreamSuite/DataStreamQueryAnalyzer/appsettings.json

@@ -5,8 +5,7 @@
   "ViewStreamDatabase": {
     "Interval": "00:00:00.1"
   },
-  "Spark": {
-  },
+  "Spark": {},
   "SparkApi": {
     "IsEnabled": false,
     "Credentials": {
@@ -53,21 +52,16 @@
           "Class": "bts.PubsubServerStreamTagHandler"
         },
         {
-          "TagType": "System_Sub",
-          "Assembly": "PubSub",
-          "Class": "BCG.SystemSubscriptionMapPublisher"
-        },
-        {
-          "TagType": "System_Stats",
-          "Assembly": "PubSub",
-          "Class": "BCG.SystemStatsPublisher"
+          "TagType": "DS_.*",
+          "Assembly": "DataStreamCore",
+          "Class": "bts.PubsubServerStreamTagHandler"
         },
         {
           "TagType": ".*",
           "Assembly": "DataStreamCore",
-         "Class": "bts.PubsubServerGenericDataHandler"
+          "Class": "bts.PubsubServerGenericDataHandler"
         }
       ]
     }
   }
-}
+}

src/DataStreamSuite/DataStreamSuite.sln

@@ -13,6 +13,8 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "PubSub", "PubSub\PubSub.csp
 EndProject
 Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "PubSubServer", "PubSubServer\PubSubServer.csproj", "{0DBD814A-E17F-4836-8EEE-6E91B00C578F}"
 EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "PersistenceServer", "PersistenceServer\PersistenceServer.csproj", "{915A604A-C4B0-4C2B-A70D-D5E0C7AA2B80}"
+EndProject
 Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "BooleanExpressions", "BooleanExpressions\BooleanExpressions.csproj", "{EE53687E-BB62-45C6-A1B0-9B8F16B4E2C1}"
 EndProject
 Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Bts.Spark.Core", "..\cs\Bts.Spark.Core\Bts.Spark.Core.csproj", "{88E70136-91AE-4F56-B243-8F4642B46D6A}"
@@ -43,6 +45,10 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Bts.Net", "..\cs\Bts.Net\Bt
 EndProject
 Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Bts.IO.Compression", "..\cs\Bts.IO.Compression\Bts.IO.Compression.csproj", "{3BF059B0-51C0-44A8-8297-9B7464DC5BA3}"
 EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "DSPublisher", "DSPublisher\DSPublisher.csproj", "{72A4C31D-1680-4E8F-AB5E-A33B3D095DD3}"
+EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "APIBridgeServer", "APIBridgeServer\APIBridgeServer.csproj", "{3B14C417-531A-4E39-80C1-A4C14BD000D5}"
+EndProject
 Global
 	GlobalSection(SolutionConfigurationPlatforms) = preSolution
 		Debug|Any CPU = Debug|Any CPU
@@ -113,6 +119,18 @@ Global
 		{0DBD814A-E17F-4836-8EEE-6E91B00C578F}.Release|x64.Build.0 = Release|Any CPU
 		{0DBD814A-E17F-4836-8EEE-6E91B00C578F}.Release|x86.ActiveCfg = Release|Any CPU
 		{0DBD814A-E17F-4836-8EEE-6E91B00C578F}.Release|x86.Build.0 = Release|Any CPU
+		{915A604A-C4B0-4C2B-A70D-D5E0C7AA2B80}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{915A604A-C4B0-4C2B-A70D-D5E0C7AA2B80}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{915A604A-C4B0-4C2B-A70D-D5E0C7AA2B80}.Debug|x64.ActiveCfg = Debug|Any CPU
+		{915A604A-C4B0-4C2B-A70D-D5E0C7AA2B80}.Debug|x64.Build.0 = Debug|Any CPU
+		{915A604A-C4B0-4C2B-A70D-D5E0C7AA2B80}.Debug|x86.ActiveCfg = Debug|Any CPU
+		{915A604A-C4B0-4C2B-A70D-D5E0C7AA2B80}.Debug|x86.Build.0 = Debug|Any CPU
+		{915A604A-C4B0-4C2B-A70D-D5E0C7AA2B80}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{915A604A-C4B0-4C2B-A70D-D5E0C7AA2B80}.Release|Any CPU.Build.0 = Release|Any CPU
+		{915A604A-C4B0-4C2B-A70D-D5E0C7AA2B80}.Release|x64.ActiveCfg = Release|Any CPU
+		{915A604A-C4B0-4C2B-A70D-D5E0C7AA2B80}.Release|x64.Build.0 = Release|Any CPU
+		{915A604A-C4B0-4C2B-A70D-D5E0C7AA2B80}.Release|x86.ActiveCfg = Release|Any CPU
+		{915A604A-C4B0-4C2B-A70D-D5E0C7AA2B80}.Release|x86.Build.0 = Release|Any CPU
 		{EE53687E-BB62-45C6-A1B0-9B8F16B4E2C1}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
 		{EE53687E-BB62-45C6-A1B0-9B8F16B4E2C1}.Debug|Any CPU.Build.0 = Debug|Any CPU
 		{EE53687E-BB62-45C6-A1B0-9B8F16B4E2C1}.Debug|x64.ActiveCfg = Debug|Any CPU
@@ -293,6 +311,30 @@ Global
 		{3BF059B0-51C0-44A8-8297-9B7464DC5BA3}.Release|x64.Build.0 = Release|Any CPU
 		{3BF059B0-51C0-44A8-8297-9B7464DC5BA3}.Release|x86.ActiveCfg = Release|Any CPU
 		{3BF059B0-51C0-44A8-8297-9B7464DC5BA3}.Release|x86.Build.0 = Release|Any CPU
+		{72A4C31D-1680-4E8F-AB5E-A33B3D095DD3}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{72A4C31D-1680-4E8F-AB5E-A33B3D095DD3}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{72A4C31D-1680-4E8F-AB5E-A33B3D095DD3}.Debug|x64.ActiveCfg = Debug|Any CPU
+		{72A4C31D-1680-4E8F-AB5E-A33B3D095DD3}.Debug|x64.Build.0 = Debug|Any CPU
+		{72A4C31D-1680-4E8F-AB5E-A33B3D095DD3}.Debug|x86.ActiveCfg = Debug|Any CPU
+		{72A4C31D-1680-4E8F-AB5E-A33B3D095DD3}.Debug|x86.Build.0 = Debug|Any CPU
+		{72A4C31D-1680-4E8F-AB5E-A33B3D095DD3}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{72A4C31D-1680-4E8F-AB5E-A33B3D095DD3}.Release|Any CPU.Build.0 = Release|Any CPU
+		{72A4C31D-1680-4E8F-AB5E-A33B3D095DD3}.Release|x64.ActiveCfg = Release|Any CPU
+		{72A4C31D-1680-4E8F-AB5E-A33B3D095DD3}.Release|x64.Build.0 = Release|Any CPU
+		{72A4C31D-1680-4E8F-AB5E-A33B3D095DD3}.Release|x86.ActiveCfg = Release|Any CPU
+		{72A4C31D-1680-4E8F-AB5E-A33B3D095DD3}.Release|x86.Build.0 = Release|Any CPU
+		{3B14C417-531A-4E39-80C1-A4C14BD000D5}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{3B14C417-531A-4E39-80C1-A4C14BD000D5}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{3B14C417-531A-4E39-80C1-A4C14BD000D5}.Debug|x64.ActiveCfg = Debug|Any CPU
+		{3B14C417-531A-4E39-80C1-A4C14BD000D5}.Debug|x64.Build.0 = Debug|Any CPU
+		{3B14C417-531A-4E39-80C1-A4C14BD000D5}.Debug|x86.ActiveCfg = Debug|Any CPU
+		{3B14C417-531A-4E39-80C1-A4C14BD000D5}.Debug|x86.Build.0 = Debug|Any CPU
+		{3B14C417-531A-4E39-80C1-A4C14BD000D5}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{3B14C417-531A-4E39-80C1-A4C14BD000D5}.Release|Any CPU.Build.0 = Release|Any CPU
+		{3B14C417-531A-4E39-80C1-A4C14BD000D5}.Release|x64.ActiveCfg = Release|Any CPU
+		{3B14C417-531A-4E39-80C1-A4C14BD000D5}.Release|x64.Build.0 = Release|Any CPU
+		{3B14C417-531A-4E39-80C1-A4C14BD000D5}.Release|x86.ActiveCfg = Release|Any CPU
+		{3B14C417-531A-4E39-80C1-A4C14BD000D5}.Release|x86.Build.0 = Release|Any CPU
 	EndGlobalSection
 	GlobalSection(SolutionProperties) = preSolution
 		HideSolutionNode = FALSE

src/DataStreamSuite/DataStreamTests/DataStreamTests.csproj

@@ -27,6 +27,7 @@
     <ProjectReference Include="..\PubSub\PubSub.csproj" />
     <ProjectReference Include="..\DataStreamCore\DataStreamCore.csproj" />
     <ProjectReference Include="..\DataStream\DataStream.csproj" />
+    <ProjectReference Include="..\SharedStreams\SharedStreams.csproj" />
   </ItemGroup>
 
 

src/DataStreamSuite/DataStreamTests/HeikinAshiStreamTests.cs

@@ -0,0 +1,121 @@
+namespace Bts.DataStreamTests;
+
+using BCG;
+
+using BlueOrb.Operators.Signal;
+
+using Bts.DataStreamCore;
+
+[TestFixture]
+public class HeikinAshiStreamTests {
+   readonly BlueTableSchema schema = new(
+      new ColumnInfo[] {
+         new("id", 0, TypeCode.Int32),
+         new("ts_event", 1, TypeCode.DateTime),
+         new("open", 2, TypeCode.Double),
+         new("high", 3, TypeCode.Double),
+         new("low", 4, TypeCode.Double),
+         new("close", 5, TypeCode.Double),
+         new("volume", 6, TypeCode.Double),
+      }, "bars");
+
+   [Test]
+   public void AppendsHistoryAndCalculatesRecursiveHeikinAshiValues() {
+      var driver = new TestDriverStream(schema, null);
+      var heikinAshi = new HeikinAshiStream(driver);
+      var packets = new List<IDataPacket>();
+      heikinAshi.onChange += nv => packets.Add((IDataPacket)nv);
+
+      driver.Reset(
+         Row(1, new DateTime(2026, 1, 1, 9, 30, 0, DateTimeKind.Utc), 100.0, 110.0, 90.0, 105.0, 10.0),
+         Row(2, new DateTime(2026, 1, 1, 9, 31, 0, DateTimeKind.Utc), 106.0, 112.0, 104.0, 110.0, 20.0)
+      );
+
+      Assert.That(packets, Has.Count.EqualTo(2));
+
+      Assert.That(packets[0].newRows.Count, Is.EqualTo(1));
+      Assert.That(packets[0].addedRows.Count, Is.EqualTo(1));
+      Assert.That(packets[0].deletedRows.Count, Is.EqualTo(0));
+
+      var first = packets[0].newRows[0];
+      Assert.That(first.get<double>("open"), Is.EqualTo(100.0).Within(1e-9));
+      Assert.That(first.get<double>("high"), Is.EqualTo(110.0).Within(1e-9));
+      Assert.That(first.get<double>("low"), Is.EqualTo(90.0).Within(1e-9));
+      Assert.That(first.get<double>("close"), Is.EqualTo(105.0).Within(1e-9));
+      Assert.That(first.get<double>("ha_open"), Is.EqualTo(102.5).Within(1e-9));
+      Assert.That(first.get<double>("ha_high"), Is.EqualTo(110.0).Within(1e-9));
+      Assert.That(first.get<double>("ha_low"), Is.EqualTo(90.0).Within(1e-9));
+      Assert.That(first.get<double>("ha_close"), Is.EqualTo(101.25).Within(1e-9));
+
+      Assert.That(packets[1].newRows.Count, Is.EqualTo(2));
+      Assert.That(packets[1].addedRows.Count, Is.EqualTo(1));
+      Assert.That(packets[1].deletedRows.Count, Is.EqualTo(0));
+
+      var second = packets[1].newRows[1];
+      Assert.That(second.get<double>("open"), Is.EqualTo(106.0).Within(1e-9));
+      Assert.That(second.get<double>("high"), Is.EqualTo(112.0).Within(1e-9));
+      Assert.That(second.get<double>("low"), Is.EqualTo(104.0).Within(1e-9));
+      Assert.That(second.get<double>("close"), Is.EqualTo(110.0).Within(1e-9));
+      Assert.That(second.get<double>("ha_open"), Is.EqualTo(101.875).Within(1e-9));
+      Assert.That(second.get<double>("ha_high"), Is.EqualTo(112.0).Within(1e-9));
+      Assert.That(second.get<double>("ha_low"), Is.EqualTo(101.875).Within(1e-9));
+      Assert.That(second.get<double>("ha_close"), Is.EqualTo(108.0).Within(1e-9));
+      Assert.That(second.get<double>("volume"), Is.EqualTo(20.0).Within(1e-9));
+   }
+
+   [Test]
+   public void RebuildsHistoryWhenSourceSnapshotChanges() {
+      var driver = new TestDriverStream(schema, null);
+      var heikinAshi = new HeikinAshiStream(driver);
+      var packets = new List<IDataPacket>();
+      heikinAshi.onChange += nv => packets.Add((IDataPacket)nv);
+
+      driver.Reset(
+         Row(1, new DateTime(2026, 1, 1, 9, 30, 0, DateTimeKind.Utc), 100.0, 110.0, 90.0, 105.0, 10.0),
+         Row(2, new DateTime(2026, 1, 1, 9, 31, 0, DateTimeKind.Utc), 106.0, 112.0, 104.0, 110.0, 20.0)
+      );
+
+      driver.Reset(
+         Row(1, new DateTime(2026, 1, 1, 9, 30, 0, DateTimeKind.Utc), 100.0, 110.0, 90.0, 105.0, 10.0),
+         Row(2, new DateTime(2026, 1, 1, 9, 31, 0, DateTimeKind.Utc), 106.0, 114.0, 103.0, 111.0, 21.0)
+      );
+
+      Assert.That(packets, Has.Count.EqualTo(3));
+
+      Assert.That(packets[2].newRows.Count, Is.EqualTo(2));
+      Assert.That(packets[2].addedRows.Count, Is.EqualTo(1));
+      Assert.That(packets[2].deletedRows.Count, Is.EqualTo(1));
+
+      var updated = packets[2].newRows[1];
+      Assert.That(updated.get<double>("open"), Is.EqualTo(106.0).Within(1e-9));
+      Assert.That(updated.get<double>("high"), Is.EqualTo(114.0).Within(1e-9));
+      Assert.That(updated.get<double>("low"), Is.EqualTo(103.0).Within(1e-9));
+      Assert.That(updated.get<double>("close"), Is.EqualTo(111.0).Within(1e-9));
+      Assert.That(updated.get<double>("ha_open"), Is.EqualTo(101.875).Within(1e-9));
+      Assert.That(updated.get<double>("ha_high"), Is.EqualTo(114.0).Within(1e-9));
+      Assert.That(updated.get<double>("ha_low"), Is.EqualTo(101.875).Within(1e-9));
+      Assert.That(updated.get<double>("ha_close"), Is.EqualTo(108.5).Within(1e-9));
+      Assert.That(updated.get<double>("volume"), Is.EqualTo(21.0).Within(1e-9));
+   }
+
+   static IBlueRow Row(int id, DateTime tsEvent, double open, double high, double low, double close, double volume) {
+      var row = new MutableBlueRow(
+         new BlueTableSchema(new[] {
+            new ColumnInfo("id", 0, TypeCode.Int32),
+            new ColumnInfo("ts_event", 1, TypeCode.DateTime),
+            new ColumnInfo("open", 2, TypeCode.Double),
+            new ColumnInfo("high", 3, TypeCode.Double),
+            new ColumnInfo("low", 4, TypeCode.Double),
+            new ColumnInfo("close", 5, TypeCode.Double),
+            new ColumnInfo("volume", 6, TypeCode.Double),
+         }, "bars"));
+      row.setValue("id", id);
+      row.setValue("ts_event", tsEvent);
+      row.setValue("open", open);
+      row.setValue("high", high);
+      row.setValue("low", low);
+      row.setValue("close", close);
+      row.setValue("volume", volume);
+      return row;
+   }
+}

src/DataStreamSuite/DataStreamTests/VWAPStreamTests.cs

@@ -0,0 +1,53 @@
+namespace Bts.DataStreamTests;
+
+using BCG;
+
+using BlueOrb.Operators.Signal;
+
+using Bts.DataStreamCore;
+
+[TestFixture]
+public class VWAPStreamTests {
+   readonly BlueTableSchema schema = new(
+      new ColumnInfo[] {
+         new("id", 0, TypeCode.Int32),
+         new("ts_event", 1, TypeCode.DateTime),
+         new("close", 2, TypeCode.Double),
+         new("volume", 3, TypeCode.Double),
+      }, "bars");
+
+   [Test]
+   public void EmitsRowsAndCalculatesRollingVWAP() {
+      var driver = new TestDriverStream(schema, null);
+      var vwap = new VWAPStream(driver, "close", "volume", 2);
+      var packets = new List<IDataPacket>();
+      vwap.onChange += nv => packets.Add((IDataPacket)nv);
+
+      driver.Reset(
+         Row(1, new DateTime(2026, 1, 1, 9, 30, 0, DateTimeKind.Utc), 100.0, 10.0),
+         Row(2, new DateTime(2026, 1, 1, 9, 30, 5, DateTimeKind.Utc), 110.0, 20.0)
+      );
+
+      Assert.That(packets, Has.Count.EqualTo(1));
+      Assert.That(packets[0].newRows.Count, Is.EqualTo(2));
+
+      var rows = packets[0].newRows.to_list().OrderBy(r => r.get<int>("id")).ToList();
+      Assert.That(rows[0].get<double>("vwap"), Is.EqualTo(100.0).Within(1e-9));
+      Assert.That(rows[1].get<double>("vwap"), Is.EqualTo((100.0 * 10.0 + 110.0 * 20.0) / 30.0).Within(1e-9));
+   }
+
+   static IBlueRow Row(int id, DateTime tsEvent, double close, double volume) {
+      var row = new MutableBlueRow(
+         new BlueTableSchema(new[] {
+            new ColumnInfo("id", 0, TypeCode.Int32),
+            new ColumnInfo("ts_event", 1, TypeCode.DateTime),
+            new ColumnInfo("close", 2, TypeCode.Double),
+            new ColumnInfo("volume", 3, TypeCode.Double),
+         }, "bars"));
+      row.setValue("id", id);
+      row.setValue("ts_event", tsEvent);
+      row.setValue("close", close);
+      row.setValue("volume", volume);
+      return row;
+   }
+}

src/DataStreamSuite/PersistenceServer/ConfigLoader.cs

@@ -0,0 +1,33 @@
+using System;
+using System.IO;
+using System.Text.Json;
+
+public static class ConfigLoader {
+   public static PersistenceServerConfig load() {
+      var config = new PersistenceServerConfig();
+      merge_from_file(config, Path.Combine(AppContext.BaseDirectory, "appsettings.json"));
+      merge_from_file(config, Path.Combine(AppContext.BaseDirectory, "appsettings.Development.json"));
+      return config;
+   }
+
+   private static void merge_from_file(PersistenceServerConfig target, string path) {
+      if (!File.Exists(path)) {
+         return;
+      }
+
+      var json = File.ReadAllText(path);
+      var loaded = JsonSerializer.Deserialize<PersistenceServerConfig>(json, new JsonSerializerOptions {
+         PropertyNameCaseInsensitive = true
+      });
+
+      if (loaded?.pub_sub is null) {
+         return;
+      }
+
+      if (target.pub_sub is null) {
+         target.pub_sub = loaded.pub_sub;
+      } else {
+         target.pub_sub.MergeWith(loaded.pub_sub);
+      }
+   }
+}

src/DataStreamSuite/PersistenceServer/PeristenceHelper.cs

@@ -0,0 +1,148 @@
+using System;
+using System.Collections.Generic;
+using System.Data.SQLite;
+using System.IO;
+using System.Linq;
+
+public class PeristenceHelper {
+   private readonly string _connectionString;
+   public string database_path { get; }
+
+   public PeristenceHelper(string? databasePath = null) {
+      var resolvedPath = databasePath ?? Path.Combine(AppContext.BaseDirectory, "PersistenceServer.sqlite");
+      database_path = resolvedPath;
+      Directory.CreateDirectory(Path.GetDirectoryName(resolvedPath)!);
+      _connectionString = $"Data Source={resolvedPath};Version=3";
+   }
+
+   public void InsertRow(string tableName, IReadOnlyDictionary<string, string> row) {
+      ValidateRow(tableName, row);
+      using var connection = OpenConnection();
+      EnsureTable(connection, tableName, row.Keys);
+
+      var columns = row.Keys.Select(QuoteIdentifier).ToArray();
+      var parameterNames = row.Keys.Select(CreateParameterName).ToArray();
+      var commandText =
+         $"INSERT INTO {QuoteIdentifier(tableName)} ({string.Join(", ", columns)}) VALUES ({string.Join(", ", parameterNames)})";
+
+      using var command = new SQLiteCommand(commandText, connection);
+      AddParameters(command, row);
+      command.ExecuteNonQuery();
+   }
+
+   public void UpdateRow(string tableName, IReadOnlyDictionary<string, string> row) {
+      ValidateRow(tableName, row);
+      using var connection = OpenConnection();
+      EnsureTable(connection, tableName, row.Keys);
+
+      var keyColumns = GetKeyColumns(row);
+      var valueColumns = row.Keys.Where(column => !keyColumns.Contains(column, StringComparer.OrdinalIgnoreCase)).ToArray();
+      if (valueColumns.Length == 0) {
+         throw new InvalidOperationException("UpdateRow requires at least one non-key column to update.");
+      }
+
+      var setClause = string.Join(", ", valueColumns.Select(column => $"{QuoteIdentifier(column)} = {CreateParameterName(column)}"));
+      var whereClause = string.Join(" AND ", keyColumns.Select(column => $"{QuoteIdentifier(column)} = {CreateWhereParameterName(column)}"));
+      var commandText = $"UPDATE {QuoteIdentifier(tableName)} SET {setClause} WHERE {whereClause}";
+
+      using var command = new SQLiteCommand(commandText, connection);
+      foreach (var column in valueColumns) {
+         command.Parameters.AddWithValue(CreateParameterName(column), row[column]);
+      }
+
+      foreach (var column in keyColumns) {
+         command.Parameters.AddWithValue(CreateWhereParameterName(column), row[column]);
+      }
+
+      command.ExecuteNonQuery();
+   }
+
+   public void RemoveRow(string tableName, IReadOnlyDictionary<string, string> row) {
+      ValidateRow(tableName, row);
+      using var connection = OpenConnection();
+      EnsureTable(connection, tableName, row.Keys);
+
+      var keyColumns = GetKeyColumns(row);
+      var whereClause = string.Join(" AND ", keyColumns.Select(column => $"{QuoteIdentifier(column)} = {CreateWhereParameterName(column)}"));
+      var commandText = $"DELETE FROM {QuoteIdentifier(tableName)} WHERE {whereClause}";
+
+      using var command = new SQLiteCommand(commandText, connection);
+      foreach (var column in keyColumns) {
+         command.Parameters.AddWithValue(CreateWhereParameterName(column), row[column]);
+      }
+
+      command.ExecuteNonQuery();
+   }
+
+   private SQLiteConnection OpenConnection() {
+      var connection = new SQLiteConnection(_connectionString);
+      connection.Open();
+      return connection;
+   }
+
+   private static void ValidateRow(string tableName, IReadOnlyDictionary<string, string> row) {
+      if (string.IsNullOrWhiteSpace(tableName)) {
+         throw new ArgumentException("Table name is required.", nameof(tableName));
+      }
+
+      if (row.Count == 0) {
+         throw new ArgumentException("Row must contain at least one column.", nameof(row));
+      }
+   }
+
+   private static string[] GetKeyColumns(IReadOnlyDictionary<string, string> row) {
+      if (row.ContainsKey("namespace") && row.ContainsKey("key")) {
+         return new[] { "namespace", "key" };
+      }
+
+      if (row.ContainsKey("key")) {
+         return new[] { "key" };
+      }
+
+      throw new InvalidOperationException("Row must contain 'key', or both 'namespace' and 'key', for update/delete.");
+   }
+
+   private static void EnsureTable(SQLiteConnection connection, string tableName, IEnumerable<string> columns) {
+      var existingColumns = GetExistingColumns(connection, tableName);
+      if (existingColumns.Count == 0) {
+         var createColumns = string.Join(", ", columns.Select(column => $"{QuoteIdentifier(column)} TEXT"));
+         using var createCommand =
+            new SQLiteCommand($"CREATE TABLE IF NOT EXISTS {QuoteIdentifier(tableName)} ({createColumns})", connection);
+         createCommand.ExecuteNonQuery();
+         existingColumns = GetExistingColumns(connection, tableName);
+      }
+
+      foreach (var column in columns) {
+         if (existingColumns.Contains(column)) {
+            continue;
+         }
+
+         using var alterCommand =
+            new SQLiteCommand($"ALTER TABLE {QuoteIdentifier(tableName)} ADD COLUMN {QuoteIdentifier(column)} TEXT", connection);
+         alterCommand.ExecuteNonQuery();
+      }
+   }
+
+   private static HashSet<string> GetExistingColumns(SQLiteConnection connection, string tableName) {
+      var columns = new HashSet<string>(StringComparer.OrdinalIgnoreCase);
+      using var command = new SQLiteCommand($"PRAGMA table_info({QuoteIdentifier(tableName)})", connection);
+      using var reader = command.ExecuteReader();
+      while (reader.Read()) {
+         columns.Add(reader["name"].ToString()!);
+      }
+
+      return columns;
+   }
+
+   private static void AddParameters(SQLiteCommand command, IReadOnlyDictionary<string, string> row) {
+      foreach (var pair in row) {
+         command.Parameters.AddWithValue(CreateParameterName(pair.Key), pair.Value);
+      }
+   }
+
+   private static string QuoteIdentifier(string identifier) => $"\"{identifier.Replace("\"", "\"\"")}\"";
+
+   private static string CreateParameterName(string columnName) => $"@p_{columnName.Replace("\"", string.Empty)}";
+
+   private static string CreateWhereParameterName(string columnName) => $"@w_{columnName.Replace("\"", string.Empty)}";
+}

src/DataStreamSuite/PersistenceServer/PersistenceCallHandler.cs

@@ -0,0 +1,155 @@
+using System;
+using System.Collections.Generic;
+using System.Net;
+using System.Text.Json;
+
+public class PersistenceCallHandler {
+   private const string KeyValueTableName = "key_value_store";
+   private readonly PeristenceHelper _persistenceHelper;
+   private readonly PubSubSync _pub_sub_sync;
+
+   public PersistenceCallHandler(PeristenceHelper persistenceHelper, PubSubSync pub_sub_sync) {
+      _persistenceHelper = persistenceHelper;
+      _pub_sub_sync = pub_sub_sync;
+   }
+
+   public void Handle(HttpListenerRequest request, string payload) {
+      Console.WriteLine("----- Incoming REST call -----");
+      Console.WriteLine($"Timestamp: {DateTimeOffset.Now:O}");
+      Console.WriteLine($"Method: {request.HttpMethod}");
+      Console.WriteLine($"Url: {request.Url}");
+      Console.WriteLine($"RawUrl: {request.RawUrl}");
+      Console.WriteLine($"RemoteEndPoint: {request.RemoteEndPoint}");
+      Console.WriteLine($"UserAgent: {request.UserAgent}");
+      Console.WriteLine($"ContentType: {request.ContentType}");
+      Console.WriteLine($"ContentLength: {request.ContentLength64}");
+
+      if (request.QueryString.Count > 0) {
+         Console.WriteLine("Query:");
+         foreach (var key in request.QueryString.AllKeys) {
+            if (key is null) {
+               continue;
+            }
+
+            Console.WriteLine($"  {key} = {request.QueryString[key]}");
+         }
+      }
+
+      if (request.Headers.Count > 0) {
+         Console.WriteLine("Headers:");
+         foreach (var headerKey in request.Headers.AllKeys) {
+            if (headerKey is null) {
+               continue;
+            }
+
+            Console.WriteLine($"  {headerKey}: {request.Headers[headerKey]}");
+         }
+      }
+
+      Console.WriteLine("Body:");
+      Console.WriteLine(string.IsNullOrWhiteSpace(payload) ? "  <empty>" : payload);
+
+      var rpcRequest = ParseRpcRequest(payload);
+      HandleRpc(rpcRequest.FunctionName, rpcRequest.Arguments);
+      Console.WriteLine("------------------------------");
+   }
+
+   private static PersistenceRpcRequest ParseRpcRequest(string payload) {
+      if (string.IsNullOrWhiteSpace(payload)) {
+         throw new InvalidOperationException("RPC payload is empty.");
+      }
+
+      using var document = JsonDocument.Parse(payload);
+      var root = document.RootElement;
+
+      if (!root.TryGetProperty("function_name", out var functionNameElement) ||
+          functionNameElement.ValueKind != JsonValueKind.String) {
+         throw new InvalidOperationException("RPC payload must contain a string 'function_name'.");
+      }
+
+      if (!root.TryGetProperty("arguments", out var argumentsElement) ||
+          argumentsElement.ValueKind != JsonValueKind.Array) {
+         throw new InvalidOperationException("RPC payload must contain an 'arguments' array.");
+      }
+
+      var arguments = new List<string>();
+      foreach (var argument in argumentsElement.EnumerateArray()) {
+         if (argument.ValueKind != JsonValueKind.String) {
+            throw new InvalidOperationException("RPC arguments must all be strings.");
+         }
+
+         arguments.Add(argument.GetString()!);
+      }
+
+      return new PersistenceRpcRequest(functionNameElement.GetString()!, arguments.ToArray());
+   }
+
+   private void HandleRpc(string functionName, params string[] arguments) {
+      switch (functionName) {
+         case "keyvalue_insert":
+            RequireArgumentCount(functionName, arguments, 3);
+            HandleKeyValueInsert(arguments[0], arguments[1], arguments[2]);
+            break;
+         case "keyvalue_update":
+            RequireArgumentCount(functionName, arguments, 3);
+            HandleKeyValueUpdate(arguments[0], arguments[1], arguments[2]);
+            break;
+         case "keyvalue_delete":
+            RequireArgumentCount(functionName, arguments, 2);
+            HandleKeyValueDelete(arguments[0], arguments[1]);
+            break;
+         default:
+            throw new InvalidOperationException($"Unknown RPC function '{functionName}'.");
+      }
+   }
+
+   private static void RequireArgumentCount(string functionName, string[] arguments, int expectedCount) {
+      if (arguments.Length != expectedCount) {
+         throw new InvalidOperationException(
+            $"RPC function '{functionName}' expected {expectedCount} arguments but received {arguments.Length}.");
+      }
+   }
+
+   private void HandleKeyValueInsert(string itemNamespace, string key, string value) {
+      var row = new Dictionary<string, string> {
+         ["namespace"] = itemNamespace,
+         ["key"] = key,
+         ["value"] = value,
+      };
+      _persistenceHelper.InsertRow(KeyValueTableName, row);
+      _pub_sub_sync.publish_table(KeyValueTableName);
+      Console.WriteLine($"RPC: keyvalue_insert(namespace='{itemNamespace}', key='{key}', value='{value}')");
+   }
+
+   private void HandleKeyValueUpdate(string itemNamespace, string key, string value) {
+      var row = new Dictionary<string, string> {
+         ["namespace"] = itemNamespace,
+         ["key"] = key,
+         ["value"] = value,
+      };
+      _persistenceHelper.UpdateRow(KeyValueTableName, row);
+      _pub_sub_sync.publish_table(KeyValueTableName);
+      Console.WriteLine($"RPC: keyvalue_update(namespace='{itemNamespace}', key='{key}', value='{value}')");
+   }
+
+   private void HandleKeyValueDelete(string itemNamespace, string key) {
+      var row = new Dictionary<string, string> {
+         ["namespace"] = itemNamespace,
+         ["key"] = key,
+      };
+      _persistenceHelper.RemoveRow(KeyValueTableName, row);
+      _pub_sub_sync.publish_table(KeyValueTableName);
+      Console.WriteLine($"RPC: keyvalue_delete(namespace='{itemNamespace}', key='{key}')");
+   }
+}
+
+public record PersistenceRpcRequest {
+   public PersistenceRpcRequest(string functionName, string[] arguments) {
+      FunctionName = functionName;
+      Arguments = arguments;
+      Console.WriteLine($"PersistenceRpcRequest created for function '{FunctionName}' with {Arguments.Length} argument(s).");
+   }
+
+   public string FunctionName { get; init; }
+   public string[] Arguments { get; init; }
+}

src/DataStreamSuite/PersistenceServer/PersistenceServer.csproj

@@ -0,0 +1,23 @@
+<Project Sdk="Microsoft.NET.Sdk">
+
+  <PropertyGroup>
+    <OutputType>Exe</OutputType>
+    <TargetFramework>net8.0</TargetFramework>
+    <Nullable>enable</Nullable>
+  </PropertyGroup>
+
+  <ItemGroup>
+    <PackageReference Include="System.Data.SQLite" Version="1.0.118" />
+  </ItemGroup>
+
+  <ItemGroup>
+    <ProjectReference Include="..\DataStreamCore\DataStreamCore.csproj" />
+  </ItemGroup>
+
+  <ItemGroup>
+    <Content Include="appsettings.json">
+      <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
+    </Content>
+  </ItemGroup>
+
+</Project>

src/DataStreamSuite/PersistenceServer/PersistenceServerConfig.cs

@@ -0,0 +1,5 @@
+using Bts.PubSub;
+
+public class PersistenceServerConfig {
+   public PubSubConfig? pub_sub { get; set; }
+}

src/DataStreamSuite/PersistenceServer/Program.cs

@@ -0,0 +1,113 @@
+using Bts.DataStreamCore;
+using BCG;
+
+using System;
+using System.IO;
+using System.Net;
+using System.Text;
+using System.Threading;
+using System.Threading.Tasks;
+
+const int Port = 5079;
+var prefix = $"http://localhost:{Port}/";
+
+var listener = new HttpListener();
+listener.Prefixes.Add(prefix);
+var config = ConfigLoader.load();
+var persistenceHelper = new PeristenceHelper();
+var pub_sub_sync = new PubSubSync(persistenceHelper.database_path, config.pub_sub);
+var callHandler = new PersistenceCallHandler(persistenceHelper, pub_sub_sync);
+
+Console.WriteLine("PersistenceServer starting.");
+Console.WriteLine($"Listening on {prefix}");
+Console.WriteLine($"DataStreamCore loaded from: {typeof(IDataPacket).Assembly.FullName}");
+Console.WriteLine("Press Ctrl+C to stop.");
+
+var shutdown = new CancellationTokenSource();
+
+Console.CancelKeyPress += (_, eventArgs) => {
+   eventArgs.Cancel = true;
+   Console.WriteLine("Shutdown requested.");
+   shutdown.Cancel();
+   if (listener.IsListening) {
+      listener.Stop();
+   }
+};
+
+try {
+   pub_sub_sync.publish_all_tables();
+   listener.Start();
+   await RunAsync(listener, callHandler, shutdown.Token);
+   return 0;
+} catch (HttpListenerException ex) when (shutdown.IsCancellationRequested) {
+   Console.WriteLine($"Listener stopped: {ex.Message}");
+   return 0;
+} catch (Exception ex) {
+   Console.Error.WriteLine($"PersistenceServer crashed: {ex}");
+   return 1;
+} finally {
+   listener.Close();
+}
+
+static async Task RunAsync(
+   HttpListener listener,
+   PersistenceCallHandler callHandler,
+   CancellationToken cancellationToken) {
+   while (!cancellationToken.IsCancellationRequested) {
+      HttpListenerContext context;
+      try {
+         context = await listener.GetContextAsync();
+      } catch (HttpListenerException) when (cancellationToken.IsCancellationRequested) {
+         break;
+      } catch (ObjectDisposedException) when (cancellationToken.IsCancellationRequested) {
+         break;
+      }
+
+      _ = Task.Run(() => HandleRequestAsync(context, callHandler, cancellationToken), cancellationToken);
+   }
+}
+
+static async Task HandleRequestAsync(
+   HttpListenerContext context,
+   PersistenceCallHandler callHandler,
+   CancellationToken cancellationToken) {
+   var request = context.Request;
+   var response = context.Response;
+
+   try {
+      var body = await ReadBodyAsync(request, cancellationToken);
+      callHandler.Handle(request, body);
+
+      var responseText =
+         $$"""
+         {"status":"ok","method":"{{request.HttpMethod}}","path":"{{request.RawUrl ?? "/"}}"}
+         """;
+
+      var bytes = Encoding.UTF8.GetBytes(responseText);
+      response.StatusCode = (int)HttpStatusCode.OK;
+      response.ContentType = "application/json";
+      response.ContentEncoding = Encoding.UTF8;
+      response.ContentLength64 = bytes.LongLength;
+      await response.OutputStream.WriteAsync(bytes, cancellationToken);
+   } catch (Exception ex) {
+      Console.Error.WriteLine($"Request handling failed: {ex}");
+      response.StatusCode = (int)HttpStatusCode.InternalServerError;
+   } finally {
+      response.OutputStream.Close();
+      response.Close();
+   }
+}
+
+static async Task<string> ReadBodyAsync(HttpListenerRequest request, CancellationToken cancellationToken) {
+   if (!request.HasEntityBody) {
+      return string.Empty;
+   }
+
+   using var reader = new StreamReader(
+      request.InputStream,
+      request.ContentEncoding ?? Encoding.UTF8,
+      detectEncodingFromByteOrderMarks: true,
+      leaveOpen: true);
+
+   return await reader.ReadToEndAsync(cancellationToken);
+}

src/DataStreamSuite/PersistenceServer/PubSubSync.cs

@@ -0,0 +1,116 @@
+using System;
+using System.Collections.Generic;
+using System.Data;
+using System.Data.SQLite;
+
+using BCG;
+using Bts.DataStreamCore;
+using Bts.PubSub;
+using Bts.PubSub.Routing;
+using Microsoft.Extensions.DependencyInjection;
+
+using bts;
+
+public class PubSubSync {
+   private readonly string _connectionString;
+   private readonly Dictionary<string, IPubTableBroker> _brokers = new(StringComparer.OrdinalIgnoreCase);
+   private readonly object _sync_root = new();
+   private readonly IServiceProvider _serviceProvider;
+   private readonly PubSubRouterConfig _router_config;
+   private readonly PubSubTagHandlerFactoryConfig _tag_handler_factory_config;
+
+   public PubSubSync(string databasePath, PubSubConfig? pubSubConfig = null) {
+      ArgumentException.ThrowIfNullOrWhiteSpace(databasePath);
+      _connectionString = $"Data Source={databasePath};Version=3";
+      var resolved_pub_sub_config = get_or_initialize_pub_sub_config(pubSubConfig);
+      _router_config = resolved_pub_sub_config.Routing ?? throw new InvalidOperationException("PubSub routing config is required.");
+      _tag_handler_factory_config = resolved_pub_sub_config.TagHandlerFactory ?? PubSubTagHandlerFactoryConfig.Default();
+      _serviceProvider = new ServiceCollection().BuildServiceProvider();
+   }
+
+   public void publish_all_tables() {
+      lock (_sync_root) {
+         foreach (var table_name in get_table_names()) {
+            publish_table_core(table_name);
+         }
+      }
+   }
+
+   public void publish_table(string table_name) {
+      ArgumentException.ThrowIfNullOrWhiteSpace(table_name);
+
+      lock (_sync_root) {
+         publish_table_core(table_name);
+      }
+   }
+
+   private void publish_table_core(string table_name) {
+      var rows = load_table(table_name);
+      var broker = get_or_create_broker(table_name);
+      broker.create_table(rows);
+      Console.WriteLine($"PubSubSync published table '{table_name}' with {rows.Count} row(s).");
+   }
+
+   private IPubTableBroker get_or_create_broker(string table_name) {
+      if (_brokers.TryGetValue(table_name, out var broker)) {
+         return broker;
+      }
+
+      var publisher_core = new PublisherCore(
+         new PubSubRouter(_router_config),
+         new PubsubTagHandlerFactory(_serviceProvider, _tag_handler_factory_config));
+      broker = new PubTableBroker(publisher_core);
+      _brokers[table_name] = broker;
+      return broker;
+   }
+
+   private string[] get_table_names() {
+      using var connection = open_connection();
+      using var command = new SQLiteCommand(
+         "SELECT name FROM sqlite_master WHERE type = 'table' AND name NOT LIKE 'sqlite_%' ORDER BY name",
+         connection);
+      using var reader = command.ExecuteReader();
+
+      var table_names = new List<string>();
+      while (reader.Read()) {
+         table_names.Add(reader.GetString(0));
+      }
+
+      return table_names.ToArray();
+   }
+
+   private IBlueRowCollection load_table(string table_name) {
+      using var connection = open_connection();
+      using var adapter = new SQLiteDataAdapter($"SELECT * FROM {quote_identifier(table_name)}", connection);
+      var table = new DataTable(table_name);
+      adapter.Fill(table);
+      table.TableName = table_name;
+      return BlueDataUtility.toIBlueRowCollection(table);
+   }
+
+   private SQLiteConnection open_connection() {
+      var connection = new SQLiteConnection(_connectionString);
+      connection.Open();
+      return connection;
+   }
+
+   private static PubSubConfig get_or_initialize_pub_sub_config(PubSubConfig? configured_pub_sub) {
+      try {
+         PubSubConfig.SetGlobalConfig(PubSubConfig.Default());
+         var existing = PubSubConfig.Instance;
+         if (configured_pub_sub is not null) {
+            existing.MergeWith(configured_pub_sub);
+         }
+         return existing;
+      } catch (InvalidOperationException) {
+         var config = PubSubConfig.Default();
+         if (configured_pub_sub is not null) {
+            config.MergeWith(configured_pub_sub);
+         }
+         PubSubConfig.SetGlobalConfig(config);
+         return config;
+      }
+   }
+
+   private static string quote_identifier(string identifier) => $"\"{identifier.Replace("\"", "\"\"")}\"";
+}

src/DataStreamSuite/PersistenceServer/appsettings.json

@@ -0,0 +1,20 @@
+{
+  "PubSub": {
+    "Routing": {
+      "PublisherRoutes": [
+        {
+          "Name": "default",
+          "ServerAddress": "bts-cloud://localhost:22019",
+          "TagRegExp": ".*"
+        }
+      ],
+      "SubscriberRoutes": [
+        {
+          "Name": "default",
+          "ServerAddress": "bts-cloud://localhost:22019",
+          "TagRegExp": ".*"
+        }
+      ]
+    }
+  }
+}

src/DataStreamSuite/PersistenceServer/client/persistence_client.py

@@ -0,0 +1,43 @@
+import json
+import sys
+import urllib.error
+import urllib.request
+
+
+SERVER_URL = "http://localhost:5079/save?source=python-client"
+
+
+def main() -> int:
+   payload = {
+      "function_name": "keyvalue_insert",
+      "arguments": [
+         "example-namespace",
+         "example-key",
+         "hello from python",
+      ],
+   }
+
+   data = json.dumps(payload).encode("utf-8")
+   request = urllib.request.Request(
+      SERVER_URL,
+      data=data,
+      headers={
+         "Content-Type": "application/json",
+         "X-Client": "PersistenceServerPythonClient",
+      },
+      method="POST",
+   )
+
+   try:
+      with urllib.request.urlopen(request) as response:
+         body = response.read().decode("utf-8")
+         print(f"status: {response.status}")
+         print(f"body: {body}")
+         return 0
+   except urllib.error.URLError as error:
+      print(f"request failed: {error}", file=sys.stderr)
+      return 1
+
+
+if __name__ == "__main__":
+   raise SystemExit(main())

src/DataStreamSuite/PubSub/Client/Communication/BlutoTCP/ClientToServerConnectionWrapper.cs

@@ -1,44 +1,44 @@
-using System;
-using System.Collections.Generic;
-using System.Text;
-using System.Runtime.InteropServices;
+// using System;
+// using System.Collections.Generic;
+// using System.Text;
+// using System.Runtime.InteropServices;
 
-namespace BCG
-{
-   public class ClientToServerConnectionWrapper
-   {
-      [DllImport("BlutoInterop.dll")]
-      public unsafe static extern void init();
+// namespace BCG
+// {
+//    public class ClientToServerConnectionWrapper
+//    {
+//       [DllImport("BlutoInterop.dll")]
+//       public unsafe static extern void init();
 
-      [DllImport("BlutoInterop.dll")]
-      public unsafe static extern void initServerConnection(byte[] ip, int port);
+//       [DllImport("BlutoInterop.dll")]
+//       public unsafe static extern void initServerConnection(byte[] ip, int port);
 
-      [DllImport("BlutoInterop.dll")]
-      public static extern void sendDataToServer(byte[] ip, int port, byte[] message, int size);
+//       [DllImport("BlutoInterop.dll")]
+//       public static extern void sendDataToServer(byte[] ip, int port, byte[] message, int size);
 
-      [DllImport("BlutoInterop.dll")]
-      public unsafe static extern void getServerData(int timeout, int maxElems, BlutoTCPClientMessage*** retpts, out int npts);
+//       [DllImport("BlutoInterop.dll")]
+//       public unsafe static extern void getServerData(int timeout, int maxElems, BlutoTCPClientMessage*** retpts, out int npts);
 
-      public static byte[] stringToBytes(string s)
-      {
-         byte[] rv = new byte[s.Length + 1];
-         for (int i = 0; i < s.Length; i++)
-            rv[i] = Convert.ToByte(s[i]);
-         rv[s.Length] = Convert.ToByte ('\0');
-         return rv;
-      }
+//       public static byte[] stringToBytes(string s)
+//       {
+//          byte[] rv = new byte[s.Length + 1];
+//          for (int i = 0; i < s.Length; i++)
+//             rv[i] = Convert.ToByte(s[i]);
+//          rv[s.Length] = Convert.ToByte ('\0');
+//          return rv;
+//       }
 
-      public static string intPtrToString(IntPtr iptr)
-      {
-         return Marshal.PtrToStringAnsi(iptr);
-      }
+//       public static string intPtrToString(IntPtr iptr)
+//       {
+//          return Marshal.PtrToStringAnsi(iptr);
+//       }
 
-      public static byte[] intPtrToByteArray(IntPtr iptr, int sz)
-      {
-         byte[] data = new byte[sz];
-         Marshal.Copy(iptr, data, 0, data.Length);
-         return data;
-      }
+//       public static byte[] intPtrToByteArray(IntPtr iptr, int sz)
+//       {
+//          byte[] data = new byte[sz];
+//          Marshal.Copy(iptr, data, 0, data.Length);
+//          return data;
+//       }
 
-   }
-}
+//    }
+// }

src/DataStreamSuite/PubSub/Client/MetaData/PubSubClientMessage.cs

@@ -39,8 +39,6 @@ public static class PubSubMessageUtils {
    }
 }
 
-// Subscribe //////////////////////////////////////////////////////////////
-///////////////////////////////////////////////////////////////////////////
 public class SubscribeMsg : IPubSubMessage, IEquatable<SubscribeMsg> {
    public string? connection;
    public string subscribing_app = ClientInfo.thisClient.app;
@@ -86,8 +84,6 @@ public class SubscribeMsg : IPubSubMessage, IEquatable<SubscribeMsg> {
       $"{GetType().Name} connection: {connection} subscribing_app: {subscribing_app} tags: {string.Concat(tags.empty_if_null())}";
 }
 
-// Unsubscribe ////////////////////////////////////////////////////////////
-///////////////////////////////////////////////////////////////////////////
 public class UnsubscribeMsg : SubscribeMsg, IEquatable<UnsubscribeMsg> {
    public bool Equals(UnsubscribeMsg? other) =>
       other is not null && base.Equals(other);
@@ -99,8 +95,6 @@ public class UnsubscribeMsg : SubscribeMsg, IEquatable<UnsubscribeMsg> {
       base.GetHashCode();
 }
 
-// Trasnaction /////////////////////////////////////////////////////////////
-///////////////////////////////////////////////////////////////////////////
 public class TransactionMsg : IPubSubMessage, IEquatable<TransactionMsg> {
    uint n_msgs;
 
@@ -127,15 +121,12 @@ public class TransactionMsg : IPubSubMessage, IEquatable<TransactionMsg> {
       n_msgs.GetHashCode();
 }
 
-// Publish ////////////////////////////////////////////////////////////////
-/// ///////////////////////////////////////////////////////////////////////
 public class PublishMsg : IPubSubMessage, IEquatable<PublishMsg> {
    public string? connection;
    public string publishing_app = ClientInfo.thisClient.app;
    public string? tag;
    public string? tag_type;
    public bool is_compressed = false;
-   // this is the size that of the msg, regarless of it being serialized or not
    public long publish_time_epoch_ms = DateTimeOffset.UtcNow.ToUnixTimeMilliseconds();
    public byte[]? data_to_publish;
 
@@ -144,8 +135,6 @@ public class PublishMsg : IPubSubMessage, IEquatable<PublishMsg> {
       Log.assert(!string.IsNullOrEmpty(tag_type));
       Log.assert(!string.IsNullOrEmpty(tag));
       Log.assert(!string.IsNullOrEmpty(connection));
-      // The 'ThrowIfNullOrEmpty' calls are redundant (given the assert statements above), but
-      // necessary for the compiler to recognize the strings won't be null when this method exits.
       ArgumentException.ThrowIfNullOrEmpty(tag_type);
       ArgumentException.ThrowIfNullOrEmpty(tag);
       ArgumentException.ThrowIfNullOrEmpty(connection);
@@ -181,7 +170,7 @@ public class PublishMsg : IPubSubMessage, IEquatable<PublishMsg> {
    }
 
    public bool Equals(PublishMsg? other) =>
-      other is not null 
+      other is not null
          && connection == other.connection
          && publishing_app == other.publishing_app
          && tag == other.tag
@@ -220,7 +209,6 @@ public enum PubSubMessageType {
 
 public class PubSubClientMessage : IPubSubMessage {
    public PubSubMessageType msg_type;
-   // only one of these wil be non null
    public SubscribeMsg? sub_msg;
    public UnsubscribeMsg? unsub_msg;
    public TransactionMsg? transaction_msg;
@@ -265,7 +253,7 @@ public class PubSubClientMessage : IPubSubMessage {
             ValidateMsg(pub_msg).serialize_into_str(str);
             break;
 
-         T ValidateMsg<T>([NotNull]T? msg, [CallerArgumentExpression(nameof(msg))]string name = "") {
+         T ValidateMsg<T>([NotNull] T? msg, [CallerArgumentExpression(nameof(msg))] string name = "") {
             if (msg is null) {
                throw new InvalidOperationException($"msg type is {msg_type} but {name} is null");
             }
@@ -383,7 +371,7 @@ public class PubSubServerMessage : IPubSubMessage, IEquatable<PubSubServerMessag
 
    [MemberNotNull(nameof(tag), nameof(tag_type))]
    public void construct_deserialize(byte[] msg) =>
-      construct_deserialize(new MemoryStream(msg) { Position = 0});
+      construct_deserialize(new MemoryStream(msg) { Position = 0 });
 
    public bool Equals(PubSubServerMessage? other) =>
       other is not null

src/DataStreamSuite/PubSub/Config/PubSubConfig.cs

@@ -48,16 +48,16 @@ public class PubSubConfig {
          HeartbeatInterval = 15000,
          MissingHeartbeatsBeforeGc = 50,
          DisposedConnectionCheckPeriod = 1000,
-         ServerPort = 12345,
+         ServerPort = 22019,
          TagTypeToAssemblyMap = new(),
          TagHandlerFactory = PubSubTagHandlerFactoryConfig.Default(),
          SimpleEchoServer = false,
          Routing = new() {
             PublisherRoutes = new() {
-               new("bts-cloud://127.0.0.1:12345", ".*")
+               new("bts-cloud://127.0.0.1:22019", ".*")
             },
             SubscriberRoutes = new() {
-               new("bts-cloud://127.0.0.1:12345", ".*")
+               new("bts-cloud://127.0.0.1:22019", ".*")
             }
          }
       };

src/DataStreamSuite/PubSub/ManagedTCP/AsyncTCPConnectionAcceptor_APM.cs

@@ -12,6 +12,7 @@ using System.Net.Sockets;
 
 // opens a TCP port and starts listening to connections
 public class AsyncTCPConnectionAcceptor_APM {
+   // This acceptor exists as an alternate implementation and is not the one currently used by the pubsub server.
    private readonly int _port;
    Socket? _accept_socket;
    readonly ITCPConnectionHandler _connection_handler;

src/DataStreamSuite/PubSub/ManagedTCP/AsyncTCPConnectionAcceptor_AsyncWrapper.cs

@@ -13,6 +13,7 @@ namespace bts {
 
    // opens a TCP port and starts listening to connections. Unlike AsyncTCPConnectionAcceptor_APM,
    // it uses TcpListener, instead of raw sockets.
+   // This is the acceptor currently used by the pubsub server via ManagedTCPMessageHub.
    public class AsyncTCPConnectionAcceptor_AsyncWrapper {
       private readonly int _accept_port;
       // private Socket _accept_socket;

src/DataStreamSuite/PubSub/ManagedTCP/AsyncTCPConnector_APM.cs

@@ -10,6 +10,7 @@ namespace bts {
     // Unlike AsyncTCPConnectionAcceptor_APM this class is static since the requests to connect are stand alone requests
     // whereas AsyncTCPConnectionAcceptor_APM is always listening for incoming connections which lends itself to more
     // statefullness and hence non static
+    // This connector exists as an alternate implementation and is not the one currently used by pubsub clients.
     public static class AsyncTCPConnector_APM {
 
         internal class SocketConnectionArgs {

src/DataStreamSuite/PubSub/ManagedTCP/AsyncTCPConnector_AsyncWrapper.cs

@@ -13,17 +13,18 @@ using System.Diagnostics.CodeAnalysis;
 
 
 namespace bts {
-   // Unlike AsyncTCPConnectionAcceptor_APM this class is static since the requests to connect are stand alone requests
-   // whereas AsyncTCPConnectionAcceptor_APM is always listening for incoming connections which lends itself to more
+   // Unlike AsyncTCPConnectionAcceptor_AsyncWrappe this class is static since the requests to connect are stand alone requests
+   // whereas AsyncTCPConnectionAcceptor_AsyncWrappe is always listening for incoming connections which lends itself to more
    // statefullness and hence non static
+   // This is the connector currently used by pubsub clients via PubSubServerManagedTCPConnection.
    public static class AsyncTCPConnector_AsyncWrapper {
       public static void connect_to(RemoteConnectionInfo server_address, ITCPConnectionHandler handler) {
          bts.Log.Gen.info($"AsyncTCPConnector_AsyncWrapper.connect_to server: {server_address}");
          //Task.Run(() => _connect_to(server_address, handler));
          ThreadPool.QueueUserWorkItem(_ => _keep_trying_blocking_function(
-            Connect, TimeSpan.FromSeconds(1), TimeSpan.FromSeconds(5)));
+            _connect, TimeSpan.FromSeconds(1), TimeSpan.FromSeconds(5)));
 
-         bool Connect() {
+         bool _connect() {
             try {
                return _connect_to(server_address, handler);
             } catch (Exception ex) {
@@ -49,7 +50,7 @@ namespace bts {
          handler.onConnection(conn);
          return true;
 
-         bool CreateTcpClient([NotNullWhen(true)]out TcpClient? tcpClient) {
+         bool CreateTcpClient([NotNullWhen(true)] out TcpClient? tcpClient) {
             try {
                tcpClient = new(server_rci.ip, server_rci.port);
                return true;

src/DataStreamSuite/PubSub/Misc/ClientInfo.cs

@@ -1,4 +1,4 @@
-using System;
+using System;
 using System.Text;
 
 namespace BCG
@@ -24,8 +24,8 @@ namespace BCG
       {
          get { return myApp; }
          set { myApp = value; }
-
       }
+
       private string myKeyQualifier;
       public string keyQualifier
       {
@@ -38,8 +38,8 @@ namespace BCG
       {
          get { return myStartTime; }
          set { myStartTime = value; }
-
       }
+
       string? myProcessString;
       public string processString
       {
@@ -49,10 +49,8 @@ namespace BCG
                myProcessString = getProcessString(";");
             return myProcessString;
          }
-
       }
 
-      
       public ClientInfo(string theMachineName, int thePid, string theProcessName, string theKeyQualifier, DateTime theStartTime)
       {
          myMachineName = theMachineName;
@@ -68,10 +66,8 @@ namespace BCG
          myMachineName = Environment.MachineName;
          myPid = System.Diagnostics.Process.GetCurrentProcess().Id;
          myApp = System.Diagnostics.Process.GetCurrentProcess().ProcessName.ToUpper();
-         string[] args = System.Environment.GetCommandLineArgs();
+         string[] args = Environment.GetCommandLineArgs();
 
-         //the key qualifier is the last argument given. If they didn't give any arguments
-         //but the program itself, then the KQ is empty. 
          if (args.Length > 1)
             myKeyQualifier = args[args.Length - 1];
          else
@@ -86,13 +82,13 @@ namespace BCG
 
       public static ClientInfo thisClient
       {
-         get 
+         get
          {
             if (myLocalClient == null)
             {
-               lock(myLock)
+               lock (myLock)
                {
-                  if(myLocalClient == null)
+                  if (myLocalClient == null)
                      myLocalClient = new ClientInfo();
                }
             }
@@ -102,7 +98,6 @@ namespace BCG
 
       public static ClientInfo fromProcessString(string procString)
       {
-
          ClientInfo outputGuy = new ClientInfo();
 
          string[] info = procString.Split(';');
@@ -112,9 +107,7 @@ namespace BCG
          outputGuy.myKeyQualifier = info[2];
          outputGuy.myPid = Int32.Parse(info[3]);
 
-
          return outputGuy;
-
       }
 
       private string getProcessString(string seperator)

src/DataStreamSuite/PubSub/Misc/IWorker.cs

@@ -3,7 +3,7 @@ using System;
 namespace BCG.Utilities.General
 {
    public interface IWorker
-	{
+   {
       void doWork();
-	}
+   }
 }

src/DataStreamSuite/PubSub/Misc/Pair.cs

@@ -79,5 +79,4 @@ namespace mbts
          return ToString().GetHashCode();
       }
    }
-
 }

src/DataStreamSuite/PubSub/Misc/StreamUtilityMethods.cs

@@ -9,7 +9,7 @@ namespace BCG {
    /// <summary>
    /// Collection of utility methods used throughout
    /// </summary>
-	public static class StreamUtilityMethods {
+   public static class StreamUtilityMethods {
 
       // extension method to write a string C++ style (len prefixed)
       public static void write_string_cpp(this BinaryWriter bw, string? str) {
@@ -98,7 +98,7 @@ namespace BCG {
          byte[] secondStreamBytes = new byte[stream.Length - splitLocation];
          for (int i = 0; i < stream.Length; ++i) {
             //for some resaon, the 'getBytesFromStream' function prepends the length of the byte array to the bytes. This means
-            //we have to ignore those bytes here. 
+            //we have to ignore those bytes here.
             if (i < splitLocation)
                firstStreamBytes[i] = streamBytes[i + 4];
             else
@@ -107,8 +107,6 @@ namespace BCG {
          }
 
          return new Stream[] { new MemoryStream(firstStreamBytes), new MemoryStream(secondStreamBytes) };
-
-
       }
    }
 }

src/DataStreamSuite/PubSub/PublishedCache.cs

@@ -4,7 +4,8 @@ using System;
 using System.Collections.Generic;
 
 namespace bts {
-   // This caches all the published data. This class is created and held by
+   // This caches all the published data, on the pubsliher (client) side. 
+   // This class is created and held by
    // PublisherCore. When data is publsihed , PublisherCore passes it to this 
    // class via the publish call. On a reconnection to the PubSubServer, 
    // PublisherCore calls get_content to get the data to re-publish, to 

src/DataStreamSuite/PubSub/PublisherCore.cs

@@ -43,20 +43,20 @@ namespace BCG {
 
       #region constructors
 
-      public PublisherCore(IPubSubRouter router, IPubSubTagHandlerFactory tagHandlerFactory) {
+      public PublisherCore(IPubSubRouter router, IPubSubTagHandlerFactory tag_handler_factory) {
          _router = router;
-         _state = new(tagHandlerFactory);
+         _state = new(tag_handler_factory);
          Thread pump_thread = new Thread(new ThreadStart(_message_pump));
          pump_thread.IsBackground = true;
          pump_thread.Start();
       }
 
-      public static PublisherCore Create(IPubSubRouter? router = null, IPubSubTagHandlerFactory? tagHandlerFactory = null, IServiceProvider? serviceProvider = null) {
+      public static PublisherCore Create(IPubSubRouter? router = null, IPubSubTagHandlerFactory? tag_handler_factory = null, IServiceProvider? service_provider = null) {
          router ??= new PubSubRouter();
-         tagHandlerFactory ??= new PubsubTagHandlerFactory(
-            serviceProvider ?? PubSubServices.ServiceProvider,
+         tag_handler_factory ??= new PubsubTagHandlerFactory(
+            service_provider ?? PubSubServices.ServiceProvider,
             PubSubConfig.Instance.TagHandlerFactory ?? PubSubTagHandlerFactoryConfig.Default());
-         return new(router, tagHandlerFactory);
+         return new(router, tag_handler_factory);
       }
 
       public void Dispose() { }
@@ -119,7 +119,7 @@ namespace BCG {
       #endregion
 
       // static void setGlobalInstance() {
-         
+
       // }
 
       #region Message Pump Thread - this is where messages are dequeued and the action performed

src/DataStreamSuite/PubSub/Server/IInternalPublisher.cs

@@ -2,6 +2,6 @@ namespace Bts.PubSub.Server;
 
 using bts;
 
-interface IInternalPublisher {
+public interface IInternalPublisher {
    void Publish(PublishMsg message);
 }

src/DataStreamSuite/PubSub/Server/IInternalPublisherQueue.cs

@@ -10,7 +10,7 @@ using System.Collections.Generic;
 /// messages to be published. It is also passed to the <see cref="BCG.PubSubMessageReactor"/>, so it
 /// can dequeue these messages and send them over the wire.
 /// </summary>
-interface IInternalPublisherQueue {
+public interface IInternalPublisherQueue {
    void Enqueue(PublishMsg message);
    List<PublishMsg> DequeueMultiple(TimeSpan ts, int maxElements);
 }

src/DataStreamSuite/PubSub/Server/IPubSubInternalPublisher.cs

@@ -0,0 +1,9 @@
+namespace BlueOrb.PubSub.Server;
+
+using BCG.Utilities.General;
+
+using System;
+
+public interface IPubSubInternalPublisher : IWorker {
+   event Action<string, string, byte[]>? onNewData;
+}

src/DataStreamSuite/PubSub/Server/IPubSubState.cs

@@ -10,6 +10,16 @@ public delegate void PubSubStateSingleTagChangeDelegate(PubSubSingleTagStateEven
 public delegate void PubSubStateChangeDelegate(string[] tags);
 public delegate void PubSubStatsDelegate(string tag, int size);
 
+public class PubSubServerTagStats {
+   public int n_updates { get; set; }
+   public long total_bytes { get; set; }
+   public int min_size { get; set; }
+   public int max_size { get; set; }
+   public int last_update_siz { get; set; }
+   public int average_size { get; set; }
+   public long last_update_time { get; set; }
+}
+
 public interface IPubSubState {
    event PubSubStateChangeDelegate? onTagMapChanged;
    event PubSubStatsDelegate? onPublishStats;
@@ -18,6 +28,7 @@ public interface IPubSubState {
 
    byte[] get_content(string tag);
    string getTagType(string tag);
+   PubSubServerTagStats get_stats(string tag);
    void get_all_connections(
       string tag,
       out Dictionary<string, string> subscribers,

src/DataStreamSuite/PubSub/Server/InternalPublisherQueue.cs

@@ -7,7 +7,7 @@ using bts;
 using System;
 using System.Collections.Generic;
 
-class InternalPublisherQueue : IInternalPublisherQueue {
+public class InternalPublisherQueue : IInternalPublisherQueue {
    readonly SyncQueue<PublishMsg> queue = new();
    readonly object lockObj = new();
 

src/DataStreamSuite/PubSub/Server/InternalTagHandling/SystemHeartbeatPublisher.cs

@@ -1,72 +1,76 @@
-using System;
-using System.Collections.Generic;
-using System.Text;
-using BCG.Utilities.General;
-using System.IO;
-using bts;
-using Bts.PubSub;
-using Bts.PubSub.Server;
-using Spark.Core;
+// using System;
+// using BCG.Utilities.General;
+// using System.IO;
+// using bts;
+// using Bts.PubSub;
+// using Bts.PubSub.Server;
+// using Spark.Core;
 
-namespace BCG {
-   class Pubsubheartbeat {
-      public Int64 hb_time_epoch;
-      public byte[] serialize() {
-         MemoryStream str = new MemoryStream();
-         BinaryWriter bw = new BinaryWriter(str);
-         bw.Write(hb_time_epoch);
-         str.Position = 0;
-         return str.ToArray();
-      }
+// namespace BCG {
+//    class Pubsubheartbeat {
+//       public Int64 hb_time_epoch;
+//       public byte[] serialize() {
+//          MemoryStream str = new MemoryStream();
+//          BinaryWriter bw = new BinaryWriter(str);
+//          bw.Write(hb_time_epoch);
+//          str.Position = 0;
+//          return str.ToArray();
+//       }
 
-      public Pubsubheartbeat() {
-         hb_time_epoch = DateTimeOffset.UtcNow.ToUnixTimeMilliseconds();
-      }
+//       public Pubsubheartbeat() {
+//          hb_time_epoch = DateTimeOffset.UtcNow.ToUnixTimeMilliseconds();
+//       }
 
-      public static Pubsubheartbeat deserialize(byte[] data) {
-         long hb_time = BitConverter.ToInt64(data);
-         return new Pubsubheartbeat() {
-            hb_time_epoch = hb_time,
-         };
-      }
-   }
-   /// <summary>
-   /// Periodicaly publishes a heartbeat, under the tag name System_Heartbeat.
-   /// A subscriber can subscribe to it, and find out of a PubSub is alive or 
-   /// not. Infact more so, if the subsriptions are working or not.
-   /// </summary>
-   internal class SystemHeartbeatPublisher : VirtualSingleton<SystemHeartbeatPublisher>, IWorker, IParameterizedWorker {
-      public static string tag_type = "System_Heartbeat";
-      public static string tag_name = "System_Heartbeat";
-      readonly IInternalPublisher internalPublisher;
-      readonly IWorker workerImpl;
+//       public static Pubsubheartbeat deserialize(byte[] data) {
+//          long hb_time = BitConverter.ToInt64(data);
+//          return new Pubsubheartbeat() {
+//             hb_time_epoch = hb_time,
+//          };
+//       }
+//    }
+//    /// <summary>
+//    /// Periodicaly publishes a heartbeat, under the tag name System_Heartbeat.
+//    /// A subscriber can subscribe to it, and find out if a PubSubServer is alive 
+//    /// or not. Infact more so, if the subsriptions are working or not.
+//    /// </summary>
+//    internal class SystemHeartbeatPublisher : VirtualSingleton<SystemHeartbeatPublisher>, IWorker, IParameterizedWorker {
+//       public static string tag_type = "System_Heartbeat";
+//       public static string tag_name = "System_Heartbeat";
 
-      public SystemHeartbeatPublisher(IInternalPublisher internalPublisher) {
-         this.internalPublisher = internalPublisher;
-         workerImpl = new SingleThreadedWorker(new AnonymousWorker(doWorkImpl));
-      }
+//       readonly IInternalPublisher internalPublisher;
+//       readonly IWorker workerImpl;
 
-      TimeSpan IParameterizedWorker.Interval => TimeSpan.FromMilliseconds(500);
+//       public SystemHeartbeatPublisher(IInternalPublisher internalPublisher) {
+//          this.internalPublisher = internalPublisher;
+//          workerImpl = new SingleThreadedWorker(new AnonymousWorker(doWorkImpl));
+//       }
 
-      #region IWorker Members
+//       TimeSpan IParameterizedWorker.Interval => TimeSpan.FromMilliseconds(500);
 
-      public void doWork() =>
-         workerImpl.doWork();
+//       #region IWorker Members
 
-      void doWorkImpl() {
-         Pubsubheartbeat hb = new Pubsubheartbeat();
-         PublishMsg msg = new PublishMsg() {
-            connection = "Internal",
-            publishing_app = ClientInfo.thisClient.app,
-            tag = "pss_hb",
-            tag_type = "Internal",
-            is_compressed = false,
-            publish_time_epoch_ms = hb.hb_time_epoch,
-            data_to_publish = hb.serialize()
-         };
-         internalPublisher.Publish(msg);
-      }
+//       public void doWork() =>
+//          workerImpl.doWork();
 
-      #endregion
-   }
-}
+//       void doWorkImpl() {
+//          // why not publish a table
+//          Pubsubheartbeat hb = new Pubsubheartbeat();
+//          PublishMsg msg = new PublishMsg() {
+//             connection = "Internal",
+//             publishing_app = ClientInfo.thisClient.app,
+//             tag = "pss_hb",
+//             tag_type = "Internal",
+//             is_compressed = false,
+//             publish_time_epoch_ms = hb.hb_time_epoch,
+//             data_to_publish = hb.serialize()
+//          };
+//          internalPublisher.Publish(msg);
+//       }
+
+//       #endregion
+
+//       private void _publish_hb() {
+
+//       }
+//    }
+// }

src/DataStreamSuite/PubSub/Server/InternalTagHandling/SystemStatsPublisher.cs

@@ -1,97 +1,97 @@
-using System;
-using System.Collections.Generic;
-using System.Text;
-using System.IO;
-using BCG.Utilities.General;
-using bts;
-using Bts.PubSub;
-using Bts.PubSub.Server;
-using Spark.Core;
+// using System;
+// using System.Collections.Generic;
+// using System.Text;
+// using System.IO;
+// using BCG.Utilities.General;
+// using bts;
+// using Bts.PubSub;
+// using Bts.PubSub.Server;
+// using Spark.Core;
 
-namespace BCG {
-   /// <summary>
-   /// Very similiar to SystemSubscriptionMapPublisher, except that it keeps updating the state as _handle_publish
-   /// is called (in_tag_stats_map). in do_work it goes through the ones who were recently updated and call an internal_publish
-   /// Simialiar to SystemSubscriptionMapPublisher, it is one half of an IPubSubServerTagHandler, ie does not react to update,
-   /// but returns the state in get_content (though it could have perhaps used update instead ofth spcialized event, onPublishStats) to 
-   /// keep track of publish sizes.
-   /// </summary>
-   class SystemStatsPublisher : VirtualSingleton<SystemStatsPublisher>, IPubSubServerTagHandler, IParameterizedWorker {
-      public static string tag_type = "System_Stats";
-      public static string tag_name = "System_AllStatistics";
-      // these tags have been touched, need to publish in the next doWork
-      private HashSet<string> _dirty_tags = new HashSet<string>();
-      private Dictionary<string, TagStatsInfo> _tag_stats_map = new Dictionary<string, TagStatsInfo>();
-      readonly IInternalPublisher internalPublisher;
-      readonly IWorker workerImpl;
+// namespace BCG {
+//    /// <summary>
+//    /// Very similiar to SystemSubscriptionMapPublisher, except that it keeps updating the state as _handle_publish
+//    /// is called (in_tag_stats_map). in do_work it goes through the ones who were recently updated and call an internal_publish
+//    /// Simialiar to SystemSubscriptionMapPublisher, it is one half of an IPubSubServerTagHandler, ie does not react to update,
+//    /// but returns the state in get_content (though it could have perhaps used update instead ofth spcialized event, onPublishStats) to 
+//    /// keep track of publish sizes.
+//    /// </summary>
+//    class SystemStatsPublisher : VirtualSingleton<SystemStatsPublisher>, IPubSubServerTagHandler, IParameterizedWorker {
+//       public static string tag_type = "System_Stats";
+//       public static string tag_name = "System_AllStatistics";
+//       // these tags have been touched, need to publish in the next doWork
+//       private HashSet<string> _dirty_tags = new HashSet<string>();
+//       private Dictionary<string, TagStatsInfo> _tag_stats_map = new Dictionary<string, TagStatsInfo>();
+//       readonly IInternalPublisher internalPublisher;
+//       readonly IWorker workerImpl;
 
-      public SystemStatsPublisher(IPubSubState state, IInternalPublisher internalPublisher) {
-         bts.Log.Gen.info("SystemStatsPublisher constructed");
-         workerImpl = new SingleThreadedWorker(new AnonymousWorker(doWorkImpl));
-         this.internalPublisher = internalPublisher;
-         state.onPublishStats += _handle_publish;
-      }
+//       public SystemStatsPublisher(IPubSubState state, IInternalPublisher internalPublisher) {
+//          bts.Log.Gen.info("SystemStatsPublisher constructed");
+//          workerImpl = new SingleThreadedWorker(new AnonymousWorker(doWorkImpl));
+//          this.internalPublisher = internalPublisher;
+//          state.onPublishStats += _handle_publish;
+//       }
 
-      public string TagType => tag_type;
-      TimeSpan IParameterizedWorker.Interval => TimeSpan.FromMilliseconds(250);
+//       public string TagType => tag_type;
+//       TimeSpan IParameterizedWorker.Interval => TimeSpan.FromMilliseconds(250);
 
-      void _handle_publish(string tag, int size) {
-         bts.Log.Gen.info($"SystemStatsPublisher._handle_publish tag: {tag} size: {size}");
-         if (tag == tag_name)
-            return;
-         if (!_tag_stats_map.TryGetValue(tag, out var tag_stats)) {
-            tag_stats = new TagStatsInfo(tag);
-            _tag_stats_map.Add(tag, tag_stats);
-         }
-         tag_stats.update(size);
-         _dirty_tags.Add(tag);
-      }
+//       void _handle_publish(string tag, int size) {
+//          bts.Log.Gen.info($"SystemStatsPublisher._handle_publish tag: {tag} size: {size}");
+//          if (tag == tag_name)
+//             return;
+//          if (!_tag_stats_map.TryGetValue(tag, out var tag_stats)) {
+//             tag_stats = new TagStatsInfo(tag);
+//             _tag_stats_map.Add(tag, tag_stats);
+//          }
+//          tag_stats.update(size);
+//          _dirty_tags.Add(tag);
+//       }
 
 
-      #region IWorker Members
+//       #region IWorker Members
 
-      public void doWork() =>
-         workerImpl.doWork();
+//       public void doWork() =>
+//          workerImpl.doWork();
 
-      void doWorkImpl() {
-         if (_dirty_tags.Count > 0) {
-            MemoryStream str = new MemoryStream();
-            List<TagStatsInfo> dirty_ts_info = new List<TagStatsInfo>();
-            foreach (string tag in _dirty_tags) {
-               //_tag_stats_map[tag].write_to(str);
-               dirty_ts_info.Add(_tag_stats_map[tag]);
-            }
-            _dirty_tags.Clear();
-            var bytes = TagStatsInfo.serialize(dirty_ts_info);
-            PublishMsg pub_msg = new PublishMsg() {
-               connection = "Internal",
-               publishing_app = "PubSubServer",
-               tag = SystemStatsPublisher.tag_name,
-               tag_type = SystemStatsPublisher.tag_type,
-               publish_time_epoch_ms = DateTimeOffset.UtcNow.ToUnixTimeMilliseconds(),
-               data_to_publish = bytes
-            };
-            internalPublisher.Publish(pub_msg);
-         }
-      }
+//       void doWorkImpl() {
+//          if (_dirty_tags.Count > 0) {
+//             MemoryStream str = new MemoryStream();
+//             List<TagStatsInfo> dirty_ts_info = new List<TagStatsInfo>();
+//             foreach (string tag in _dirty_tags) {
+//                //_tag_stats_map[tag].write_to(str);
+//                dirty_ts_info.Add(_tag_stats_map[tag]);
+//             }
+//             _dirty_tags.Clear();
+//             var bytes = TagStatsInfo.serialize(dirty_ts_info);
+//             PublishMsg pub_msg = new PublishMsg() {
+//                connection = "Internal",
+//                publishing_app = "PubSubServer",
+//                tag = SystemStatsPublisher.tag_name,
+//                tag_type = SystemStatsPublisher.tag_type,
+//                publish_time_epoch_ms = DateTimeOffset.UtcNow.ToUnixTimeMilliseconds(),
+//                data_to_publish = bytes
+//             };
+//             internalPublisher.Publish(pub_msg);
+//          }
+//       }
 
-      #endregion
+//       #endregion
 
-      #region IPubSubServerTagHandler Members
+//       #region IPubSubServerTagHandler Members
 
-      public void update(string tag, byte[]? msg) { }
+//       public void update(string tag, byte[]? msg) { }
 
-      public byte[] get_content(string tag) {
-         var ser = TagStatsInfo.serialize(_tag_stats_map.Values);
-         return ser;
-         // MemoryStream str = new MemoryStream();
-         // BinaryWriter bw = new BinaryWriter(str);
-         // bw.Write(_tag_stats_map.Count);
-         // foreach (TagStatsInfo t in _tag_stats_map.Values)
-         //    t.write_to(bw.BaseStream);
-         // return str.ToArray();
-      }
+//       public byte[] get_content(string tag) {
+//          var ser = TagStatsInfo.serialize(_tag_stats_map.Values);
+//          return ser;
+//          // MemoryStream str = new MemoryStream();
+//          // BinaryWriter bw = new BinaryWriter(str);
+//          // bw.Write(_tag_stats_map.Count);
+//          // foreach (TagStatsInfo t in _tag_stats_map.Values)
+//          //    t.write_to(bw.BaseStream);
+//          // return str.ToArray();
+//       }
 
-      #endregion
-   }
-}
+//       #endregion
+//    }
+// }

src/DataStreamSuite/PubSub/Server/InternalTagHandling/SystemSubscriptionMapPublisher.cs

@@ -1,117 +1,117 @@
-using System;
-using System.Collections.Generic;
-using System.Text;
-using BCG.Utilities.General;
-using System.IO;
-using System.Threading;
-using mbts;
-using System.Threading.Tasks;
-using bts;
-using Bts.PubSub;
-using Bts.PubSub.Server;
-using Spark.Core;
+// using System;
+// using System.Collections.Generic;
+// using System.Text;
+// using BCG.Utilities.General;
+// using System.IO;
+// using System.Threading;
+// using mbts;
+// using System.Threading.Tasks;
+// using bts;
+// using Bts.PubSub;
+// using Bts.PubSub.Server;
+// using Spark.Core;
 
-namespace BCG {
-   /// <summary>
-   /// This class publishes "System_Sub" to the PubSubServer using the 
-   /// internal_publish. The data associated with the "System_Sub" tag 
-   /// is the list fo all subscribers and publishers.
-   /// This class, SystemSubscriptionMapPublisher, gets the data by listening 
-   /// to PubsubState.onTagMapChanged.
-   /// It also serves as an IPubSubServerTagHandler, and resides in SingleTagData.
-   /// update calls are no-ops and on get_content it returns the total state
-   /// It does not have much state except for the dirty state. It builds the state to publish
-   /// as well as return in get_content, from calling into PububState
-   /// </summary>
-   class SystemSubscriptionMapPublisher : VirtualSingleton<SystemSubscriptionMapPublisher>, IPubSubServerTagHandler, IParameterizedWorker {
-      private IPubSubState _state;
-      private HashSet<string> _dirty_tags = new HashSet<string>();
-      private readonly string _ip = SysInfoManager.getIP();
-      public static string tag_type = "System_Sub";
-      public static string tag_name = "System_AllSubscriptions";
-      readonly IInternalPublisher internalPublisher;
-      readonly IWorker workerImpl;
-      readonly object syncRoot = new();
+// namespace BCG {
+//    /// <summary>
+//    /// This class publishes "System_Sub" to the PubSubServer using the 
+//    /// internal_publish. The data associated with the "System_Sub" tag 
+//    /// is the list fo all subscribers and publishers.
+//    /// This class, SystemSubscriptionMapPublisher, gets the data by listening 
+//    /// to PubsubState.onTagMapChanged.
+//    /// It also serves as an IPubSubServerTagHandler, and resides in SingleTagData.
+//    /// update calls are no-ops and on get_content it returns the total state
+//    /// It does not have much state except for the dirty state. It builds the state to publish
+//    /// as well as return in get_content, from calling into PububState
+//    /// </summary>
+//    class SystemSubscriptionMapPublisher : VirtualSingleton<SystemSubscriptionMapPublisher>, IPubSubServerTagHandler, IParameterizedWorker {
+//       private IPubSubState _state;
+//       private HashSet<string> _dirty_tags = new HashSet<string>();
+//       private readonly string _ip = SysInfoManager.getIP();
+//       public static string tag_type = "System_Sub";
+//       public static string tag_name = "System_AllSubscriptions";
+//       readonly IInternalPublisher internalPublisher;
+//       readonly IWorker workerImpl;
+//       readonly object syncRoot = new();
 
-      public SystemSubscriptionMapPublisher(IPubSubState state, IInternalPublisher internalPublisher) {
-         bts.Log.Gen.info("SystemSubscriptionMapPublisher.constructor");
-         this.internalPublisher = internalPublisher;
-         workerImpl = new SingleThreadedWorker(new AnonymousWorker(_internal_publish));
-         _state = state;
-         _state.onTagMapChanged += _on_tag_state_changes;
-      }
+//       public SystemSubscriptionMapPublisher(IPubSubState state, IInternalPublisher internalPublisher) {
+//          bts.Log.Gen.info("SystemSubscriptionMapPublisher.constructor");
+//          this.internalPublisher = internalPublisher;
+//          workerImpl = new SingleThreadedWorker(new AnonymousWorker(_internal_publish));
+//          _state = state;
+//          _state.onTagMapChanged += _on_tag_state_changes;
+//       }
 
-      public string TagType => tag_type;
+//       public string TagType => tag_type;
 
-      public TimeSpan Interval => TimeSpan.FromMilliseconds(500);
+//       public TimeSpan Interval => TimeSpan.FromMilliseconds(500);
 
-      private void _on_tag_state_changes(string[] tags) {
-         Func<string> tags_to_string = () => {
-            var rv = String.Empty;
-            foreach (var t in tags) {
-               rv += t + " ";
-            }
-            rv.Trim();
-            return rv;
-         };
-         bts.Log.Gen.info($"SystemSubscriptionMapPublisher._on_tag_state_changes tags: {tags_to_string()}");
-         lock (syncRoot) {
-            _dirty_tags.AddRange(tags);
-         }
-      }
+//       private void _on_tag_state_changes(string[] tags) {
+//          Func<string> tags_to_string = () => {
+//             var rv = String.Empty;
+//             foreach (var t in tags) {
+//                rv += t + " ";
+//             }
+//             rv.Trim();
+//             return rv;
+//          };
+//          bts.Log.Gen.info($"SystemSubscriptionMapPublisher._on_tag_state_changes tags: {tags_to_string()}");
+//          lock (syncRoot) {
+//             _dirty_tags.AddRange(tags);
+//          }
+//       }
 
-      private void _internal_publish() {
-         lock (syncRoot) {
-            if (_dirty_tags.Count > 0) {
-               List<TagPubSubInfo> tpsis = new List<TagPubSubInfo>();
-               foreach (string tag in _dirty_tags) {
-                  Dictionary<string, string> subscribers, active_pubslishers;
-                  ICollection<Pair<string, string>> dead_publishers;
-                  _state.get_all_connections(tag, out subscribers, out active_pubslishers, out dead_publishers);
-                  TagPubSubInfo ti = new TagPubSubInfo(tag, subscribers, active_pubslishers,
-                                                      dead_publishers, _ip);
-                  tpsis.Add(ti);
-               }
-               _dirty_tags.Clear();
-               var ser_bytes = TagPubSubInfo.serialize(tpsis);
-               PublishMsg pub_msg = new PublishMsg() {
-                  publish_time_epoch_ms = 0,
-                  tag = SystemSubscriptionMapPublisher.tag_name,
-                  tag_type = SystemSubscriptionMapPublisher.tag_type,
-                  connection = "Internal",
-                  is_compressed = false,
-                  data_to_publish = ser_bytes
-               };
-               internalPublisher.Publish(pub_msg);
-            }
-         }
-      }
+//       private void _internal_publish() {
+//          lock (syncRoot) {
+//             if (_dirty_tags.Count > 0) {
+//                List<TagPubSubInfo> tpsis = new List<TagPubSubInfo>();
+//                foreach (string tag in _dirty_tags) {
+//                   Dictionary<string, string> subscribers, active_pubslishers;
+//                   ICollection<Pair<string, string>> dead_publishers;
+//                   _state.get_all_connections(tag, out subscribers, out active_pubslishers, out dead_publishers);
+//                   TagPubSubInfo ti = new TagPubSubInfo(tag, subscribers, active_pubslishers,
+//                                                       dead_publishers, _ip);
+//                   tpsis.Add(ti);
+//                }
+//                _dirty_tags.Clear();
+//                var ser_bytes = TagPubSubInfo.serialize(tpsis);
+//                PublishMsg pub_msg = new PublishMsg() {
+//                   publish_time_epoch_ms = 0,
+//                   tag = SystemSubscriptionMapPublisher.tag_name,
+//                   tag_type = SystemSubscriptionMapPublisher.tag_type,
+//                   connection = "Internal",
+//                   is_compressed = false,
+//                   data_to_publish = ser_bytes
+//                };
+//                internalPublisher.Publish(pub_msg);
+//             }
+//          }
+//       }
 
-      #region IWorker Members
+//       #region IWorker Members
 
-      public void doWork() =>
-         workerImpl.doWork();
+//       public void doWork() =>
+//          workerImpl.doWork();
 
-      #endregion
+//       #endregion
 
-      #region IPubSubServerTagHandler Members
+//       #region IPubSubServerTagHandler Members
 
-      public void update(string tag, byte[]? msg) { }
+//       public void update(string tag, byte[]? msg) { }
 
-      public byte[] get_content(string t) {
-         ICollection<string> all_tags = _state.all_tags;
-         List<TagPubSubInfo> ti_list = new List<TagPubSubInfo>();
-         foreach (string tag in all_tags) {
-            Dictionary<string, string> subscribers, activePubslishers;
-            ICollection<Pair<string, string>> deadPublishers;
-            _state.get_all_connections(tag, out subscribers, out activePubslishers, out deadPublishers);
-            TagPubSubInfo ti = new TagPubSubInfo(tag, subscribers, activePubslishers, deadPublishers, _ip);
-            ti_list.Add(ti);
-         }
-         var rv = TagPubSubInfo.serialize(ti_list);
-         return rv;
-      }
+//       public byte[] get_content(string t) {
+//          ICollection<string> all_tags = _state.all_tags;
+//          List<TagPubSubInfo> ti_list = new List<TagPubSubInfo>();
+//          foreach (string tag in all_tags) {
+//             Dictionary<string, string> subscribers, activePubslishers;
+//             ICollection<Pair<string, string>> deadPublishers;
+//             _state.get_all_connections(tag, out subscribers, out activePubslishers, out deadPublishers);
+//             TagPubSubInfo ti = new TagPubSubInfo(tag, subscribers, activePubslishers, deadPublishers, _ip);
+//             ti_list.Add(ti);
+//          }
+//          var rv = TagPubSubInfo.serialize(ti_list);
+//          return rv;
+//       }
 
-      #endregion
-   }
-}
+//       #endregion
+//    }
+// }

src/DataStreamSuite/PubSub/Server/PubSubMessageReactor.cs

@@ -13,46 +13,70 @@ using Bts.PubSub.Server;
 using Microsoft.Extensions.Options;
 using Microsoft.Extensions.Hosting;
 using System.Threading.Tasks;
+using BlueOrb.PubSub.Server;
+using System.Linq;
 
 namespace BCG {
    // This is the main PubSub Server class where it all starts (called by Program.cs). Its constructed and start is called and thats it.
    // The only outward facing members of this class is start
    // It has an IPubsubMessageHub, that it gets messages from, and updates the PubSubState (which really is all the pubsub server state)
-   class PubSubMessageReactor : IHostedService {
+   public class PubSubMessageReactor : IHostedService {
       private IPubsubMessageHub _msg_hub;
       // private delegate void MyMsgHandler(string sender, Stream msg, MetaData m);
       // private Dictionary<string, MyMsgHandler> myOperationToHandler = new Dictionary<string, MyMsgHandler>();
       private Thread _msg_polling_thread;
       // all the data,list of subscribers etc are in the state
       private IPubSubState _state;
-      private IInternalPublisherQueue _internal_pub_queue;
+      private List<IPubSubInternalPublisher>? _internal_publishers;
       private Stopwatch _stop_watch = new Stopwatch();
       TaskCompletionSource? executionTask;
       CancellationTokenSource? executionCts;
 
-      public PubSubMessageReactor(IPubSubState state, IInternalPublisherQueue internalPublisherQueue, IOptions<PubSubConfig> config)
-         : this(state, internalPublisherQueue, config.Value) { }
+      // public PubSubMessageReactor(IPubSubState state, IOptions<PubSubConfig> config)
+      //    : this(state, null, config.Value) {
+      //    bts.Log.Gen.info("PubSubMessageReactor.cons 1 System Initialized");
+      // }
 
-      public PubSubMessageReactor(IPubSubState state, IInternalPublisherQueue internalPublisherQueue, PubSubConfig config)
+      public PubSubMessageReactor(IPubSubState state, IEnumerable<IPubSubInternalPublisher>? internalPublishers, PubSubConfig config)
          : this(
             state,
-            internalPublisherQueue,
+            internalPublishers,
             config.ServerPort.assert_value(),
             TimeSpan.FromMilliseconds(config.HeartbeatInterval.assert_value()),
             config.MissingHeartbeatsBeforeGc.assert_value(),
-            TimeSpan.FromMilliseconds(config.DisposedConnectionCheckPeriod.assert_value())) { }
+            TimeSpan.FromMilliseconds(config.DisposedConnectionCheckPeriod.assert_value())) {
+         bts.Log.Gen.info("PubSubMessageReactor.cons 2 System Initialized");
+      }
 
-      public PubSubMessageReactor(IPubSubState state, IInternalPublisherQueue internalPublisherQueue, int port, TimeSpan heartbeatInterval, int maxMissedHeartbeats, TimeSpan collectDisposedConnectionsInterval) {
-         bts.Log.Gen.info("PubSubMessageReactor.cons System Initialized");
+      // public PubSubMessageReactor(IPubSubState state, int port, TimeSpan heartbeatInterval, int maxMissedHeartbeats, TimeSpan collectDisposedConnectionsInterval)
+      //    : this(state, null, port, heartbeatInterval, maxMissedHeartbeats, collectDisposedConnectionsInterval) {
+      //    bts.Log.Gen.info("PubSubMessageReactor.cons 3 System Initialized");
+      // }
+
+      public PubSubMessageReactor(IPubSubState state, IEnumerable<IPubSubInternalPublisher>? internalPublishers, int port, TimeSpan heartbeatInterval, int maxMissedHeartbeats, TimeSpan collectDisposedConnectionsInterval) {
+         bts.Log.Gen.info("PubSubMessageReactor.cons 4 System Initialized");
 
          _state = state;
-         _internal_pub_queue = internalPublisherQueue;
          _msg_polling_thread = new Thread(new ThreadStart(_polling_thread_func));
          _msg_polling_thread.Priority = ThreadPriority.AboveNormal;
          _msg_hub = new ManagedTCPMessageHub(port, heartbeatInterval, maxMissedHeartbeats, collectDisposedConnectionsInterval);
          // this is the only event this class gets from hub
          _msg_hub.register_connection_callback(new PubSubCallbacks.PubSubServerConnectionStatusChange(_handle_client_connection));
 
+         _internal_publishers = internalPublishers is null
+            ? null
+            : new List<IPubSubInternalPublisher>(internalPublishers);
+         //Log.Gen.info($"internalPublishers: {_internal_publishers.First().GetType()}");
+         _internal_publishers?.ForEach(publisher => publisher.onNewData += (tag, tag_type, data) => {
+            PublishMsg pub_msg = new PublishMsg() {
+               connection = "Internal;",
+               tag = tag,
+               tag_type = tag_type,
+               data_to_publish = data
+            };
+            _handle_pub("Internal", pub_msg, true);
+         });
+
       }
 
       // the connection callback is the only event we get back from the message_hub.
@@ -89,29 +113,17 @@ namespace BCG {
       /// </summary>
       private void _polling_thread_func() {
          Log.Gen.info("PubSubMessageReactor._polling_thread_func starting thread function _polling_thread_func");
-         TimeSpan poll_timeout = new TimeSpan(0, 0, 0, 0, 2500);
          _msg_hub.activate();
-         Action handle_internal_pubs = () => {
-            Log.Gen.info($"PubSubMessageReactor._polling_thread_func handle_internal_pubs");
-            var internal_pubs = _internal_pub_queue.DequeueMultiple(TimeSpan.FromSeconds(1), 64000);
-            //Log.Gen.info($"PubSubMessageReactor._polling_thread_func internal_pubs count: {internal_pubs.Count}");
-            foreach (var msg in internal_pubs) {
-               try {
-                  _handle_pub("Internal", msg, true);
-               } catch (Exception ex) {
-                  bts.Log.Gen.error($"PubSubMessage>reactor._polling_thread_func ex: {ex}");
-               }
-            }
-         };
-
          while (executionCts is null || !executionCts.IsCancellationRequested) {
             try {
+               foreach (var w in _internal_publishers) {
+                  w.doWork();
+               }
                //bts.Log.Gen.info($"PubSubMessageReactor._polling_thread_func pre get_updates");
                var received_data = _msg_hub.get_all_updates();
                var n_msgs = received_data.Count;
                if (n_msgs == 0) {
-                  handle_internal_pubs();
-                  System.Threading.Thread.Sleep(TimeSpan.FromMilliseconds(250));
+                  System.Threading.Thread.Sleep(TimeSpan.FromMilliseconds(10));
                   continue;
                }
                bts.Log.Gen.info($"PubSubMessageReactor._polling_thread_func post get_updates n_msgs: {n_msgs}");
@@ -119,9 +131,10 @@ namespace BCG {
                   // multiple pubs for the same tags should be batched here
                   ArgumentException.ThrowIfNullOrEmpty(kv.first);
                   ArgumentNullException.ThrowIfNull(kv.second);
+                  var stop_watch = Stopwatch.StartNew();
                   _handle_client_msg(kv.first, kv.second);
+                  bts.Log.Gen.info($"PubSubMessageReactor._polling_thread_func _handle_client_msg sender: {kv.first} processing time: {stop_watch.ElapsedMilliseconds} msecs");
                }
-               handle_internal_pubs();
             } catch (Exception ex) {
                bts.Log.Gen.error($"PubSubMessageReactor._polling_thread_func ex: {ex}");
                if (ex is OutOfMemoryException) {
@@ -209,7 +222,7 @@ namespace BCG {
       }
       private void _handle_pub(string sender, PublishMsg pub_msg, bool is_internal_pub) {
          bts.Log.assert(!pub_msg.is_compressed);
-         //bts.Log.Gen.info($"PubSubMessageReactor._handle_pub sender: {sender} pub_msg: {pub_msg} is_internal_pub: {is_internal_pub}");
+         bts.Log.Gen.info($"PubSubMessageReactor._handle_pub sender: {sender} pub_msg: {pub_msg} is_internal_pub: {is_internal_pub}");
          // this is where pubsub's data state is updated
          _state.update_pub_data(pub_msg);
          ArgumentNullException.ThrowIfNull(pub_msg.tag);

src/DataStreamSuite/PubSub/Server/PubSubSingleTagStateEvent.cs

@@ -1,11 +1,12 @@
-using System.Text;
+using System.Text;
 using System.IO;
+
 namespace BCG;
 
 /// <summary>
 /// This class represents an event happening to a single tag in a pubsub (or substation).
 /// It could be that a tag is being published, or it could be a subscription to a tag or
-/// an unsubscription to a tag. 
+/// an unsubscription to a tag.
 /// </summary>
 public class PubSubSingleTagStateEvent {
    public PubSubSingleTagStateEvent(string tag, EventType eventType, string connection, string app) {
@@ -32,7 +33,6 @@ public class PubSubSingleTagStateEvent {
       return fromStream(mem);
    }
 
-   //for debugging
    public override string ToString() {
       var bd = new StringBuilder();
       bd.Append(tag + " ");
@@ -49,7 +49,6 @@ public class PubSubSingleTagStateEvent {
          case EventType.unsubscription:
             bd.Append("was unsubscribed to");
             break;
-
       }
       bd.Append(" app: " + app);
       bd.Append(" connection: " + connection);

src/DataStreamSuite/PubSub/Server/PubSubState.cs

@@ -1,4 +1,5 @@
 using bts;
+
 using Bts.PubSub;
 using Bts.PubSub.Server;
 
@@ -23,7 +24,7 @@ namespace BCG {
    /// </summary>
    /// 
    public class PubSubState : IPubSubState {
-
+      // TODO:remove events
       private event PubSubStateSingleTagChangeDelegate? onSingleTagChange;
       public event PubSubStateChangeDelegate? onTagMapChanged;
       public event PubSubStatsDelegate? onPublishStats;
@@ -95,6 +96,10 @@ namespace BCG {
 
       #endregion
 
+      private IPubSubServerTagHandler _create_handler(string tag, string tag_type) {
+         return null;
+      }
+
       #region data based state changes
       public void update_pub_data(bts.PublishMsg pub_msg) {
          ArgumentException.ThrowIfNullOrEmpty(pub_msg.tag);
@@ -164,6 +169,12 @@ namespace BCG {
             return _tag_to_data_map[tag].tag_type;
       }
 
+      public PubSubServerTagStats get_stats(string tag) {
+         if (!_tag_to_data_map.ContainsKey(tag))
+            return null;
+         return _tag_to_data_map[tag].get_stats();
+      }
+
       /// <summary>
       /// returns all the applications 'connected' to one tag, which
       /// includes a Dictionary of connection vs subscribers,

src/DataStreamSuite/PubSub/Server/SingleTagData.cs

@@ -7,10 +7,11 @@ using System.Collections.Generic;
 using BCG.Utilities;
 using mbts;
 using BCG;
+using Bts.PubSub.Server;
 
 namespace bts {
    /// <summary>
-   /// this is the data (not pub/subs/connections) for one tag. This exists in both the PublisherCOre on the 
+   /// this is the data (not pub/subs/connections) for one tag. This exists in both the PublisherCore on the 
    /// publisher side, as well as the PubSubServer
    /// 2 main functions
    /// 1. handle_publish, called in response to a pubsub publish
@@ -19,14 +20,14 @@ namespace bts {
    /// </summary>
    public class SingleTagData {
       private readonly IPubSubServerTagHandler? _tag_handler;
+      private readonly PubSubServerTagStats _stats = new();
 
       public SingleTagData(string tag, string tag_type, IPubSubServerTagHandler tag_handler) {
-         bts.Log.Gen.info($"SingleTagData constructed for tag: {tag} tag_type: {tag_type}");
+         bts.Log.Gen.info($"SingleTagData.cons tag: {tag} tag_type: {tag_type} tag_handler: {tag_handler.GetType()}");
          this.tag = tag;
          this.tag_type = tag_type;
          _tag_handler = tag_handler;
          Log.notnull(_tag_handler);
-         Log.Gen.info($"SingleTagData.cons tag: {tag} tag_type: {tag_type} _tag_handler: {_tag_handler}");
       }
 
       public string tag { get; }
@@ -38,6 +39,7 @@ namespace bts {
          string handler_str = _tag_handler != null ? _tag_handler.GetType().ToString() : "NULL";
          bts.Log.Gen.info($"SingleTagData.handle_publish tag: {tag} pub_msg: {pub_msg} handler: {handler_str} ");
          try {
+            _update_stats(pub_msg);
             // tag_handler may be null. If it throws (a null reference ex), we'll catch and handle
             // it like any other ex.
             _tag_handler!.update(tag, pub_msg.data_to_publish);
@@ -51,11 +53,35 @@ namespace bts {
          }
       }
 
+      void _update_stats(PublishMsg pub_msg) {
+         ArgumentNullException.ThrowIfNull(pub_msg.data_to_publish);
+
+         int update_size = pub_msg.data_to_publish.Length;
+         _stats.n_updates++;
+         _stats.total_bytes += update_size;
+         _stats.last_update_siz = update_size;
+         _stats.last_update_time = DateTime.UtcNow.Ticks;
+
+         if (_stats.n_updates == 1) {
+            _stats.min_size = update_size;
+            _stats.max_size = update_size;
+         } else {
+            _stats.min_size = Math.Min(_stats.min_size, update_size);
+            _stats.max_size = Math.Max(_stats.max_size, update_size);
+         }
+
+         _stats.average_size = (int)(_stats.total_bytes / _stats.n_updates);
+      }
+
       public byte[] get_current_content() {
          ArgumentNullException.ThrowIfNull(_tag_handler);
          var rv = _tag_handler.get_content(tag);
          Log.Gen.info($"SingleTagData.get_current_content tag: {tag} rv (n bytes): {rv.Length}");
          return rv;
       }
+
+      public PubSubServerTagStats get_stats() {
+         return _stats;
+      }
    }
 }

src/DataStreamSuite/PubSub/Services/PubSubServices.cs

@@ -11,11 +11,14 @@ using Bts.PubSub.Server;
 using Microsoft.Extensions.Configuration;
 using Microsoft.Extensions.DependencyInjection;
 using Microsoft.Extensions.Hosting;
+using Microsoft.Extensions.Options;
 
 using System;
+using System.Collections.Generic;
 using System.Linq;
 using System.Threading;
 using System.Threading.Tasks;
+using BlueOrb.PubSub.Server;
 
 public static class PubSubServices {
    // TODO: get rid of this static singleton. For now, we need it to support
@@ -49,9 +52,9 @@ public static class PubSubServices {
          .AddSingleton<IManagedSubscriber, ManagedSubscriber>(sp => sp.GetRequiredService<ManagedSubscriber>())
          .AddSingleton<IWorker, ManagedSubscriber>(sp => sp.GetRequiredService<ManagedSubscriber>());
 
-   public static IServiceCollection AddHeartbeatPublisher(this IServiceCollection services) =>
-      services.AddSingleton<SystemHeartbeatPublisher>()
-         .AddSingleton<IWorker, SystemHeartbeatPublisher>(sp => sp.GetRequiredService<SystemHeartbeatPublisher>());
+   //public static IServiceCollection AddHeartbeatPublisher(this IServiceCollection services) =>
+   // services.AddSingleton<SystemHeartbeatPublisher>()
+   //    .AddSingleton<IWorker, SystemHeartbeatPublisher>(sp => sp.GetRequiredService<SystemHeartbeatPublisher>());
 
    public static IServiceCollection AddPublisherCore(this IServiceCollection services) =>
       services.AddSingleton<IPublisherCore, PublisherCore>();
@@ -62,15 +65,15 @@ public static class PubSubServices {
    public static IServiceCollection AddPubSubState(this IServiceCollection services) =>
       services.AddSingleton<IPubSubState, PubSubState>();
 
-   public static IServiceCollection AddSubscriptionMapPublisher(this IServiceCollection services) =>
-      services.AddSingleton<SystemSubscriptionMapPublisher>()
-         .AddSingleton<IPubSubServerTagHandler, SystemSubscriptionMapPublisher>(sp => sp.GetRequiredService<SystemSubscriptionMapPublisher>())
-         .AddSingleton<IWorker, SystemSubscriptionMapPublisher>(sp => sp.GetRequiredService<SystemSubscriptionMapPublisher>());
+   // public static IServiceCollection AddSubscriptionMapPublisher(this IServiceCollection services) =>
+   //    services.AddSingleton<SystemSubscriptionMapPublisher>()
+   //       .AddSingleton<IPubSubServerTagHandler, SystemSubscriptionMapPublisher>(sp => sp.GetRequiredService<SystemSubscriptionMapPublisher>())
+   //       .AddSingleton<IWorker, SystemSubscriptionMapPublisher>(sp => sp.GetRequiredService<SystemSubscriptionMapPublisher>());
 
-   public static IServiceCollection AddSystemStatsPublisher(this IServiceCollection services) =>
-      services.AddSingleton<SystemStatsPublisher>()
-         .AddSingleton<IPubSubServerTagHandler, SystemStatsPublisher>(sp => sp.GetRequiredService<SystemStatsPublisher>())
-         .AddSingleton<IWorker, SystemStatsPublisher>(sp => sp.GetRequiredService<SystemStatsPublisher>());
+   // public static IServiceCollection AddSystemStatsPublisher(this IServiceCollection services) =>
+   //    services.AddSingleton<SystemStatsPublisher>()
+   //       .AddSingleton<IPubSubServerTagHandler, SystemStatsPublisher>(sp => sp.GetRequiredService<SystemStatsPublisher>())
+   //       .AddSingleton<IWorker, SystemStatsPublisher>(sp => sp.GetRequiredService<SystemStatsPublisher>());
 
    public static IServiceCollection AddTagHandlerFactory(this IServiceCollection services, IConfiguration configurationSection) =>
       services.Configure<PubSubTagHandlerFactoryConfig>(configurationSection)
@@ -82,16 +85,19 @@ public static class PubSubServices {
          .AddSingleton<IPubSubRouter, PubSubRouter>();
 
    public static IServiceCollection AddPubSubMessageReactor(this IServiceCollection services) =>
-      services.AddHostedService<PubSubMessageReactor>();
+      services.AddSingleton<IHostedService>(sp => new PubSubMessageReactor(
+         sp.GetRequiredService<IPubSubState>(),
+         new List<IPubSubInternalPublisher>(sp.GetServices<IPubSubInternalPublisher>()),
+         sp.GetRequiredService<IOptions<PubSubConfig>>().Value));
 
    public static IServiceCollection AddPubSubServer(this IServiceCollection services, IConfiguration configurationSection) {
       services.Configure<PubSubConfig>(configurationSection);
       PubSubConfig.SetGlobalConfig(configurationSection.Get<PubSubConfig>() ?? PubSubConfig.Default());
       return services.AddGlobalServiceProvider()
          .AddTagHandlerFactory(configurationSection.GetSection(nameof(PubSubConfig.TagHandlerFactory)))
-         .AddSystemStatsPublisher()
-         .AddHeartbeatPublisher()
-         .AddSubscriptionMapPublisher()
+         //.AddSystemStatsPublisher()
+         //.AddHeartbeatPublisher()
+         //.AddSubscriptionMapPublisher()
          .AddPubSubState()
          .AddInternalPublisher()
          .AddPubSubMessageReactor();

src/DataStreamSuite/PubSub/documentation/internal_readme.txt

@@ -47,9 +47,9 @@ SystemStatsPublisher.init(this);
 SystemHeartbeatPublisher.init();
 ---------------------------------------------------------------
 How internal tags are published?
-There are 3 internal publishers (ie compute somerthing an dpublish tit PubSubServer, such that clients can subscribe to it):
+There are 3 internal publishers (ie compute somerthing and publish it to PubSubServer, such that clients can subscribe to it):
 SystemStatsPublisher, SystemSubscriptionMapPublisher and SystemHeartbeatPublisher
-These are somewhat tricky because they are both listeners to PubUsbState event and intrnally publish, but are also IPubSubServerTagHandler
+These are somewhat tricky because they are both listeners to PubSubState event and intrnally publish, but are also IPubSubServerTagHandler
 SystemSubscriptionMapPublisher:
 just one instance, a static init is called by PubSubState and it passes itself. SystemSubscriptionMapPublisher subscribes to the event
 gets the affected tags, marks them dirty, in doWork creates the state and calls the internal_publish.

src/DataStreamSuite/PubSubCommon/.codexobj/PubSubCommon/PubSubCommon.csproj.nuget.dgspec.json

@@ -0,0 +1,160 @@
+{
+  "format": 1,
+  "restore": {
+    "C:\\Users\\Owner\\spark\\src\\DataStreamSuite\\PubSubCommon\\PubSubCommon.csproj": {}
+  },
+  "projects": {
+    "C:\\Users\\Owner\\spark\\src\\cs\\Bts.Spark.Core\\Bts.Spark.Core.csproj": {
+      "version": "3.2.0-preview",
+      "restore": {
+        "projectUniqueName": "C:\\Users\\Owner\\spark\\src\\cs\\Bts.Spark.Core\\Bts.Spark.Core.csproj",
+        "projectName": "Bts.Spark.Core",
+        "projectPath": "C:\\Users\\Owner\\spark\\src\\cs\\Bts.Spark.Core\\Bts.Spark.Core.csproj",
+        "packagesPath": "c:\\Users\\Owner\\spark\\.dotnet-cli\\.nuget\\packages\\",
+        "outputPath": "C:\\Users\\Owner\\spark\\src\\cs\\Bts.Spark.Core\\.codexobj\\PubSubCommon\\",
+        "projectStyle": "PackageReference",
+        "fallbackFolders": [
+          "C:\\Program Files (x86)\\Microsoft Visual Studio\\Shared\\NuGetPackages"
+        ],
+        "configFilePaths": [
+          "C:\\Users\\Owner\\AppData\\Roaming\\NuGet\\NuGet.Config",
+          "C:\\Program Files (x86)\\NuGet\\Config\\Microsoft.VisualStudio.FallbackLocation.config",
+          "C:\\Program Files (x86)\\NuGet\\Config\\Microsoft.VisualStudio.Offline.config"
+        ],
+        "originalTargetFrameworks": [
+          "net8.0"
+        ],
+        "sources": {
+          "C:\\Program Files (x86)\\Microsoft SDKs\\NuGetPackages\\": {},
+          "C:\\Program Files\\dotnet\\library-packs": {},
+          "https://api.nuget.org/v3/index.json": {}
+        },
+        "frameworks": {
+          "net8.0": {
+            "targetAlias": "net8.0",
+            "projectReferences": {}
+          }
+        },
+        "warningProperties": {
+          "allWarningsAsErrors": true
+        },
+        "restoreAuditProperties": {
+          "enableAudit": "false",
+          "auditLevel": "low",
+          "auditMode": "direct"
+        },
+        "SdkAnalysisLevel": "10.0.100"
+      },
+      "frameworks": {
+        "net8.0": {
+          "targetAlias": "net8.0",
+          "dependencies": {
+            "Google.Protobuf": {
+              "target": "Package",
+              "version": "[3.24.3, )"
+            },
+            "Microsoft.Extensions.Hosting": {
+              "target": "Package",
+              "version": "[10.0.0, )"
+            },
+            "System.Reactive": {
+              "target": "Package",
+              "version": "[6.0.0, )"
+            },
+            "protobuf-net": {
+              "target": "Package",
+              "version": "[3.2.26, )"
+            }
+          },
+          "imports": [
+            "net461",
+            "net462",
+            "net47",
+            "net471",
+            "net472",
+            "net48",
+            "net481"
+          ],
+          "assetTargetFallback": true,
+          "warn": true,
+          "frameworkReferences": {
+            "Microsoft.NETCore.App": {
+              "privateAssets": "all"
+            }
+          },
+          "runtimeIdentifierGraphPath": "C:\\Program Files\\dotnet\\sdk\\10.0.100/PortableRuntimeIdentifierGraph.json"
+        }
+      }
+    },
+    "C:\\Users\\Owner\\spark\\src\\DataStreamSuite\\PubSubCommon\\PubSubCommon.csproj": {
+      "version": "3.2.0-preview",
+      "restore": {
+        "projectUniqueName": "C:\\Users\\Owner\\spark\\src\\DataStreamSuite\\PubSubCommon\\PubSubCommon.csproj",
+        "projectName": "Bts.PubSubCommon",
+        "projectPath": "C:\\Users\\Owner\\spark\\src\\DataStreamSuite\\PubSubCommon\\PubSubCommon.csproj",
+        "packagesPath": "c:\\Users\\Owner\\spark\\.dotnet-cli\\.nuget\\packages\\",
+        "outputPath": "C:\\Users\\Owner\\spark\\src\\DataStreamSuite\\PubSubCommon\\.codexobj\\PubSubCommon\\",
+        "projectStyle": "PackageReference",
+        "fallbackFolders": [
+          "C:\\Program Files (x86)\\Microsoft Visual Studio\\Shared\\NuGetPackages"
+        ],
+        "configFilePaths": [
+          "C:\\Users\\Owner\\AppData\\Roaming\\NuGet\\NuGet.Config",
+          "C:\\Program Files (x86)\\NuGet\\Config\\Microsoft.VisualStudio.FallbackLocation.config",
+          "C:\\Program Files (x86)\\NuGet\\Config\\Microsoft.VisualStudio.Offline.config"
+        ],
+        "originalTargetFrameworks": [
+          "net8.0"
+        ],
+        "sources": {
+          "C:\\Program Files (x86)\\Microsoft SDKs\\NuGetPackages\\": {},
+          "C:\\Program Files\\dotnet\\library-packs": {},
+          "https://api.nuget.org/v3/index.json": {}
+        },
+        "frameworks": {
+          "net8.0": {
+            "targetAlias": "net8.0",
+            "projectReferences": {
+              "C:\\Users\\Owner\\spark\\src\\cs\\Bts.Spark.Core\\Bts.Spark.Core.csproj": {
+                "projectPath": "C:\\Users\\Owner\\spark\\src\\cs\\Bts.Spark.Core\\Bts.Spark.Core.csproj"
+              }
+            }
+          }
+        },
+        "warningProperties": {
+          "warnAsError": [
+            "NU1605"
+          ]
+        },
+        "restoreAuditProperties": {
+          "enableAudit": "false",
+          "auditLevel": "low",
+          "auditMode": "direct"
+        },
+        "SdkAnalysisLevel": "10.0.100"
+      },
+      "frameworks": {
+        "net8.0": {
+          "targetAlias": "net8.0",
+          "imports": [
+            "net461",
+            "net462",
+            "net47",
+            "net471",
+            "net472",
+            "net48",
+            "net481"
+          ],
+          "assetTargetFallback": true,
+          "warn": true,
+          "frameworkReferences": {
+            "Microsoft.NETCore.App": {
+              "privateAssets": "all"
+            }
+          },
+          "runtimeIdentifierGraphPath": "C:\\Program Files\\dotnet\\sdk\\10.0.100/PortableRuntimeIdentifierGraph.json"
+        }
+      }
+    }
+  }
+}

src/DataStreamSuite/PubSubCommon/.codexobj/PubSubCommon/PubSubCommon.csproj.nuget.g.props

@@ -0,0 +1,19 @@
+<?xml version="1.0" encoding="utf-8" standalone="no"?>
+<Project ToolsVersion="14.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
+  <PropertyGroup Condition=" '$(ExcludeRestorePackageImports)' != 'true' ">
+    <RestoreSuccess Condition=" '$(RestoreSuccess)' == '' ">True</RestoreSuccess>
+    <RestoreTool Condition=" '$(RestoreTool)' == '' ">NuGet</RestoreTool>
+    <ProjectAssetsFile Condition=" '$(ProjectAssetsFile)' == '' ">$(MSBuildThisFileDirectory)project.assets.json</ProjectAssetsFile>
+    <NuGetPackageRoot Condition=" '$(NuGetPackageRoot)' == '' ">$(UserProfile)\spark\.dotnet-cli\.nuget\packages\</NuGetPackageRoot>
+    <NuGetPackageFolders Condition=" '$(NuGetPackageFolders)' == '' ">c:\Users\Owner\spark\.dotnet-cli\.nuget\packages\;C:\Program Files (x86)\Microsoft Visual Studio\Shared\NuGetPackages</NuGetPackageFolders>
+    <NuGetProjectStyle Condition=" '$(NuGetProjectStyle)' == '' ">PackageReference</NuGetProjectStyle>
+    <NuGetToolVersion Condition=" '$(NuGetToolVersion)' == '' ">7.0.0</NuGetToolVersion>
+  </PropertyGroup>
+  <ItemGroup Condition=" '$(ExcludeRestorePackageImports)' != 'true' ">
+    <SourceRoot Include="c:\Users\Owner\spark\.dotnet-cli\.nuget\packages\" />
+    <SourceRoot Include="C:\Program Files (x86)\Microsoft Visual Studio\Shared\NuGetPackages\" />
+  </ItemGroup>
+  <ImportGroup Condition=" '$(ExcludeRestorePackageImports)' != 'true' ">
+    <Import Project="$(NuGetPackageRoot)microsoft.extensions.configuration.usersecrets\10.0.0\buildTransitive\net8.0\Microsoft.Extensions.Configuration.UserSecrets.props" Condition="Exists('$(NuGetPackageRoot)microsoft.extensions.configuration.usersecrets\10.0.0\buildTransitive\net8.0\Microsoft.Extensions.Configuration.UserSecrets.props')" />
+  </ImportGroup>
+</Project>

src/DataStreamSuite/PubSubCommon/.codexobj/PubSubCommon/PubSubCommon.csproj.nuget.g.targets

@@ -0,0 +1,10 @@
+<?xml version="1.0" encoding="utf-8" standalone="no"?>
+<Project ToolsVersion="14.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
+  <ImportGroup Condition=" '$(ExcludeRestorePackageImports)' != 'true' ">
+    <Import Project="$(NuGetPackageRoot)system.text.json\10.0.0\buildTransitive\net8.0\System.Text.Json.targets" Condition="Exists('$(NuGetPackageRoot)system.text.json\10.0.0\buildTransitive\net8.0\System.Text.Json.targets')" />
+    <Import Project="$(NuGetPackageRoot)microsoft.extensions.options\10.0.0\buildTransitive\net8.0\Microsoft.Extensions.Options.targets" Condition="Exists('$(NuGetPackageRoot)microsoft.extensions.options\10.0.0\buildTransitive\net8.0\Microsoft.Extensions.Options.targets')" />
+    <Import Project="$(NuGetPackageRoot)microsoft.extensions.configuration.binder\10.0.0\buildTransitive\netstandard2.0\Microsoft.Extensions.Configuration.Binder.targets" Condition="Exists('$(NuGetPackageRoot)microsoft.extensions.configuration.binder\10.0.0\buildTransitive\netstandard2.0\Microsoft.Extensions.Configuration.Binder.targets')" />
+    <Import Project="$(NuGetPackageRoot)microsoft.extensions.logging.abstractions\10.0.0\buildTransitive\net8.0\Microsoft.Extensions.Logging.Abstractions.targets" Condition="Exists('$(NuGetPackageRoot)microsoft.extensions.logging.abstractions\10.0.0\buildTransitive\net8.0\Microsoft.Extensions.Logging.Abstractions.targets')" />
+    <Import Project="$(NuGetPackageRoot)microsoft.extensions.configuration.usersecrets\10.0.0\buildTransitive\net8.0\Microsoft.Extensions.Configuration.UserSecrets.targets" Condition="Exists('$(NuGetPackageRoot)microsoft.extensions.configuration.usersecrets\10.0.0\buildTransitive\net8.0\Microsoft.Extensions.Configuration.UserSecrets.targets')" />
+  </ImportGroup>
+</Project>

src/DataStreamSuite/PubSubCommon/.codexobj/PubSubCommon/project.nuget.cache

@@ -0,0 +1,46 @@
+{
+  "version": 2,
+  "dgSpecHash": "dCKLVXTwwMs=",
+  "success": true,
+  "projectFilePath": "C:\\Users\\Owner\\spark\\src\\DataStreamSuite\\PubSubCommon\\PubSubCommon.csproj",
+  "expectedPackageFiles": [
+    "c:\\Users\\Owner\\spark\\.dotnet-cli\\.nuget\\packages\\google.protobuf\\3.24.3\\google.protobuf.3.24.3.nupkg.sha512",
+    "c:\\Users\\Owner\\spark\\.dotnet-cli\\.nuget\\packages\\microsoft.extensions.configuration\\10.0.0\\microsoft.extensions.configuration.10.0.0.nupkg.sha512",
+    "c:\\Users\\Owner\\spark\\.dotnet-cli\\.nuget\\packages\\microsoft.extensions.configuration.abstractions\\10.0.0\\microsoft.extensions.configuration.abstractions.10.0.0.nupkg.sha512",
+    "c:\\Users\\Owner\\spark\\.dotnet-cli\\.nuget\\packages\\microsoft.extensions.configuration.binder\\10.0.0\\microsoft.extensions.configuration.binder.10.0.0.nupkg.sha512",
+    "c:\\Users\\Owner\\spark\\.dotnet-cli\\.nuget\\packages\\microsoft.extensions.configuration.commandline\\10.0.0\\microsoft.extensions.configuration.commandline.10.0.0.nupkg.sha512",
+    "c:\\Users\\Owner\\spark\\.dotnet-cli\\.nuget\\packages\\microsoft.extensions.configuration.environmentvariables\\10.0.0\\microsoft.extensions.configuration.environmentvariables.10.0.0.nupkg.sha512",
+    "c:\\Users\\Owner\\spark\\.dotnet-cli\\.nuget\\packages\\microsoft.extensions.configuration.fileextensions\\10.0.0\\microsoft.extensions.configuration.fileextensions.10.0.0.nupkg.sha512",
+    "c:\\Users\\Owner\\spark\\.dotnet-cli\\.nuget\\packages\\microsoft.extensions.configuration.json\\10.0.0\\microsoft.extensions.configuration.json.10.0.0.nupkg.sha512",
+    "c:\\Users\\Owner\\spark\\.dotnet-cli\\.nuget\\packages\\microsoft.extensions.configuration.usersecrets\\10.0.0\\microsoft.extensions.configuration.usersecrets.10.0.0.nupkg.sha512",
+    "c:\\Users\\Owner\\spark\\.dotnet-cli\\.nuget\\packages\\microsoft.extensions.dependencyinjection\\10.0.0\\microsoft.extensions.dependencyinjection.10.0.0.nupkg.sha512",
+    "c:\\Users\\Owner\\spark\\.dotnet-cli\\.nuget\\packages\\microsoft.extensions.dependencyinjection.abstractions\\10.0.0\\microsoft.extensions.dependencyinjection.abstractions.10.0.0.nupkg.sha512",
+    "c:\\Users\\Owner\\spark\\.dotnet-cli\\.nuget\\packages\\microsoft.extensions.diagnostics\\10.0.0\\microsoft.extensions.diagnostics.10.0.0.nupkg.sha512",
+    "c:\\Users\\Owner\\spark\\.dotnet-cli\\.nuget\\packages\\microsoft.extensions.diagnostics.abstractions\\10.0.0\\microsoft.extensions.diagnostics.abstractions.10.0.0.nupkg.sha512",
+    "c:\\Users\\Owner\\spark\\.dotnet-cli\\.nuget\\packages\\microsoft.extensions.fileproviders.abstractions\\10.0.0\\microsoft.extensions.fileproviders.abstractions.10.0.0.nupkg.sha512",
+    "c:\\Users\\Owner\\spark\\.dotnet-cli\\.nuget\\packages\\microsoft.extensions.fileproviders.physical\\10.0.0\\microsoft.extensions.fileproviders.physical.10.0.0.nupkg.sha512",
+    "c:\\Users\\Owner\\spark\\.dotnet-cli\\.nuget\\packages\\microsoft.extensions.filesystemglobbing\\10.0.0\\microsoft.extensions.filesystemglobbing.10.0.0.nupkg.sha512",
+    "c:\\Users\\Owner\\spark\\.dotnet-cli\\.nuget\\packages\\microsoft.extensions.hosting\\10.0.0\\microsoft.extensions.hosting.10.0.0.nupkg.sha512",
+    "c:\\Users\\Owner\\spark\\.dotnet-cli\\.nuget\\packages\\microsoft.extensions.hosting.abstractions\\10.0.0\\microsoft.extensions.hosting.abstractions.10.0.0.nupkg.sha512",
+    "c:\\Users\\Owner\\spark\\.dotnet-cli\\.nuget\\packages\\microsoft.extensions.logging\\10.0.0\\microsoft.extensions.logging.10.0.0.nupkg.sha512",
+    "c:\\Users\\Owner\\spark\\.dotnet-cli\\.nuget\\packages\\microsoft.extensions.logging.abstractions\\10.0.0\\microsoft.extensions.logging.abstractions.10.0.0.nupkg.sha512",
+    "c:\\Users\\Owner\\spark\\.dotnet-cli\\.nuget\\packages\\microsoft.extensions.logging.configuration\\10.0.0\\microsoft.extensions.logging.configuration.10.0.0.nupkg.sha512",
+    "c:\\Users\\Owner\\spark\\.dotnet-cli\\.nuget\\packages\\microsoft.extensions.logging.console\\10.0.0\\microsoft.extensions.logging.console.10.0.0.nupkg.sha512",
+    "c:\\Users\\Owner\\spark\\.dotnet-cli\\.nuget\\packages\\microsoft.extensions.logging.debug\\10.0.0\\microsoft.extensions.logging.debug.10.0.0.nupkg.sha512",
+    "c:\\Users\\Owner\\spark\\.dotnet-cli\\.nuget\\packages\\microsoft.extensions.logging.eventlog\\10.0.0\\microsoft.extensions.logging.eventlog.10.0.0.nupkg.sha512",
+    "c:\\Users\\Owner\\spark\\.dotnet-cli\\.nuget\\packages\\microsoft.extensions.logging.eventsource\\10.0.0\\microsoft.extensions.logging.eventsource.10.0.0.nupkg.sha512",
+    "c:\\Users\\Owner\\spark\\.dotnet-cli\\.nuget\\packages\\microsoft.extensions.options\\10.0.0\\microsoft.extensions.options.10.0.0.nupkg.sha512",
+    "c:\\Users\\Owner\\spark\\.dotnet-cli\\.nuget\\packages\\microsoft.extensions.options.configurationextensions\\10.0.0\\microsoft.extensions.options.configurationextensions.10.0.0.nupkg.sha512",
+    "c:\\Users\\Owner\\spark\\.dotnet-cli\\.nuget\\packages\\microsoft.extensions.primitives\\10.0.0\\microsoft.extensions.primitives.10.0.0.nupkg.sha512",
+    "c:\\Users\\Owner\\spark\\.dotnet-cli\\.nuget\\packages\\protobuf-net\\3.2.26\\protobuf-net.3.2.26.nupkg.sha512",
+    "c:\\Users\\Owner\\spark\\.dotnet-cli\\.nuget\\packages\\protobuf-net.core\\3.2.26\\protobuf-net.core.3.2.26.nupkg.sha512",
+    "c:\\Users\\Owner\\spark\\.dotnet-cli\\.nuget\\packages\\system.collections.immutable\\7.0.0\\system.collections.immutable.7.0.0.nupkg.sha512",
+    "c:\\Users\\Owner\\spark\\.dotnet-cli\\.nuget\\packages\\system.diagnostics.diagnosticsource\\10.0.0\\system.diagnostics.diagnosticsource.10.0.0.nupkg.sha512",
+    "c:\\Users\\Owner\\spark\\.dotnet-cli\\.nuget\\packages\\system.diagnostics.eventlog\\10.0.0\\system.diagnostics.eventlog.10.0.0.nupkg.sha512",
+    "c:\\Users\\Owner\\spark\\.dotnet-cli\\.nuget\\packages\\system.io.pipelines\\10.0.0\\system.io.pipelines.10.0.0.nupkg.sha512",
+    "c:\\Users\\Owner\\spark\\.dotnet-cli\\.nuget\\packages\\system.reactive\\6.0.0\\system.reactive.6.0.0.nupkg.sha512",
+    "c:\\Users\\Owner\\spark\\.dotnet-cli\\.nuget\\packages\\system.text.encodings.web\\10.0.0\\system.text.encodings.web.10.0.0.nupkg.sha512",
+    "c:\\Users\\Owner\\spark\\.dotnet-cli\\.nuget\\packages\\system.text.json\\10.0.0\\system.text.json.10.0.0.nupkg.sha512"
+  ],
+  "logs": []
+}

src/DataStreamSuite/PubSubCommon/.codexobj/PubSubCommonOnly/PubSubCommon.csproj.nuget.dgspec.json

@@ -0,0 +1,160 @@
+{
+  "format": 1,
+  "restore": {
+    "C:\\Users\\Owner\\spark\\src\\DataStreamSuite\\PubSubCommon\\PubSubCommon.csproj": {}
+  },
+  "projects": {
+    "C:\\Users\\Owner\\spark\\src\\cs\\Bts.Spark.Core\\Bts.Spark.Core.csproj": {
+      "version": "3.2.0-preview",
+      "restore": {
+        "projectUniqueName": "C:\\Users\\Owner\\spark\\src\\cs\\Bts.Spark.Core\\Bts.Spark.Core.csproj",
+        "projectName": "Bts.Spark.Core",
+        "projectPath": "C:\\Users\\Owner\\spark\\src\\cs\\Bts.Spark.Core\\Bts.Spark.Core.csproj",
+        "packagesPath": "c:\\Users\\Owner\\spark\\.dotnet-cli\\.nuget\\packages\\",
+        "outputPath": "C:\\Users\\Owner\\spark\\src\\cs\\Bts.Spark.Core\\.codexobj\\PubSubCommonOnly\\",
+        "projectStyle": "PackageReference",
+        "fallbackFolders": [
+          "C:\\Program Files (x86)\\Microsoft Visual Studio\\Shared\\NuGetPackages"
+        ],
+        "configFilePaths": [
+          "C:\\Users\\Owner\\AppData\\Roaming\\NuGet\\NuGet.Config",
+          "C:\\Program Files (x86)\\NuGet\\Config\\Microsoft.VisualStudio.FallbackLocation.config",
+          "C:\\Program Files (x86)\\NuGet\\Config\\Microsoft.VisualStudio.Offline.config"
+        ],
+        "originalTargetFrameworks": [
+          "net8.0"
+        ],
+        "sources": {
+          "C:\\Program Files (x86)\\Microsoft SDKs\\NuGetPackages\\": {},
+          "C:\\Program Files\\dotnet\\library-packs": {},
+          "https://api.nuget.org/v3/index.json": {}
+        },
+        "frameworks": {
+          "net8.0": {
+            "targetAlias": "net8.0",
+            "projectReferences": {}
+          }
+        },
+        "warningProperties": {
+          "allWarningsAsErrors": true
+        },
+        "restoreAuditProperties": {
+          "enableAudit": "false",
+          "auditLevel": "low",
+          "auditMode": "direct"
+        },
+        "SdkAnalysisLevel": "10.0.100"
+      },
+      "frameworks": {
+        "net8.0": {
+          "targetAlias": "net8.0",
+          "dependencies": {
+            "Google.Protobuf": {
+              "target": "Package",
+              "version": "[3.24.3, )"
+            },
+            "Microsoft.Extensions.Hosting": {
+              "target": "Package",
+              "version": "[10.0.0, )"
+            },
+            "System.Reactive": {
+              "target": "Package",
+              "version": "[6.0.0, )"
+            },
+            "protobuf-net": {
+              "target": "Package",
+              "version": "[3.2.26, )"
+            }
+          },
+          "imports": [
+            "net461",
+            "net462",
+            "net47",
+            "net471",
+            "net472",
+            "net48",
+            "net481"
+          ],
+          "assetTargetFallback": true,
+          "warn": true,
+          "frameworkReferences": {
+            "Microsoft.NETCore.App": {
+              "privateAssets": "all"
+            }
+          },
+          "runtimeIdentifierGraphPath": "C:\\Program Files\\dotnet\\sdk\\10.0.100/PortableRuntimeIdentifierGraph.json"
+        }
+      }
+    },
+    "C:\\Users\\Owner\\spark\\src\\DataStreamSuite\\PubSubCommon\\PubSubCommon.csproj": {
+      "version": "3.2.0-preview",
+      "restore": {
+        "projectUniqueName": "C:\\Users\\Owner\\spark\\src\\DataStreamSuite\\PubSubCommon\\PubSubCommon.csproj",
+        "projectName": "Bts.PubSubCommon",
+        "projectPath": "C:\\Users\\Owner\\spark\\src\\DataStreamSuite\\PubSubCommon\\PubSubCommon.csproj",
+        "packagesPath": "c:\\Users\\Owner\\spark\\.dotnet-cli\\.nuget\\packages\\",
+        "outputPath": "C:\\Users\\Owner\\spark\\src\\DataStreamSuite\\PubSubCommon\\.codexobj\\PubSubCommonOnly\\",
+        "projectStyle": "PackageReference",
+        "fallbackFolders": [
+          "C:\\Program Files (x86)\\Microsoft Visual Studio\\Shared\\NuGetPackages"
+        ],
+        "configFilePaths": [
+          "C:\\Users\\Owner\\AppData\\Roaming\\NuGet\\NuGet.Config",
+          "C:\\Program Files (x86)\\NuGet\\Config\\Microsoft.VisualStudio.FallbackLocation.config",
+          "C:\\Program Files (x86)\\NuGet\\Config\\Microsoft.VisualStudio.Offline.config"
+        ],
+        "originalTargetFrameworks": [
+          "net8.0"
+        ],
+        "sources": {
+          "C:\\Program Files (x86)\\Microsoft SDKs\\NuGetPackages\\": {},
+          "C:\\Program Files\\dotnet\\library-packs": {},
+          "https://api.nuget.org/v3/index.json": {}
+        },
+        "frameworks": {
+          "net8.0": {
+            "targetAlias": "net8.0",
+            "projectReferences": {
+              "C:\\Users\\Owner\\spark\\src\\cs\\Bts.Spark.Core\\Bts.Spark.Core.csproj": {
+                "projectPath": "C:\\Users\\Owner\\spark\\src\\cs\\Bts.Spark.Core\\Bts.Spark.Core.csproj"
+              }
+            }
+          }
+        },
+        "warningProperties": {
+          "warnAsError": [
+            "NU1605"
+          ]
+        },
+        "restoreAuditProperties": {
+          "enableAudit": "false",
+          "auditLevel": "low",
+          "auditMode": "direct"
+        },
+        "SdkAnalysisLevel": "10.0.100"
+      },
+      "frameworks": {
+        "net8.0": {
+          "targetAlias": "net8.0",
+          "imports": [
+            "net461",
+            "net462",
+            "net47",
+            "net471",
+            "net472",
+            "net48",
+            "net481"
+          ],
+          "assetTargetFallback": true,
+          "warn": true,
+          "frameworkReferences": {
+            "Microsoft.NETCore.App": {
+              "privateAssets": "all"
+            }
+          },
+          "runtimeIdentifierGraphPath": "C:\\Program Files\\dotnet\\sdk\\10.0.100/PortableRuntimeIdentifierGraph.json"
+        }
+      }
+    }
+  }
+}

src/DataStreamSuite/PubSubCommon/.codexobj/PubSubCommonOnly/PubSubCommon.csproj.nuget.g.props

@@ -0,0 +1,19 @@
+<?xml version="1.0" encoding="utf-8" standalone="no"?>
+<Project ToolsVersion="14.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
+  <PropertyGroup Condition=" '$(ExcludeRestorePackageImports)' != 'true' ">
+    <RestoreSuccess Condition=" '$(RestoreSuccess)' == '' ">True</RestoreSuccess>
+    <RestoreTool Condition=" '$(RestoreTool)' == '' ">NuGet</RestoreTool>
+    <ProjectAssetsFile Condition=" '$(ProjectAssetsFile)' == '' ">$(MSBuildThisFileDirectory)project.assets.json</ProjectAssetsFile>
+    <NuGetPackageRoot Condition=" '$(NuGetPackageRoot)' == '' ">$(UserProfile)\spark\.dotnet-cli\.nuget\packages\</NuGetPackageRoot>
+    <NuGetPackageFolders Condition=" '$(NuGetPackageFolders)' == '' ">c:\Users\Owner\spark\.dotnet-cli\.nuget\packages\;C:\Program Files (x86)\Microsoft Visual Studio\Shared\NuGetPackages</NuGetPackageFolders>
+    <NuGetProjectStyle Condition=" '$(NuGetProjectStyle)' == '' ">PackageReference</NuGetProjectStyle>
+    <NuGetToolVersion Condition=" '$(NuGetToolVersion)' == '' ">7.0.0</NuGetToolVersion>
+  </PropertyGroup>
+  <ItemGroup Condition=" '$(ExcludeRestorePackageImports)' != 'true' ">
+    <SourceRoot Include="c:\Users\Owner\spark\.dotnet-cli\.nuget\packages\" />
+    <SourceRoot Include="C:\Program Files (x86)\Microsoft Visual Studio\Shared\NuGetPackages\" />
+  </ItemGroup>
+  <ImportGroup Condition=" '$(ExcludeRestorePackageImports)' != 'true' ">
+    <Import Project="$(NuGetPackageRoot)microsoft.extensions.configuration.usersecrets\10.0.0\buildTransitive\net8.0\Microsoft.Extensions.Configuration.UserSecrets.props" Condition="Exists('$(NuGetPackageRoot)microsoft.extensions.configuration.usersecrets\10.0.0\buildTransitive\net8.0\Microsoft.Extensions.Configuration.UserSecrets.props')" />
+  </ImportGroup>
+</Project>

src/DataStreamSuite/PubSubCommon/.codexobj/PubSubCommonOnly/PubSubCommon.csproj.nuget.g.targets

@@ -0,0 +1,10 @@
+<?xml version="1.0" encoding="utf-8" standalone="no"?>
+<Project ToolsVersion="14.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
+  <ImportGroup Condition=" '$(ExcludeRestorePackageImports)' != 'true' ">
+    <Import Project="$(NuGetPackageRoot)system.text.json\10.0.0\buildTransitive\net8.0\System.Text.Json.targets" Condition="Exists('$(NuGetPackageRoot)system.text.json\10.0.0\buildTransitive\net8.0\System.Text.Json.targets')" />
+    <Import Project="$(NuGetPackageRoot)microsoft.extensions.options\10.0.0\buildTransitive\net8.0\Microsoft.Extensions.Options.targets" Condition="Exists('$(NuGetPackageRoot)microsoft.extensions.options\10.0.0\buildTransitive\net8.0\Microsoft.Extensions.Options.targets')" />
+    <Import Project="$(NuGetPackageRoot)microsoft.extensions.configuration.binder\10.0.0\buildTransitive\netstandard2.0\Microsoft.Extensions.Configuration.Binder.targets" Condition="Exists('$(NuGetPackageRoot)microsoft.extensions.configuration.binder\10.0.0\buildTransitive\netstandard2.0\Microsoft.Extensions.Configuration.Binder.targets')" />
+    <Import Project="$(NuGetPackageRoot)microsoft.extensions.logging.abstractions\10.0.0\buildTransitive\net8.0\Microsoft.Extensions.Logging.Abstractions.targets" Condition="Exists('$(NuGetPackageRoot)microsoft.extensions.logging.abstractions\10.0.0\buildTransitive\net8.0\Microsoft.Extensions.Logging.Abstractions.targets')" />
+    <Import Project="$(NuGetPackageRoot)microsoft.extensions.configuration.usersecrets\10.0.0\buildTransitive\net8.0\Microsoft.Extensions.Configuration.UserSecrets.targets" Condition="Exists('$(NuGetPackageRoot)microsoft.extensions.configuration.usersecrets\10.0.0\buildTransitive\net8.0\Microsoft.Extensions.Configuration.UserSecrets.targets')" />
+  </ImportGroup>
+</Project>

src/DataStreamSuite/PubSubCommon/.codexobj/PubSubCommonOnly/project.nuget.cache

@@ -0,0 +1,46 @@
+{
+  "version": 2,
+  "dgSpecHash": "KMZlkNqmRvs=",
+  "success": true,
+  "projectFilePath": "C:\\Users\\Owner\\spark\\src\\DataStreamSuite\\PubSubCommon\\PubSubCommon.csproj",
+  "expectedPackageFiles": [
+    "c:\\Users\\Owner\\spark\\.dotnet-cli\\.nuget\\packages\\google.protobuf\\3.24.3\\google.protobuf.3.24.3.nupkg.sha512",
+    "c:\\Users\\Owner\\spark\\.dotnet-cli\\.nuget\\packages\\microsoft.extensions.configuration\\10.0.0\\microsoft.extensions.configuration.10.0.0.nupkg.sha512",
+    "c:\\Users\\Owner\\spark\\.dotnet-cli\\.nuget\\packages\\microsoft.extensions.configuration.abstractions\\10.0.0\\microsoft.extensions.configuration.abstractions.10.0.0.nupkg.sha512",
+    "c:\\Users\\Owner\\spark\\.dotnet-cli\\.nuget\\packages\\microsoft.extensions.configuration.binder\\10.0.0\\microsoft.extensions.configuration.binder.10.0.0.nupkg.sha512",
+    "c:\\Users\\Owner\\spark\\.dotnet-cli\\.nuget\\packages\\microsoft.extensions.configuration.commandline\\10.0.0\\microsoft.extensions.configuration.commandline.10.0.0.nupkg.sha512",
+    "c:\\Users\\Owner\\spark\\.dotnet-cli\\.nuget\\packages\\microsoft.extensions.configuration.environmentvariables\\10.0.0\\microsoft.extensions.configuration.environmentvariables.10.0.0.nupkg.sha512",
+    "c:\\Users\\Owner\\spark\\.dotnet-cli\\.nuget\\packages\\microsoft.extensions.configuration.fileextensions\\10.0.0\\microsoft.extensions.configuration.fileextensions.10.0.0.nupkg.sha512",
+    "c:\\Users\\Owner\\spark\\.dotnet-cli\\.nuget\\packages\\microsoft.extensions.configuration.json\\10.0.0\\microsoft.extensions.configuration.json.10.0.0.nupkg.sha512",
+    "c:\\Users\\Owner\\spark\\.dotnet-cli\\.nuget\\packages\\microsoft.extensions.configuration.usersecrets\\10.0.0\\microsoft.extensions.configuration.usersecrets.10.0.0.nupkg.sha512",
+    "c:\\Users\\Owner\\spark\\.dotnet-cli\\.nuget\\packages\\microsoft.extensions.dependencyinjection\\10.0.0\\microsoft.extensions.dependencyinjection.10.0.0.nupkg.sha512",
+    "c:\\Users\\Owner\\spark\\.dotnet-cli\\.nuget\\packages\\microsoft.extensions.dependencyinjection.abstractions\\10.0.0\\microsoft.extensions.dependencyinjection.abstractions.10.0.0.nupkg.sha512",
+    "c:\\Users\\Owner\\spark\\.dotnet-cli\\.nuget\\packages\\microsoft.extensions.diagnostics\\10.0.0\\microsoft.extensions.diagnostics.10.0.0.nupkg.sha512",
+    "c:\\Users\\Owner\\spark\\.dotnet-cli\\.nuget\\packages\\microsoft.extensions.diagnostics.abstractions\\10.0.0\\microsoft.extensions.diagnostics.abstractions.10.0.0.nupkg.sha512",
+    "c:\\Users\\Owner\\spark\\.dotnet-cli\\.nuget\\packages\\microsoft.extensions.fileproviders.abstractions\\10.0.0\\microsoft.extensions.fileproviders.abstractions.10.0.0.nupkg.sha512",
+    "c:\\Users\\Owner\\spark\\.dotnet-cli\\.nuget\\packages\\microsoft.extensions.fileproviders.physical\\10.0.0\\microsoft.extensions.fileproviders.physical.10.0.0.nupkg.sha512",
+    "c:\\Users\\Owner\\spark\\.dotnet-cli\\.nuget\\packages\\microsoft.extensions.filesystemglobbing\\10.0.0\\microsoft.extensions.filesystemglobbing.10.0.0.nupkg.sha512",
+    "c:\\Users\\Owner\\spark\\.dotnet-cli\\.nuget\\packages\\microsoft.extensions.hosting\\10.0.0\\microsoft.extensions.hosting.10.0.0.nupkg.sha512",
+    "c:\\Users\\Owner\\spark\\.dotnet-cli\\.nuget\\packages\\microsoft.extensions.hosting.abstractions\\10.0.0\\microsoft.extensions.hosting.abstractions.10.0.0.nupkg.sha512",
+    "c:\\Users\\Owner\\spark\\.dotnet-cli\\.nuget\\packages\\microsoft.extensions.logging\\10.0.0\\microsoft.extensions.logging.10.0.0.nupkg.sha512",
+    "c:\\Users\\Owner\\spark\\.dotnet-cli\\.nuget\\packages\\microsoft.extensions.logging.abstractions\\10.0.0\\microsoft.extensions.logging.abstractions.10.0.0.nupkg.sha512",
+    "c:\\Users\\Owner\\spark\\.dotnet-cli\\.nuget\\packages\\microsoft.extensions.logging.configuration\\10.0.0\\microsoft.extensions.logging.configuration.10.0.0.nupkg.sha512",
+    "c:\\Users\\Owner\\spark\\.dotnet-cli\\.nuget\\packages\\microsoft.extensions.logging.console\\10.0.0\\microsoft.extensions.logging.console.10.0.0.nupkg.sha512",
+    "c:\\Users\\Owner\\spark\\.dotnet-cli\\.nuget\\packages\\microsoft.extensions.logging.debug\\10.0.0\\microsoft.extensions.logging.debug.10.0.0.nupkg.sha512",
+    "c:\\Users\\Owner\\spark\\.dotnet-cli\\.nuget\\packages\\microsoft.extensions.logging.eventlog\\10.0.0\\microsoft.extensions.logging.eventlog.10.0.0.nupkg.sha512",
+    "c:\\Users\\Owner\\spark\\.dotnet-cli\\.nuget\\packages\\microsoft.extensions.logging.eventsource\\10.0.0\\microsoft.extensions.logging.eventsource.10.0.0.nupkg.sha512",
+    "c:\\Users\\Owner\\spark\\.dotnet-cli\\.nuget\\packages\\microsoft.extensions.options\\10.0.0\\microsoft.extensions.options.10.0.0.nupkg.sha512",
+    "c:\\Users\\Owner\\spark\\.dotnet-cli\\.nuget\\packages\\microsoft.extensions.options.configurationextensions\\10.0.0\\microsoft.extensions.options.configurationextensions.10.0.0.nupkg.sha512",
+    "c:\\Users\\Owner\\spark\\.dotnet-cli\\.nuget\\packages\\microsoft.extensions.primitives\\10.0.0\\microsoft.extensions.primitives.10.0.0.nupkg.sha512",
+    "c:\\Users\\Owner\\spark\\.dotnet-cli\\.nuget\\packages\\protobuf-net\\3.2.26\\protobuf-net.3.2.26.nupkg.sha512",
+    "c:\\Users\\Owner\\spark\\.dotnet-cli\\.nuget\\packages\\protobuf-net.core\\3.2.26\\protobuf-net.core.3.2.26.nupkg.sha512",
+    "c:\\Users\\Owner\\spark\\.dotnet-cli\\.nuget\\packages\\system.collections.immutable\\7.0.0\\system.collections.immutable.7.0.0.nupkg.sha512",
+    "c:\\Users\\Owner\\spark\\.dotnet-cli\\.nuget\\packages\\system.diagnostics.diagnosticsource\\10.0.0\\system.diagnostics.diagnosticsource.10.0.0.nupkg.sha512",
+    "c:\\Users\\Owner\\spark\\.dotnet-cli\\.nuget\\packages\\system.diagnostics.eventlog\\10.0.0\\system.diagnostics.eventlog.10.0.0.nupkg.sha512",
+    "c:\\Users\\Owner\\spark\\.dotnet-cli\\.nuget\\packages\\system.io.pipelines\\10.0.0\\system.io.pipelines.10.0.0.nupkg.sha512",
+    "c:\\Users\\Owner\\spark\\.dotnet-cli\\.nuget\\packages\\system.reactive\\6.0.0\\system.reactive.6.0.0.nupkg.sha512",
+    "c:\\Users\\Owner\\spark\\.dotnet-cli\\.nuget\\packages\\system.text.encodings.web\\10.0.0\\system.text.encodings.web.10.0.0.nupkg.sha512",
+    "c:\\Users\\Owner\\spark\\.dotnet-cli\\.nuget\\packages\\system.text.json\\10.0.0\\system.text.json.10.0.0.nupkg.sha512"
+  ],
+  "logs": []
+}

src/DataStreamSuite/PubSubServer/Program.cs

@@ -1,5 +1,8 @@
 using Bts.Extensions.Hosting;
+using Bts.PubSub;
 using Bts.PubSub.Services;
+using Bts.PubSub.Server;
+using BlueOrb.PubSub.Server;
 
 using Microsoft.Extensions.Configuration;
 using Microsoft.Extensions.DependencyInjection;
@@ -8,6 +11,9 @@ using Microsoft.Extensions.Hosting;
 using Spark.Core.Services;
 
 using System;
+using System.Collections.Generic;
+using BCG;
+using bts;
 
 var host = HostBuilderExtensions.CreateBtsBuilder(args)
    .ConfigureServices(ConfigureServices)
@@ -32,6 +38,36 @@ static void ConfigureServices(HostBuilderContext context, IServiceCollection ser
    /// constructors that accept these types as arguments.
    services.AddSpark(context.Configuration.GetSection("Spark"))
       .AddMasterWorker(context.Configuration.GetSection("MasterWorker"))
+      .AddSingleton<PubSubServerHBPublisher>()
+      .AddSingleton<IPubSubInternalPublisher>(sp => sp.GetRequiredService<PubSubServerHBPublisher>())
+      .AddSingleton<PubSubServerStatsPublisher>()
+      .AddSingleton<IPubSubInternalPublisher>(sp => sp.GetRequiredService<PubSubServerStatsPublisher>())
+      .AddSingleton<PubSubServerStatePublisher>()
+      .AddSingleton<IPubSubInternalPublisher>(sp => sp.GetRequiredService<PubSubServerStatePublisher>())
+      .AddSingleton<RestTableUpdateReceiver>()
+      .AddSingleton<IPubSubInternalPublisher>(sp => sp.GetRequiredService<RestTableUpdateReceiver>())
+      .AddSingleton<WebSocketTableUpdateReceiver>()
+      .AddSingleton<IPubSubInternalPublisher>(sp => sp.GetRequiredService<WebSocketTableUpdateReceiver>())
       .AddPubSubServer(context.Configuration.GetSection("PubSub"))
       .AddGarbageCollector(context.Configuration.GetSection("GarbageCollector"));
 }
+
+static void init_oldway(HostBuilderContext context, IServiceCollection services) {
+   var pubSubSection = context.Configuration.GetSection("PubSub");
+   var pubSubConfig = pubSubSection.Get<PubSubConfig>() ?? PubSubConfig.Default();
+   PubSubConfig.SetGlobalConfig(pubSubConfig);
+
+   services.Configure<PubSubTagHandlerFactoryConfig>(pubSubSection.GetSection(nameof(PubSubConfig.TagHandlerFactory)));
+   services.AddSingleton<IHostedService>(sp => {
+      var tagHandlerFactory = new PubsubTagHandlerFactory(sp, pubSubConfig.TagHandlerFactory!);
+      var state = new PubSubState(tagHandlerFactory);
+      var internalPublishers = new List<IPubSubInternalPublisher> {
+         new PubSubServerHBPublisher(),
+         new PubSubServerStatsPublisher(state),
+         new PubSubServerStatePublisher(state),
+         new RestTableUpdateReceiver(),
+         new WebSocketTableUpdateReceiver(),
+      };
+      return new PubSubMessageReactor(state, internalPublishers, pubSubConfig);
+   });
+}

src/DataStreamSuite/PubSubServer/PubSubMessageReactorFactory.cs

@@ -0,0 +1,27 @@
+namespace Bts.PubSubServer;
+
+using BCG;
+
+using Bts.PubSub;
+using Bts.PubSub.Server;
+
+using Microsoft.Extensions.DependencyInjection;
+using System;
+using System.Collections.Generic;
+using bts;
+using BlueOrb.PubSub.Server;
+
+public static class PubSubMessageReactorFactory {
+   public static PubSubMessageReactor Create(IServiceProvider serviceProvider, PubSubConfig config) {
+      var tagHandlerFactory = ActivatorUtilities.CreateInstance<PubsubTagHandlerFactory>(serviceProvider, config.TagHandlerFactory!);
+      var state = new PubSubState(tagHandlerFactory);
+      var internalPublishers = new List<IPubSubInternalPublisher> {
+         new PubSubServerHBPublisher(),
+         new PubSubServerStatsPublisher(state),
+         new PubSubServerStatePublisher(state),
+         new RestTableUpdateReceiver(),
+         new WebSocketTableUpdateReceiver(),
+      };
+      return new PubSubMessageReactor(state, internalPublishers, config);
+   }
+}

src/DataStreamSuite/PubSubServer/PubSubServer.csproj

@@ -10,7 +10,8 @@
     <When Condition="'$(UseProjectRefs)' == 'true'">
       <ItemGroup>
         <ProjectReference Include="..\..\cs\Bts.Extensions.Hosting\Bts.Extensions.Hosting.csproj" />
-        <ProjectReference Include="..\PubSub\PubSub.csproj" />
+        <ProjectReference Include="..\PubSub\PubSub.csproj" /> 
+        <ProjectReference Include="..\DataStreamCore\DataStreamCore.csproj" />       
       </ItemGroup>
     </When>
     <Otherwise>

src/DataStreamSuite/PubSubServer/PubSubServerHBPublisher.cs

@@ -0,0 +1,50 @@
+namespace BlueOrb.PubSub.Server;
+
+using BCG;
+
+using System;
+using System.Linq;
+using System.Net;
+
+using bts;
+
+public class PubSubServerHBPublisher : IPubSubInternalPublisher {
+   readonly IBlueTableSchema _schema = BlueDataUtility.form_schema("PubSubServerHeartbeat",
+      Tuple.Create("id", TypeCode.Int32),
+      Tuple.Create("ip", TypeCode.String),
+      Tuple.Create("hb_time", TypeCode.Int64));
+
+   readonly TimeSpan _publish_interval = TimeSpan.FromMilliseconds(250);
+   readonly IStreamIOMsgSerializer _serializer = new UniversalStreamIOSerializer();
+   DateTime _last_send_time = DateTime.MinValue;
+
+   public event Action<string, string, byte[]>? onNewData;
+
+   public void doWork() {
+      if (DateTime.Now - _last_send_time > _publish_interval) {
+         _publish();
+      }
+   }
+
+   void _publish() {
+      bts.Log.Gen.info("PubSubServerHBPublisher._publish");
+      var table = new BlueTable(_schema);
+      var row = new MutableBlueRow(_schema);
+      row.setValue("id", 0);
+      row.setValue("ip", _get_machine_ip());
+      row.setValue("hb_time", DateTimeOffset.UtcNow.ToUnixTimeMilliseconds());
+      table.addRow(row);
+
+      StreamIOMessage.CreateMessage create_msg = new(table);
+      StreamIOMessage stream_io_msg = new(create_msg);
+      byte[] msg_bytes = _serializer.serialize(stream_io_msg);
+      onNewData?.Invoke("DS_ps_heartbeats", "DS_", msg_bytes);
+
+      _last_send_time = DateTime.Now;
+   }
+
+   static string _get_machine_ip() =>
+      Dns.GetHostEntry(Dns.GetHostName()).AddressList
+         .FirstOrDefault(addr => addr.AddressFamily == System.Net.Sockets.AddressFamily.InterNetwork)?
+         .ToString() ?? "127.0.0.1";
+}

src/DataStreamSuite/PubSubServer/PubSubServerStatePublisher.cs

@@ -0,0 +1,90 @@
+namespace BlueOrb.PubSub.Server;
+
+using BCG;
+
+using Bts.PubSub.Server;
+
+using mbts;
+
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Net;
+
+using bts;
+using BlueOrb.Data;
+
+
+public class PubSubServerStatePublisher : IPubSubInternalPublisher {
+   readonly IBlueTableSchema _schema = BlueDataUtility.form_schema("PubSubServerState",
+      Tuple.Create("id", TypeCode.Int32),
+      Tuple.Create("tag", TypeCode.String),
+      Tuple.Create("ps_server", TypeCode.String),
+      Tuple.Create("subscribers", TypeCode.String),
+      Tuple.Create("publishers", TypeCode.String),
+      Tuple.Create("dead_publishers", TypeCode.String));
+
+   readonly IPubSubState _state;
+   IBlueRowCollection _last_fired_table = null;
+   readonly TimeSpan _publish_interval = TimeSpan.FromMilliseconds(1000);
+   readonly IStreamIOMsgSerializer _serializer = new UniversalStreamIOSerializer();
+   DateTime _last_send_time = DateTime.MinValue;
+
+   public PubSubServerStatePublisher(IPubSubState state) {
+      _state = state;
+   }
+
+   public event Action<string, string, byte[]>? onNewData;
+
+   public void doWork() {
+      if (DateTime.Now - _last_send_time > _publish_interval) {
+         _publish();
+      }
+   }
+
+   void _publish() {
+      bts.Log.Gen.info("PubSubServerStatePublisher._publish");
+      var table = new BlueTable(_schema);
+      int id = 0;
+      string ps_server = _get_machine_ip();
+
+      foreach (string tag in _state.all_tags.OrderBy(t => t)) {
+         _state.get_all_connections(tag, out var subscribers, out var active_publishers, out var dead_publishers);
+
+         var row = new MutableBlueRow(_schema);
+         row.setValue("id", id++);
+         row.setValue("tag", tag);
+         row.setValue("ps_server", ps_server);
+         row.setValue("subscribers", _join_values(subscribers.Values));
+         row.setValue("publishers", _join_values(active_publishers.Values));
+         row.setValue("dead_publishers", _join_dead_publishers(dead_publishers));
+         table.addRow(row);
+      }
+      IBlueRowCollection d = null, a = null;
+      table.compare_tables_deep(_last_fired_table, out d, out a);
+      if (d.Count == 0 && a.Count == 0)
+         return;
+
+      StreamIOMessage.CreateMessage create_msg = new(table);
+      StreamIOMessage stream_io_msg = new(create_msg);
+      byte[] msg_bytes = _serializer.serialize(stream_io_msg);
+      onNewData?.Invoke("DS_ps_server_state", "DS_", msg_bytes);
+      _last_fired_table = table;
+      _last_send_time = DateTime.Now;
+   }
+
+   static string _join_values(IEnumerable<string> values) =>
+      String.Join(",", values.Where(v => !String.IsNullOrEmpty(v)).Distinct().OrderBy(v => v));
+
+   static string _join_dead_publishers(IEnumerable<Pair<string, string>> dead_publishers) =>
+      String.Join(",", dead_publishers
+         .Select(p => p.second)
+         .Where(v => !String.IsNullOrEmpty(v))
+         .Distinct()
+         .OrderBy(v => v));
+
+   static string _get_machine_ip() =>
+      Dns.GetHostEntry(Dns.GetHostName()).AddressList
+         .FirstOrDefault(addr => addr.AddressFamily == System.Net.Sockets.AddressFamily.InterNetwork)?
+         .ToString() ?? "127.0.0.1";
+}

src/DataStreamSuite/PubSubServer/PubSubServerStatsPublisher.cs

@@ -0,0 +1,72 @@
+namespace BlueOrb.PubSub.Server;
+
+using BCG;
+
+using Bts.PubSub.Server;
+
+using System;
+using System.Linq;
+using bts;
+
+public class PubSubServerStatsPublisher : IPubSubInternalPublisher {
+   readonly IBlueTableSchema _schema = BlueDataUtility.form_schema("PubSubServerStats",
+      Tuple.Create("id", TypeCode.Int32),
+      Tuple.Create("server", TypeCode.String),
+      Tuple.Create("tag", TypeCode.String),
+      Tuple.Create("n_updates", TypeCode.Int32),
+      Tuple.Create("total_bytes", TypeCode.Int64),
+      Tuple.Create("min_size", TypeCode.Int32),
+      Tuple.Create("max_size", TypeCode.Int32),
+      Tuple.Create("last_update_siz", TypeCode.Int32),
+      Tuple.Create("average_size", TypeCode.Int32),
+      Tuple.Create("last_update_time", TypeCode.Int64));
+
+   readonly IPubSubState _state;
+   readonly TimeSpan _publish_interval = TimeSpan.FromMilliseconds(1000);
+   readonly IStreamIOMsgSerializer _serializer = new UniversalStreamIOSerializer();
+   DateTime _last_send_time = DateTime.MinValue;
+
+   public PubSubServerStatsPublisher(IPubSubState state) {
+      _state = state;
+   }
+
+   public event Action<string, string, byte[]>? onNewData;
+
+   public void doWork() {
+      if (DateTime.Now - _last_send_time > _publish_interval) {
+         _publish();
+      }
+   }
+
+   void _publish() {
+      bts.Log.Gen.info("PubSubServerStatsPublisher._publish");
+      var table = new BlueTable(_schema);
+      int id = 0;
+
+      foreach (string tag in _state.all_tags.OrderBy(t => t)) {
+         var stats = _state.get_stats(tag);
+         if (stats is null)
+            continue;
+
+         var row = new MutableBlueRow(_schema);
+         row.setValue("id", id++);
+         row.setValue("server", Environment.MachineName);
+         row.setValue("tag", tag);
+         row.setValue("n_updates", stats.n_updates);
+         row.setValue("total_bytes", stats.total_bytes);
+         row.setValue("min_size", stats.min_size);
+         row.setValue("max_size", stats.max_size);
+         row.setValue("last_update_siz", stats.last_update_siz);
+         row.setValue("average_size", stats.average_size);
+         row.setValue("last_update_time", stats.last_update_time);
+         table.addRow(row);
+      }
+
+      StreamIOMessage.CreateMessage create_msg = new(table);
+      StreamIOMessage stream_io_msg = new(create_msg);
+      byte[] msg_bytes = _serializer.serialize(stream_io_msg);
+      onNewData?.Invoke("DS_ps_server_stats", "DS_", msg_bytes);
+
+      _last_send_time = DateTime.Now;
+   }
+}

src/DataStreamSuite/PubSubServer/RestClient/create_table_client.py

@@ -0,0 +1,76 @@
+import argparse
+import json
+import sys
+import urllib.error
+import urllib.request
+
+
+class RestPubTableBroker:
+   def __init__(self, url: str):
+      self._url = url
+
+   def create_table(self, table_name: str, rows: list[dict]) -> None:
+      payload = {
+         "op": "create_table",
+         "table_name": table_name,
+         "rows": rows,
+      }
+      self._post_json(payload)
+
+   def _post_json(self, payload: dict) -> None:
+      body = json.dumps(payload).encode("utf-8")
+      request = urllib.request.Request(
+         self._url,
+         data=body,
+         headers={"Content-Type": "application/json"},
+         method="POST",
+      )
+      with urllib.request.urlopen(request, timeout=10) as response:
+         response_body = response.read().decode("utf-8", errors="replace")
+         print(f"status={response.status} body={response_body}")
+
+
+def _build_sample_rows() -> list[dict]:
+   return [
+      {"id": 0, "i": 10, "d": 10.5, "s": "row_0"},
+      {"id": 1, "i": 20, "d": 20.25, "s": "row_1"},
+      {"id": 2, "i": 30, "d": 30.75, "s": "row_2"},
+   ]
+
+
+def main() -> int:
+   parser = argparse.ArgumentParser(
+      description="POST a create_table-style JSON payload to PubSubServer."
+   )
+   parser.add_argument(
+      "--url",
+      default="http://localhost:19091/",
+      help="REST listener URL. Default: http://localhost:19091/",
+   )
+   parser.add_argument(
+      "--table-name",
+      default="DS_rest_test_table",
+      help="Logical table name to include in the JSON payload.",
+   )
+   args = parser.parse_args()
+
+   broker = RestPubTableBroker(args.url)
+   rows = _build_sample_rows()
+
+   print(f"POST {args.url}")
+   print(json.dumps({"op": "create_table", "table_name": args.table_name, "rows": rows}, indent=2))
+
+   try:
+      broker.create_table(args.table_name, rows)
+      return 0
+   except urllib.error.HTTPError as ex:
+      body = ex.read().decode("utf-8", errors="replace")
+      print(f"http error status={ex.code} body={body}", file=sys.stderr)
+      return 1
+   except urllib.error.URLError as ex:
+      print(f"url error: {ex}", file=sys.stderr)
+      return 1
+
+
+if __name__ == "__main__":
+   raise SystemExit(main())