DataStreamCore Json stuff

* IsSocketStateOpen
* IsClosedProperly

.editorconfig

@@ -207,17 +207,23 @@ dotnet_naming_symbols.parameters.applicable_accessibilities = public,internal,pr
 dotnet_naming_symbols.private_symbols.applicable_kinds = property,method,field,event,delegate
 dotnet_naming_symbols.private_symbols.applicable_accessibilities = protected,private
 
+# local symbols
+dotnet_naming_symbols.local_symbols.applicable_kinds = local
+
+# parameter symbols
+dotnet_naming_symbols.parameter_symbols.applicable_kinds = parameter
+
 # Naming styles
 
-dotnet_naming_style.snake_case.required_prefix =
-dotnet_naming_style.snake_case.required_suffix =
-dotnet_naming_style.snake_case.word_separator = _
-dotnet_naming_style.snake_case.capitalization = all_lower
+dotnet_naming_style.pascal_case.required_prefix =
+dotnet_naming_style.pascal_case.required_suffix =
+dotnet_naming_style.pascal_case.word_separator =
+dotnet_naming_style.pascal_case.capitalization = pascal_case
 
-dotnet_naming_style.private_snake_case.required_prefix = _
-dotnet_naming_style.private_snake_case.required_suffix =
-dotnet_naming_style.private_snake_case.word_separator = _
-dotnet_naming_style.private_snake_case.capitalization = all_lower
+dotnet_naming_style.camel_case.required_prefix =
+dotnet_naming_style.camel_case.required_suffix =
+dotnet_naming_style.camel_case.word_separator =
+dotnet_naming_style.camel_case.capitalization = camel_case
 
 dotnet_naming_style.user_types.required_prefix =
 dotnet_naming_style.user_types.required_suffix =
@@ -229,6 +235,10 @@ dotnet_naming_style.interface.required_suffix =
 dotnet_naming_style.interface.word_separator =
 dotnet_naming_style.interface.capitalization = pascal_case
 
+dotnet_naming_style.local.capitalization = camel_case
+
+dotnet_naming_style.parameter.capitalization = camel_case
+
 # rules
 dotnet_naming_rule.user_types.symbols = user_types
 dotnet_naming_rule.user_types.style = user_types
@@ -238,14 +248,18 @@ dotnet_naming_rule.interfaces.symbols = interface
 dotnet_naming_rule.interfaces.style = interface_style
 dotnet_naming_rule.interfaces.severity = suggestion
 
-dotnet_naming_rule.public_members_snake_case.symbols = public_symbols
-dotnet_naming_rule.public_members_snake_case.style = snake_case
-dotnet_naming_rule.public_members_snake_case.severity = suggestion
+dotnet_naming_rule.public_members.symbols = public_symbols
+dotnet_naming_rule.public_members.style = pascal_case
+dotnet_naming_rule.public_members.severity = suggestion
 
-dotnet_naming_rule.private_members_snake_case.symbols = private_symbols
-dotnet_naming_rule.private_members_snake_case.style = private_snake_case
-dotnet_naming_rule.private_members_snake_case.severity = suggestion
+dotnet_naming_rule.private_members.symbols = private_symbols
+dotnet_naming_rule.private_members.style = camel_case
+dotnet_naming_rule.private_members.severity = suggestion
 
-dotnet_naming_rule.parameters.symbols = parameters
-dotnet_naming_rule.parameters.style = snake_case
+dotnet_naming_rule.variables_are_camel_case.severity = suggestion
+dotnet_naming_rule.variables_are_camel_case.symbols = local_symbols
+dotnet_naming_rule.variables_are_camel_case.style = local
+
+dotnet_naming_rule.parameters.symbols = parameter_symbols
+dotnet_naming_rule.parameters.style = parameter
 dotnet_naming_rule.parameters.severity = suggestion

.gitignore

@@ -47,8 +47,8 @@ trace_data/
 /src/.clang-format
 /src/server/sparkd/cmds/spartan/analysis/.ipynb_checkpoints
 /src/server/sparkd/cmds/spartan/analysis/data
-*.tmp/*
-/.venv*/*
+.tmp/
+.venv*/
 .cache/
 /src/Spark.Admin.WebApp/tsconfig.*.g.json
 /src/Spark.Admin.WebApp/src/bin/

CMakePresets.json

@@ -29,8 +29,8 @@
         "CMAKE_TOOLCHAIN_FILE": "${sourceDir}/vcpkg/scripts/buildsystems/vcpkg.cmake",
         "EXTERNALS_ROOT": "${sourceDir}/externals/windows",
         "CONFIG_DEFAULTS_PATH": "config/profiles/default/*.conf",
-        "DOTNET_BINARY_DIR": "${sourceDir}/build/$<CONFIG>/bin/net7.0",
-        "DOTNET_WINDOWS_BINARY_DIR": "${sourceDir}/build/$<CONFIG>/bin/net7.0-windows"
+        "DOTNET_BINARY_DIR": "${sourceDir}/build/$<CONFIG>/bin/net8.0",
+        "DOTNET_WINDOWS_BINARY_DIR": "${sourceDir}/build/$<CONFIG>/bin/net8.0-windows"
       }
     },
 
@@ -197,7 +197,7 @@
         "CMAKE_CXX_FLAGS_RELEASE": "-fPIC -Wall -pedantic -Wextra -Werror -Wno-missing-braces -Wno-ignored-qualifiers --std=c++20 -fdiagnostics-color -fno-omit-frame-pointer -DNDEBUG -O3",
         "CMAKE_INSTALL_PREFIX": "${sourceDir}/install/${presetName}",
         "CONFIG_DEFAULTS_PATH": "config/profiles/default/*.conf",
-        "DOTNET_BINARY_DIR": "${sourceDir}/build/$<CONFIG>/bin/net7.0"
+        "DOTNET_BINARY_DIR": "${sourceDir}/build/$<CONFIG>/bin/net8.0"
       }
     },
 

docker/sparkd/dev/Dockerfile

@@ -16,7 +16,7 @@ RUN apt-get update && apt-get install -y \
     sqlite3 \
     tzdata
 
-# clang
+# clang 13, 15 (so we can build old branches)
 RUN apt-get update && apt-get install -y \
     clang-13 \
     clang-format-13 \
@@ -25,6 +25,19 @@ RUN apt-get update && apt-get install -y \
     clang-format-15 \
     clangd-15
 
+# clang 17 (so we can build new branches)
+RUN \
+    curl -fsSL https://apt.llvm.org/llvm-snapshot.gpg.key 2> /dev/null \
+    | gpg --dearmor - \
+    | tee /usr/share/keyrings/llvm-snapshot.gpg > /dev/null \
+    && echo "deb [signed-by=/usr/share/keyrings/llvm-snapshot.gpg] http://apt.llvm.org/jammy/ llvm-toolchain-jammy-17 main" \
+    | tee -a "/etc/apt/sources.list.d/llvm.list" \
+    && apt-get update \
+    && apt-get install -y \
+    clang-17 \
+    clang-format-17 \
+    clangd-17
+
 # g++
 RUN apt-get update && apt-get install -y \
     g++-11 \
@@ -37,8 +50,12 @@ RUN apt-get update && apt-get install -y \
     && python3 -m pip install --upgrade pip
 
 # dotnet
-RUN apt-get update && apt-get install -y \
-    dotnet7
+RUN \
+   curl -fsSLO "https://packages.microsoft.com/config/ubuntu/$(lsb_release -rs)/packages-microsoft-prod.deb" \
+   && dpkg -i packages-microsoft-prod.deb \
+   && rm packages-microsoft-prod.deb \
+   && apt-get update \
+   && apt-get install -y dotnet-sdk-8.0
 
 # ssh
 RUN apt-get update && apt-get install -y \
@@ -53,10 +70,6 @@ RUN \
     && echo "deb [signed-by=/usr/share/keyrings/kitware-archive-keyring.gpg] https://apt.kitware.com/ubuntu/ $(lsb_release -cs) main" \
     | tee /etc/apt/sources.list.d/kitware.list > /dev/null \
     && apt-get update \
-    && rm /usr/share/keyrings/kitware-archive-keyring.gpg \
-    && apt-get install -y kitware-archive-keyring \
-    && echo "deb [signed-by=/usr/share/keyrings/kitware-archive-keyring.gpg] https://apt.kitware.com/ubuntu/ $(lsb_release -cs)-rc main" \
-    | tee -a /etc/apt/sources.list.d/kitware.list >/dev/null \
     && apt-get install -y cmake
 
 # node
@@ -95,6 +108,7 @@ RUN apt-get update && apt-get install -y \
 # dev utils
 RUN apt-get update && apt-get install -y \ 
     fish \
+    gdb \
     git-all \
     sudo \
     vim \
@@ -102,12 +116,12 @@ RUN apt-get update && apt-get install -y \
 
 # set up default executables (update-alternatives)
 RUN \
-    update-alternatives --install /usr/bin/clang++ clang++ /usr/bin/clang++-13 100; \
-    update-alternatives --install /usr/bin/clang clang /usr/bin/clang-13 100; \
-    update-alternatives --install /usr/bin/clang-format clang-format /usr/bin/clang-format-13 100; \
-    update-alternatives --install /usr/bin/clangd clangd /usr/bin/clangd-13 100; \
-    update-alternatives --install /usr/bin/g++ g++ /usr/bin/g++-11 100; \
-    update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-11 100
+    update-alternatives --install /usr/bin/clang++ clang++ /usr/bin/clang++-17 100; \
+    update-alternatives --install /usr/bin/clang clang /usr/bin/clang-17 100; \
+    update-alternatives --install /usr/bin/clang-format clang-format /usr/bin/clang-format-17 100; \
+    update-alternatives --install /usr/bin/clangd clangd /usr/bin/clangd-17 100; \
+    update-alternatives --install /usr/bin/g++ g++ /usr/bin/g++-12 100; \
+    update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-12 100
 
 # Add compiler symlinks for ccache 
 RUN \
@@ -117,6 +131,8 @@ RUN \
     ln -s /usr/bin/ccache /usr/local/bin/clang++-13; \
     ln -s /usr/bin/ccache /usr/local/bin/clang-15; \
     ln -s /usr/bin/ccache /usr/local/bin/clang++-15; \
+    ln -s /usr/bin/ccache /usr/local/bin/clang-17; \
+    ln -s /usr/bin/ccache /usr/local/bin/clang++-17; \
     ln -s /usr/bin/ccache /usr/local/bin/gcc; \
     ln -s /usr/bin/ccache /usr/local/bin/g++; \
     ln -s /usr/bin/ccache /usr/local/bin/gcc-11; \

src/DataStreamSuite/ArithmeticExpressions/ArithmeticExpressions.csproj

@@ -1,7 +1,7 @@
 <Project Sdk="Microsoft.NET.Sdk">
 
   <PropertyGroup>
-    <TargetFramework>net7.0</TargetFramework>
+    <TargetFramework>net8.0</TargetFramework>
     <ImplicitUsings>enable</ImplicitUsings>
     <Nullable>enable</Nullable>
   </PropertyGroup>

src/DataStreamSuite/BaseUtilities/BaseUtilities.csproj

@@ -1,7 +1,7 @@
 <Project Sdk="Microsoft.NET.Sdk">
 
   <PropertyGroup>
-    <TargetFramework>net7.0</TargetFramework>
+    <TargetFramework>net8.0</TargetFramework>
     <ImplicitUsings>enable</ImplicitUsings>
     <Nullable>enable</Nullable>
   </PropertyGroup>

src/DataStreamSuite/BlueRegistry/BlueRegistry.csproj

@@ -1,7 +1,7 @@
 <Project Sdk="Microsoft.NET.Sdk">
 
   <PropertyGroup>
-    <TargetFramework>net7.0-windows</TargetFramework>
+    <TargetFramework>net8.0-windows</TargetFramework>
     <ImplicitUsings>enable</ImplicitUsings>
     <Nullable>enable</Nullable>
     <UseWindowsForms>true</UseWindowsForms>

src/DataStreamSuite/BooleanExpressions/BooleanExpressions.csproj

@@ -1,7 +1,7 @@
 <Project Sdk="Microsoft.NET.Sdk">
 
   <PropertyGroup>
-    <TargetFramework>net7.0</TargetFramework>
+    <TargetFramework>net8.0</TargetFramework>
     <ImplicitUsings>enable</ImplicitUsings>
     <Nullable>enable</Nullable>
   </PropertyGroup>

src/DataStreamSuite/Bts.DataStream.Websocket/Bts.DataStream.Websocket.csproj

@@ -1,7 +1,7 @@
 <Project Sdk="Microsoft.NET.Sdk">
 
   <PropertyGroup>
-    <TargetFramework>net7.0</TargetFramework>
+    <TargetFramework>net8.0</TargetFramework>
     <ImplicitUsings>enable</ImplicitUsings>
     <Nullable>enable</Nullable>
   </PropertyGroup>

src/DataStreamSuite/Bts.DataStream.Websocket/DataStreamDataSync.cs

@@ -0,0 +1,119 @@
+namespace Bts.DataStream.Websocket;
+
+using BCG;
+
+using bts;
+using bts.collections;
+using bts.websocket_core;
+
+using Bts.DataStreamExpressions;
+using Bts.Websocket;
+
+using mbts.utils;
+
+using Spark.Core.Disposables;
+
+using System.Text.Json;
+using System.Text.Json.Nodes;
+
+public class DataStreamDataSync(JsonSerializerOptions? serializerOpts = null) : JsonRpcDataSync(serializerOpts) {
+   readonly Dictionary<WebsocketClientID, List<ClientSubscription>> clients = [];
+
+   protected override void onConnect(WebsocketConnectNotification notification) {
+      var id = notification.Id;
+      lock (lockObj) {
+         clients.Add(id, []);
+      }
+      Log.Api.info($"DataStreamWebsocketDataSync client connected. clientId: {notification.Id}");
+      if (notification.Message is not null) {
+         onIncomingMessage(notification.Message);
+      }
+   }
+
+   protected override void onDisconnect(WebsocketDisconnectNotification notification) {
+      Log.Api.warn($"DataStreamWebsocketDataSync client disconnected. clientId: {notification.Id}");
+      unsubscribeAll(notification.Id);
+      clients.Remove(notification.Id);
+   }
+
+   protected override void onRequest(WebsocketClientID id, JsonRpcRequest<JsonObject?> req) {
+      switch (req.Method!.ToLower()) {
+         case "subscribe":
+            subscribe(id, req);
+            break;
+         case "unsubscribe":
+            unsubscribe(id, req);
+            break;
+         case "unsubscribe_all":
+            unsubscribeAll(id);
+            break;
+         default:
+            Log.Api.error($"Received request of unknown type: {req}");
+            break;
+      }
+   }
+
+   void subscribe(WebsocketClientID id, JsonRpcRequest<JsonObject?> req) {
+      var subParams = req.Params!.Deserialize<SubscriptionParams>()!;
+      var exp = DataStreamExpressionsInterpreter.Evaluate(subParams.Query!);
+      var stream = exp.getView(new Dictionary<string, StatementExpression>(0));
+      var sub = new ClientSubscription(id, subParams, Disposable.Create(Dispose));
+      lock (lockObj) {
+         clients[id].Add(sub);
+         sendResponse(id, req.Id!.Value, req.Params);
+         stream.onChange += OnChange;
+      }
+
+      void OnChange(object? newValue) {
+         var dp = ExceptionAssert.of_type<DataPacket>(newValue);
+         var rpcParams = new {
+            dp.addedRows,
+            dp.deletedRows
+         };
+         sendNotification(id, "subscription", rpcParams);
+      }
+
+      void Dispose() =>
+         stream.onChange -= OnChange;
+   }
+
+   void unsubscribe(WebsocketClientID id, JsonRpcRequest<JsonObject?> req) {
+      var subParams = req.Params!.Deserialize<SubscriptionParams>();
+      var subParamsStr = JsonSerializer.Serialize(subParams, serializerOpts);
+      // Minimize amount of code inside lock (just need it to find/remove the subscription)
+      List<ClientSubscription>? subs = null;
+      ClientSubscription? sub = null;
+      lock (lockObj) {
+         subs = clients.GetValueOrDefault(id);
+         subs?.TryRemoveFirst(s => s!.Params.Equals(subParams), out sub);
+      }
+      if (subs is null) {
+         // Didn't find client (log/throw).
+         var msg = $"Unable to unsubscribe: did not find client. Id: {id}.\nParams:\n{subParamsStr}";
+         Log.Api.error(msg);
+         throw new Exception(msg);
+      }
+      if (sub is null) {
+         // Found client, but didn't find subscription (log/throw).
+         var msg = $"Unable to unsubscribe: did not find the specified subscription. " +
+            $"Id: {id}. Request:\n{subParams}";
+         Log.Api.error(msg);
+         throw new Exception(msg);
+      }
+      // Found client and subscription (dispose/remove subscription).
+      Log.Api.info($"Unsubscribing client. Id: {id}.\nParams:\n{subParamsStr}");
+      sub.Dispose();
+   }
+
+   void unsubscribeAll(WebsocketClientID id) {
+      lock (lockObj) {
+         if (!clients.TryGetValue(id, out var subs)) {
+            throw new Exception(
+               $"Unable to unsubscribe: did not find client with id {id}");
+         }
+         subs.DisposeAll();
+         subs.Clear();
+      }
+   }
+
+}

src/DataStreamSuite/Bts.DataStream.Websocket/DataSync.cs

@@ -1,229 +1,228 @@
-namespace Bts.DataStream.Websocket;
-
-using BCG;
-
-using bts;
-using bts.api.core;
-using bts.collections;
-
-using Bts.DataStreamExpressions;
-using Bts.Websocket;
-
-using mbts.utils;
-
-using Spark.Core.Disposables;
-
-using System.Reactive.Subjects;
-using System.Text.Json;
-using System.Text.Json.Nodes;
-
-/// <summary>
-/// Represents an I/O model (<see cref="ISubject{TSource, TResult}"/>), that accepts websocket
-/// requests (from clients) as input, and emits data/notifications (to clients) as output.
-/// This class's chief role is managing data subscriptions to sparkd on the clients' behalf.
-/// </summary>
-public class DataSync(ServiceContext serviceCtx, JsonSerializerOptions? serializerOpts = null) :
-   ISubject<WebsocketConnectionUpdate, IEnumerable<WebsocketOutgoingMessage>> {
-   readonly ServiceContext serviceCtx = serviceCtx;
-   readonly JsonSerializerOptions serializerOpts = serializerOpts ?? JsonSerializerOptions.Default;
-   readonly Dictionary<WebsocketClientID, List<ClientSubscription>> clients = [];
-   readonly object lockObj = new();
-
-   event Action<IEnumerable<WebsocketOutgoingMessage>>? requestSend;
-
-   public void OnCompleted() =>
-      Log.Api.info("DataSync incoming request stream has completed.");
-
-   public void OnError(Exception error) =>
-      Log.Api.error($"DataSync error: {error}");
-
-   public void OnNext(WebsocketConnectionUpdate value) {
-      switch (value.Type) {
-         case WebsocketConnectionUpdateType.IncomingMessage:
-            onIncomingMessage(value.IncomingMessage);
-            break;
-         case WebsocketConnectionUpdateType.Connect:
-            onConnect(value.Connect);
-            break;
-         case WebsocketConnectionUpdateType.Disconnect:
-            onDisconnect(value.Disconnect);
-            break;
-         default:
-            Log.assert(false, $"Unexpected WebsocketConnectionUpdate type: {value.Type}");
-            break;
-      }
-   }
-
-   public IDisposable Subscribe(IObserver<IEnumerable<WebsocketOutgoingMessage>> observer) {
-      lock (lockObj) {
-         requestSend += observer.OnNext;
-         return Disposable.Create(dispose);
-      }
-      void dispose() {
-         lock (lockObj) {
-            requestSend -= observer.OnNext;
-         }
-      }
-   }
-
-   static JsonRpcId? extractRpcId(string messageBody) {
-      try {
-         return JsonNode.Parse(messageBody)
-            ?.AsObject()
-            ?.GetValueOrDefault("id")
-            ?.Deserialize<JsonRpcId>();
-      } catch {
-         return null;
-      }
-   }
-
-   void onConnect(WebsocketConnectNotification notification) {
-      var id = notification.Id;
-      lock (lockObj) {
-         clients.Add(id, new());
-      }
-      Log.Api.info($"DataStreamWebsocketDataSync client connected. clientId: {notification.Id}");
-      if (notification.Message is not null) {
-         onIncomingMessage(notification.Message);
-      }
-   }
-
-   void onDisconnect(WebsocketDisconnectNotification notification) {
-      Log.Api.warn($"DataStreamWebsocketDataSync client disconnected. clientId: {notification.Id}");
-      unsubscribeAll(notification.Id);
-      clients.Remove(notification.Id);
-   }
-
-   void onIncomingMessage(WebsocketIncomingMessage message) {
-      var id = message.sender_client_id;
-      if (!JsonRpcRequest.TryDeserialize(message, out var req, out var err)) {
-         onBadRequest(message, err);
-         return;
-      }
-      switch (req.Method!.ToLower()) {
-         case "subscribe":
-            subscribe(id, req);
-            break;
-         case "unsubscribe":
-            unsubscribe(id, req);
-            break;
-         case "unsubscribe_all":
-            unsubscribeAll(id);
-            break;
-         default:
-            Log.Api.error($"Received request of unknown type: {req}");
-            break;
-      }
-   }
-
-   void sendNotification<T>(WebsocketClientID id, string method, T rpcParams) {
-      var rpcNotification = JsonRpcRequest.CreateNotification(method, rpcParams);
-      var rpcNotificationStr = JsonSerializer.Serialize(rpcNotification, serializerOpts);
-      send(new WebsocketOutgoingMessage[] { new(id, rpcNotificationStr) });
-   }
-
-   void sendResponse<T>(WebsocketClientID id, JsonRpcId rpcId, T rpcParams) {
-      var rpcRes = JsonRpcResponse.Create(rpcId, rpcParams);
-      var rpcResStr = JsonSerializer.Serialize(rpcRes, serializerOpts);
-      send(new WebsocketOutgoingMessage[] { new(id, rpcResStr )});
-   }
-
-   void sendError<T>(WebsocketClientID id, JsonRpcId rpcId, JsonRpcError<T> error) {
-      var rpcRes = JsonRpcResponse.CreateError(rpcId, error);
-      var rpcResStr = JsonSerializer.Serialize(rpcRes, serializerOpts);
-      send(new WebsocketOutgoingMessage[] { new(id, rpcResStr) });
-   }
-
-   bool send(IEnumerable<WebsocketOutgoingMessage> messages) {
-      lock (lockObj) {
-         if (requestSend is null) {
-            Log.Api.warn(
-               $"Unable to send message(s) because '{nameof(requestSend)}' handler is null");
-            return false;
-         }
-         requestSend(messages);
-         return true;
-      }
-   }
-
-   void subscribe(WebsocketClientID id, JsonRpcRequest<JsonObject?> req) {
-      var subParams = req.Params!.Deserialize<SubscriptionParams>()!;
-      var exp = DataStreamExpressionsInterpreter.Evaluate(subParams.Query!);
-      var stream = exp.getView(new Dictionary<string, StatementExpression>(0));
-      var sub = new ClientSubscription(id, subParams, Disposable.Create(dispose));
-      lock (lockObj) {
-         clients[id].Add(sub);
-         sendResponse(id, req.Id!.Value, req.Params);
-         stream.onChange += onChange;
-      }
-
-      void onChange(object? newValue) {
-         var dp = ExceptionAssert.of_type<DataPacket>(newValue);
-         var rpcParams = new {
-            addedRows = dp.addedRows,
-            deletedRows = dp.deletedRows
-         };
-         sendNotification(id, "subscription", rpcParams);
-      }
-
-      void dispose() =>
-         stream.onChange -= onChange;
-   }
-
-   void unsubscribe(WebsocketClientID id, JsonRpcRequest<JsonObject?> req) {
-      var subParams = req.Params!.Deserialize<SubscriptionParams>();
-      var subParamsStr = JsonSerializer.Serialize(subParams, serializerOpts);
-      // Minimize amount of code inside lock (just need it to find/remove the subscription)
-      List<ClientSubscription>? subs = null;
-      ClientSubscription? sub = null;
-      lock (lockObj) {
-         subs = clients.GetValueOrDefault(id);
-         subs?.TryRemoveFirst(s => s!.Params.Equals(subParams), out sub);
-      }
-      if (subs is null) {
-         // Didn't find client (log/throw).
-         var msg = $"Unable to unsubscribe: did not find client. Id: {id}.\nParams:\n{subParamsStr}";
-         Log.Api.error(msg);
-         throw new Exception(msg);
-      }
-      if (sub is null) {
-         // Found client, but didn't find subscription (log/throw).
-         var msg = $"Unable to unsubscribe: did not find the specified subscription. " +
-            $"Id: {id}. Request:\n{subParams}";
-         Log.Api.error(msg);
-         throw new Exception(msg);
-      }
-      // Found client and subscription (dispose/remove subscription).
-      Log.Api.info($"Unsubscribing client. Id: {id}.\nParams:\n{subParamsStr}");
-      sub.Dispose();
-   }
-
-   void unsubscribeAll(WebsocketClientID id) {
-      lock (lockObj) {
-         if (!clients.TryGetValue(id, out var subs)) {
-            throw new Exception(
-               $"Unable to unsubscribe: did not find client with id {id}");
-         }
-         subs.DisposeAll();
-         subs.Clear();
-      }
-   }
-
-   void onBadRequest(WebsocketIncomingMessage req, string? errorMsg) {
-      var rpcId = extractRpcId(req.message_body);
-      var logMsg = $"Recieved invalid request:\n{req.message_body}";
-      if (errorMsg is not null) {
-         logMsg += $"\n\n**Exception**\n{errorMsg}";
-      }
-      Log.Api.error(logMsg);
-      if (rpcId is null) {
-         return;
-      }
-      // Send error back to client
-      var rpcErr = JsonRpcError.Create(
-         (long)JsonRpcErrorCode.InvalidRequest,
-         $"Received invalid request: {errorMsg ?? "unknown error"}",
-         req.message_body);
-      sendError(req.sender_client_id, rpcId.Value, rpcErr);
-   }
-}
+namespace Bts.DataStream.Websocket;
+
+using BCG;
+
+using bts;
+using bts.api.core;
+using bts.collections;
+
+using Bts.DataStreamExpressions;
+using Bts.Websocket;
+
+using mbts.utils;
+
+using Spark.Core.Disposables;
+
+using System.Reactive.Subjects;
+using System.Text.Json;
+using System.Text.Json.Nodes;
+
+/// <summary>
+/// Represents an I/O model (<see cref="ISubject{TSource, TResult}"/>), that accepts websocket
+/// requests (from clients) as input, and emits data/notifications (to clients) as output.
+/// This class's chief role is managing data subscriptions to sparkd on the clients' behalf.
+/// </summary>
+public class DataSync(JsonSerializerOptions? serializerOpts = null) :
+   ISubject<WebsocketConnectionUpdate, IEnumerable<WebsocketOutgoingMessage>> {
+   protected readonly object lockObj = new();
+   readonly JsonSerializerOptions serializerOpts = serializerOpts ?? JsonSerializerOptions.Default;
+   readonly Dictionary<WebsocketClientID, List<ClientSubscription>> clients = [];
+
+   event Action<IEnumerable<WebsocketOutgoingMessage>>? requestSend;
+
+   public void OnCompleted() =>
+      Log.Api.info("DataSync incoming request stream has completed.");
+
+   public void OnError(Exception error) =>
+      Log.Api.error($"DataSync error: {error}");
+
+   public void OnNext(WebsocketConnectionUpdate value) {
+      switch (value.Type) {
+         case WebsocketConnectionUpdateType.IncomingMessage:
+            onIncomingMessage(value.IncomingMessage);
+            break;
+         case WebsocketConnectionUpdateType.Connect:
+            onConnect(value.Connect);
+            break;
+         case WebsocketConnectionUpdateType.Disconnect:
+            onDisconnect(value.Disconnect);
+            break;
+         default:
+            Log.assert(false, $"Unexpected WebsocketConnectionUpdate type: {value.Type}");
+            break;
+      }
+   }
+
+   public IDisposable Subscribe(IObserver<IEnumerable<WebsocketOutgoingMessage>> observer) {
+      lock (lockObj) {
+         requestSend += observer.OnNext;
+         return Disposable.Create(Dispose);
+      }
+      void Dispose() {
+         lock (lockObj) {
+            requestSend -= observer.OnNext;
+         }
+      }
+   }
+
+   static JsonRpcId? extractRpcId(string messageBody) {
+      try {
+         return JsonNode.Parse(messageBody)
+            ?.AsObject()
+            ?.GetValueOrDefault("id")
+            ?.Deserialize<JsonRpcId>();
+      } catch {
+         return null;
+      }
+   }
+
+   void onConnect(WebsocketConnectNotification notification) {
+      var id = notification.Id;
+      lock (lockObj) {
+         clients.Add(id, []);
+      }
+      Log.Api.info($"DataStreamWebsocketDataSync client connected. clientId: {notification.Id}");
+      if (notification.Message is not null) {
+         onIncomingMessage(notification.Message);
+      }
+   }
+
+   void onDisconnect(WebsocketDisconnectNotification notification) {
+      Log.Api.warn($"DataStreamWebsocketDataSync client disconnected. clientId: {notification.Id}");
+      unsubscribeAll(notification.Id);
+      clients.Remove(notification.Id);
+   }
+
+   void onIncomingMessage(WebsocketIncomingMessage message) {
+      var id = message.sender_client_id;
+      if (!JsonRpcRequest.TryDeserialize(message, out var req, out var err)) {
+         onBadRequest(message, err);
+         return;
+      }
+      switch (req.Method!.ToLower()) {
+         case "subscribe":
+            subscribe(id, req);
+            break;
+         case "unsubscribe":
+            unsubscribe(id, req);
+            break;
+         case "unsubscribe_all":
+            unsubscribeAll(id);
+            break;
+         default:
+            Log.Api.error($"Received request of unknown type: {req}");
+            break;
+      }
+   }
+
+   void sendNotification<T>(WebsocketClientID id, string method, T rpcParams) {
+      var rpcNotification = JsonRpcRequest.CreateNotification(method, rpcParams);
+      var rpcNotificationStr = JsonSerializer.Serialize(rpcNotification, serializerOpts);
+      send([new(id, rpcNotificationStr)]);
+   }
+
+   void sendResponse<T>(WebsocketClientID id, JsonRpcId rpcId, T rpcParams) {
+      var rpcRes = JsonRpcResponse.Create(rpcId, rpcParams);
+      var rpcResStr = JsonSerializer.Serialize(rpcRes, serializerOpts);
+      send([new(id, rpcResStr )]);
+   }
+
+   void sendError<T>(WebsocketClientID id, JsonRpcId rpcId, JsonRpcError<T> error) {
+      var rpcRes = JsonRpcResponse.CreateError(rpcId, error);
+      var rpcResStr = JsonSerializer.Serialize(rpcRes, serializerOpts);
+      send([new(id, rpcResStr)]);
+   }
+
+   bool send(IEnumerable<WebsocketOutgoingMessage> messages) {
+      lock (lockObj) {
+         if (requestSend is null) {
+            Log.Api.warn(
+               $"Unable to send message(s) because '{nameof(requestSend)}' handler is null");
+            return false;
+         }
+         requestSend(messages);
+         return true;
+      }
+   }
+
+   void subscribe(WebsocketClientID id, JsonRpcRequest<JsonObject?> req) {
+      var subParams = req.Params!.Deserialize<SubscriptionParams>()!;
+      var exp = DataStreamExpressionsInterpreter.Evaluate(subParams.Query!);
+      var stream = exp.getView(new Dictionary<string, StatementExpression>(0));
+      var sub = new ClientSubscription(id, subParams, Disposable.Create(Dispose));
+      lock (lockObj) {
+         clients[id].Add(sub);
+         sendResponse(id, req.Id!.Value, req.Params);
+         stream.onChange += OnChange;
+      }
+
+      void OnChange(object? newValue) {
+         var dp = ExceptionAssert.of_type<DataPacket>(newValue);
+         var rpcParams = new {
+            dp.addedRows,
+            dp.deletedRows
+         };
+         sendNotification(id, "subscription", rpcParams);
+      }
+
+      void Dispose() =>
+         stream.onChange -= OnChange;
+   }
+
+   void unsubscribe(WebsocketClientID id, JsonRpcRequest<JsonObject?> req) {
+      var subParams = req.Params!.Deserialize<SubscriptionParams>();
+      var subParamsStr = JsonSerializer.Serialize(subParams, serializerOpts);
+      // Minimize amount of code inside lock (just need it to find/remove the subscription)
+      List<ClientSubscription>? subs = null;
+      ClientSubscription? sub = null;
+      lock (lockObj) {
+         subs = clients.GetValueOrDefault(id);
+         subs?.TryRemoveFirst(s => s!.Params.Equals(subParams), out sub);
+      }
+      if (subs is null) {
+         // Didn't find client (log/throw).
+         var msg = $"Unable to unsubscribe: did not find client. Id: {id}.\nParams:\n{subParamsStr}";
+         Log.Api.error(msg);
+         throw new Exception(msg);
+      }
+      if (sub is null) {
+         // Found client, but didn't find subscription (log/throw).
+         var msg = $"Unable to unsubscribe: did not find the specified subscription. " +
+            $"Id: {id}. Request:\n{subParams}";
+         Log.Api.error(msg);
+         throw new Exception(msg);
+      }
+      // Found client and subscription (dispose/remove subscription).
+      Log.Api.info($"Unsubscribing client. Id: {id}.\nParams:\n{subParamsStr}");
+      sub.Dispose();
+   }
+
+   void unsubscribeAll(WebsocketClientID id) {
+      lock (lockObj) {
+         if (!clients.TryGetValue(id, out var subs)) {
+            throw new Exception(
+               $"Unable to unsubscribe: did not find client with id {id}");
+         }
+         subs.DisposeAll();
+         subs.Clear();
+      }
+   }
+
+   void onBadRequest(WebsocketIncomingMessage req, string? errorMsg) {
+      var rpcId = extractRpcId(req.message_body);
+      var logMsg = $"Recieved invalid request:\n{req.message_body}";
+      if (errorMsg is not null) {
+         logMsg += $"\n\n**Exception**\n{errorMsg}";
+      }
+      Log.Api.error(logMsg);
+      if (rpcId is null) {
+         return;
+      }
+      // Send error back to client
+      var rpcErr = JsonRpcError.Create(
+         (long)JsonRpcErrorCode.InvalidRequest,
+         $"Received invalid request: {errorMsg ?? "unknown error"}",
+         req.message_body);
+      sendError(req.sender_client_id, rpcId.Value, rpcErr);
+   }
+}

src/DataStreamSuite/Bts.DataStream.Websocket/MultilineStringJsonConverter.cs

@@ -1,48 +1,48 @@
-namespace Bts.DataStream.Websocket;
-
-using Spark.Core;
-
-using System;
-using System.Text.Json;
-using System.Text.Json.Serialization;
-
-class MultilineStringJsonConverter : JsonConverter<string> {
-   public override string? Read(ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options) {
-      if (typeToConvert != typeof(string)) {
-         throw new ArgumentException($"Unable to parse {typeToConvert}", nameof(typeToConvert));
-      }
-      return reader.TokenType switch {
-         JsonTokenType.String => stringImpl(ref reader),
-         JsonTokenType.StartArray => arrayImpl(ref reader),
-         _ => throw new ArgumentException(
-            $"Unable to parse {typeof(SubscriptionParams)} from {reader.TokenType}"),
-      };
-
-      static string? stringImpl(ref Utf8JsonReader reader) =>
-         new(reader.GetString()!);
-
-      static string? arrayImpl(ref Utf8JsonReader reader) {
-         var lines = new List<string>(2);
-         while (reader.Read()) {
-            switch (reader.TokenType) {
-               case JsonTokenType.String:
-                  lines.Add(reader.GetString()!);
-                  break;
-               case JsonTokenType.EndArray:
-                  return string.Join("\n", lines);
-            }
-         }
-         throw new JsonException(
-            $"Failed to parse {typeof(SubscriptionParams)} (did not reach end of array)");
-      }
-   }
-
-   public override void Write(Utf8JsonWriter writer, string value, JsonSerializerOptions options) {
-      var lines = value.Split("\n");
-      if (lines.Length < 2) {
-         writer.WriteRawValue(value);
-         return;
-      }
-      writer.WriteRawValue($"[{", ".join(lines)}]");
-   }
-}
+namespace Bts.DataStream.Websocket;
+
+using Spark.Core;
+
+using System;
+using System.Text.Json;
+using System.Text.Json.Serialization;
+
+class MultilineStringJsonConverter : JsonConverter<string> {
+   public override string? Read(ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options) {
+      if (typeToConvert != typeof(string)) {
+         throw new ArgumentException($"Unable to parse {typeToConvert}", nameof(typeToConvert));
+      }
+      return reader.TokenType switch {
+         JsonTokenType.String => StringImpl(ref reader),
+         JsonTokenType.StartArray => ArrayImpl(ref reader),
+         _ => throw new ArgumentException(
+            $"Unable to parse {typeof(SubscriptionParams)} from {reader.TokenType}"),
+      };
+
+      static string? StringImpl(ref Utf8JsonReader reader) =>
+         new(reader.GetString()!);
+
+      static string? ArrayImpl(ref Utf8JsonReader reader) {
+         var lines = new List<string>(2);
+         while (reader.Read()) {
+            switch (reader.TokenType) {
+               case JsonTokenType.String:
+                  lines.Add(reader.GetString()!);
+                  break;
+               case JsonTokenType.EndArray:
+                  return string.Join("\n", lines);
+            }
+         }
+         throw new JsonException(
+            $"Failed to parse {typeof(SubscriptionParams)} (did not reach end of array)");
+      }
+   }
+
+   public override void Write(Utf8JsonWriter writer, string value, JsonSerializerOptions options) {
+      var lines = value.Split("\n");
+      if (lines.Length < 2) {
+         writer.WriteRawValue(value);
+         return;
+      }
+      writer.WriteRawValue($"[{", ".join(lines)}]");
+   }
+}

src/DataStreamSuite/Bts.DataStream.Websocket/Session.cs

@@ -1,27 +1,31 @@
-namespace Bts.DataStream.Websocket;
-
-using bts;
-using bts.api.core;
-
-using Bts.Websocket;
-
-using Google.Protobuf;
-
-using System.Text.Json;
-
-public static class Session {
-   static int sessionIdCounter;
-
-   public static async Task Start(
-      ServiceContext context, WebsocketServerConfig config, JsonSerializerOptions? opts) {
-      opts ??= JsonSerializerOptions.Default;
-      var sessionId = Interlocked.Increment(ref sessionIdCounter);
-      var gateway = new WebsocketGatewayObservable(config);
-      var sync = new DataSync(context, opts);
-      using var syncSub = gateway.Subscribe(sync);
-      using var gwSub = sync.Subscribe(gateway);
-      Log.Api.info($"Starting DataStream Websocket Session {sessionId}");
-      Log.Api.info($"DataStream Websocket Session {sessionId} config:\n{JsonFormatter.Default.Format(config)}");
-      await gateway.ConnectAsync();
-   }
-}
+namespace Bts.DataStream.Websocket;
+
+using bts;
+using bts.api.core;
+
+using Bts.Websocket;
+
+using Google.Protobuf;
+
+using Spark.Core.Disposables;
+
+
+using System.Text.Json;
+
+public static class Session {
+   static int sessionIdCounter;
+
+   public static async Task Start(
+      ServiceContext context, WebsocketServerConfig config, JsonSerializerOptions? opts) {
+      opts ??= JsonSerializerOptions.Default;
+      var sessionId = Interlocked.Increment(ref sessionIdCounter);
+      var gateway = new WebsocketGatewayObservable(config);
+      var sync = new DataSync(context, opts);
+      using (gateway.Subscribe(sync))
+      using (sync.Subscribe(gateway)) {
+         Log.Api.info($"Starting DataStream Websocket Session {sessionId}");
+         Log.Api.info($"DataStream Websocket Session {sessionId} config:\n{JsonFormatter.Default.Format(config)}");
+         await gateway.ConnectAsync();
+      }
+   }
+}

src/DataStreamSuite/Bts.DataStream.Websocket/SubscriptionParams.cs

@@ -1,110 +1,110 @@
-namespace Bts.DataStream.Websocket;
-
-using Spark.Core;
-
-using System;
-using System.Diagnostics.CodeAnalysis;
-using System.Text.Json;
-using System.Text.Json.Serialization;
-
-public class SubscriptionParams : IEquatable<SubscriptionParams> {
-   static class PropNames {
-      public const string QUERY = "query";
-   }
-
-   public SubscriptionParams(string? query) {
-      Query = query;
-   }
-
-   // Serializer supports plain string, or an array of lines for convenience
-   [MaybeNull]
-   [JsonPropertyName(PropNames.QUERY)]
-   [JsonConverter(typeof(MultilineStringJsonConverter))]
-   public string? Query { get; set; }
-
-   public bool Equals(SubscriptionParams? other) =>
-      other is not null &&
-      Query == other.Query;
-
-   class SubscriptionParamsConverter : JsonConverter<SubscriptionParams> {
-      public override bool CanConvert(Type typeToConvert) =>
-         typeToConvert == typeof(SubscriptionParams);
-
-      public override SubscriptionParams Read(
-         ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options) {
-         if (typeToConvert != typeof(SubscriptionParams)) {
-            throw new ArgumentException($"Unable to parse {typeToConvert}", nameof(typeToConvert));
-         }
-         return read(ref reader);
-      }
-
-      SubscriptionParams read(ref Utf8JsonReader reader) {
-         return reader.TokenType switch {
-            JsonTokenType.String => stringImpl(ref reader),
-            JsonTokenType.StartArray => arrayImpl(ref reader),
-            JsonTokenType.StartObject => objectImpl(ref reader),
-            _ => throw new ArgumentException(
-               $"Unable to parse {typeof(SubscriptionParams)} from {reader.TokenType}"),
-         };
-
-         static SubscriptionParams stringImpl(ref Utf8JsonReader reader) =>
-            new(reader.GetString()!);
-
-         static SubscriptionParams arrayImpl(ref Utf8JsonReader reader) {
-            var lines = new List<string>(2);
-            while (reader.Read()) {
-               switch (reader.TokenType) {
-                  case JsonTokenType.String:
-                     lines.Add(reader.GetString()!);
-                     break;
-                  case JsonTokenType.EndArray:
-                     return new("\n".join(lines));
-               }
-            }
-            throw new JsonException(
-               $"Failed to parse {typeof(SubscriptionParams)} (did not reach end of array)");
-         }
-
-         static SubscriptionParams objectImpl(ref Utf8JsonReader reader) {
-            var lines = new List<string>(2);
-            while (reader.Read()) {
-               switch (reader.TokenType) {
-                  case JsonTokenType.PropertyName:
-                     var propName = reader.GetString()!;
-                     if (propName == PropNames.QUERY) {
-                        switch (reader.TokenType) {
-                           case JsonTokenType.StartArray:
-                              return arrayImpl(ref reader);
-                           case JsonTokenType.String:
-                              return stringImpl(ref reader);
-                        }
-                     }
-                     break;
-                  case JsonTokenType.String:
-                     lines.Add(reader.GetString()!);
-                     break;
-                  case JsonTokenType.EndArray:
-                     return new("\n".join(lines));
-               }
-            }
-            throw new JsonException(
-               $"Failed to parse {typeof(SubscriptionParams)} (did not reach end of array)");
-         }
-      }
-
-      public override void Write(
-         Utf8JsonWriter writer, SubscriptionParams value, JsonSerializerOptions options) {
-         var q = value.Query;
-         var lines = q?.Split("\n");
-         if (lines is null || lines.Length < 2) {
-            writer.WriteString(PropNames.QUERY, value.Query);
-            return;
-         }
-         writer.WriteStartArray(PropNames.QUERY);
-         foreach (var l in lines) {
-            writer.WriteStringValue(l);
-         }
-         writer.WriteEndArray();
-      }
-   }
-}
+namespace Bts.DataStream.Websocket;
+
+using Spark.Core;
+
+using System;
+using System.Diagnostics.CodeAnalysis;
+using System.Text.Json;
+using System.Text.Json.Serialization;
+
+public class SubscriptionParams : IEquatable<SubscriptionParams> {
+   static class PropNames {
+      public const string QUERY = "query";
+   }
+
+   public SubscriptionParams(string? query) {
+      Query = query;
+   }
+
+   // Serializer supports plain string, or an array of lines for convenience
+   [MaybeNull]
+   [JsonPropertyName(PropNames.QUERY)]
+   [JsonConverter(typeof(MultilineStringJsonConverter))]
+   public string? Query { get; set; }
+
+   public bool Equals(SubscriptionParams? other) =>
+      other is not null &&
+      Query == other.Query;
+
+   class SubscriptionParamsConverter : JsonConverter<SubscriptionParams> {
+      public override bool CanConvert(Type typeToConvert) =>
+         typeToConvert == typeof(SubscriptionParams);
+
+      public override SubscriptionParams Read(
+         ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options) {
+         if (typeToConvert != typeof(SubscriptionParams)) {
+            throw new ArgumentException($"Unable to parse {typeToConvert}", nameof(typeToConvert));
+         }
+         return read(ref reader);
+      }
+
+      SubscriptionParams read(ref Utf8JsonReader reader) {
+         return reader.TokenType switch {
+            JsonTokenType.String => stringImpl(ref reader),
+            JsonTokenType.StartArray => arrayImpl(ref reader),
+            JsonTokenType.StartObject => objectImpl(ref reader),
+            _ => throw new ArgumentException(
+               $"Unable to parse {typeof(SubscriptionParams)} from {reader.TokenType}"),
+         };
+
+         static SubscriptionParams stringImpl(ref Utf8JsonReader reader) =>
+            new(reader.GetString()!);
+
+         static SubscriptionParams arrayImpl(ref Utf8JsonReader reader) {
+            var lines = new List<string>(2);
+            while (reader.Read()) {
+               switch (reader.TokenType) {
+                  case JsonTokenType.String:
+                     lines.Add(reader.GetString()!);
+                     break;
+                  case JsonTokenType.EndArray:
+                     return new("\n".join(lines));
+               }
+            }
+            throw new JsonException(
+               $"Failed to parse {typeof(SubscriptionParams)} (did not reach end of array)");
+         }
+
+         static SubscriptionParams objectImpl(ref Utf8JsonReader reader) {
+            var lines = new List<string>(2);
+            while (reader.Read()) {
+               switch (reader.TokenType) {
+                  case JsonTokenType.PropertyName:
+                     var propName = reader.GetString()!;
+                     if (propName == PropNames.QUERY) {
+                        switch (reader.TokenType) {
+                           case JsonTokenType.StartArray:
+                              return arrayImpl(ref reader);
+                           case JsonTokenType.String:
+                              return stringImpl(ref reader);
+                        }
+                     }
+                     break;
+                  case JsonTokenType.String:
+                     lines.Add(reader.GetString()!);
+                     break;
+                  case JsonTokenType.EndArray:
+                     return new("\n".join(lines));
+               }
+            }
+            throw new JsonException(
+               $"Failed to parse {typeof(SubscriptionParams)} (did not reach end of array)");
+         }
+      }
+
+      public override void Write(
+         Utf8JsonWriter writer, SubscriptionParams value, JsonSerializerOptions options) {
+         var q = value.Query;
+         var lines = q?.Split("\n");
+         if (lines is null || lines.Length < 2) {
+            writer.WriteString(PropNames.QUERY, value.Query);
+            return;
+         }
+         writer.WriteStartArray(PropNames.QUERY);
+         foreach (var l in lines) {
+            writer.WriteStringValue(l);
+         }
+         writer.WriteEndArray();
+      }
+   }
+}

src/DataStreamSuite/Bts.DataStream.Websocket/ViewStreamUpdate.cs

@@ -0,0 +1,41 @@
+#nullable enable
+using BCG;
+
+using Spark.Core.Json;
+
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text.Json;
+using System.Text.Json.Serialization;
+
+namespace Bts.DataStreamCore;
+
+class ViewStreamUpdate {
+   [JsonPropertyName("tableName")]
+   public string? TableName { get; set; }
+
+   [JsonPropertyName("columns")]
+   public List<ColumnInfo>? Columns { get; set; }
+
+   [JsonPropertyName("addedRows")]
+   public List<Dictionary<string, object?>>? AddedRows { get; set; }
+
+   [JsonPropertyName("deletedRows")]
+   public List<Dictionary<string, object?>>? DeletedRows { get; set; }
+
+   public DataPacket ToDataPacket(BlueTable state, bool applyChanges = true) {
+      var schema = new BlueTableSchema([.. Columns!], TableName);
+      var addedRowsTable = new BlueTable(schema);
+      var deletedRowsTable = new BlueTable(schema);
+      if (applyChanges) {
+         state.deleteRows(deletedRowsTable);
+         state.addRows(addedRowsTable);
+      }
+      return new(state, addedRowsTable, deletedRowsTable);
+   }
+
+   public static string ToJson() {
+      return "";
+   }
+}

src/DataStreamSuite/Bts.DataStream.WebsocketClient/Bts.DataStream.WebsocketClient.csproj

@@ -1,7 +1,7 @@
 <Project Sdk="Microsoft.NET.Sdk.Web">
 
   <PropertyGroup>
-    <TargetFramework>net7.0</TargetFramework>
+    <TargetFramework>net8.0</TargetFramework>
     <Nullable>enable</Nullable>
     <ImplicitUsings>enable</ImplicitUsings>
     <EnableDefaultContentItems>True</EnableDefaultContentItems>

src/DataStreamSuite/Bts.DataStream.WebsocketServer/Bts.DataStream.WebsocketServer.csproj

@@ -2,7 +2,7 @@
 
   <PropertyGroup>
     <OutputType>Exe</OutputType>
-    <TargetFramework>net7.0</TargetFramework>
+    <TargetFramework>net8.0</TargetFramework>
     <ImplicitUsings>enable</ImplicitUsings>
     <Nullable>enable</Nullable>
   </PropertyGroup>

src/DataStreamSuite/Bts.DataStreamWebApp.Client/.eslintrc.cjs

@@ -0,0 +1,20 @@
+module.exports = {
+  root: true,
+  env: { browser: true, es2020: true },
+  extends: [
+    'eslint:recommended',
+    'plugin:react/recommended',
+    'plugin:react/jsx-runtime',
+    'plugin:react-hooks/recommended',
+  ],
+  ignorePatterns: ['dist', '.eslintrc.cjs'],
+  parserOptions: { ecmaVersion: 'latest', sourceType: 'module' },
+  settings: { react: { version: '18.2' } },
+  plugins: ['react-refresh'],
+  rules: {
+    'react-refresh/only-export-components': [
+      'warn',
+      { allowConstantExport: true },
+    ],
+  },
+}

src/DataStreamSuite/Bts.DataStreamWebApp.Client/.gitignore

@@ -0,0 +1,24 @@
+# Logs
+logs
+*.log
+npm-debug.log*
+yarn-debug.log*
+yarn-error.log*
+pnpm-debug.log*
+lerna-debug.log*
+
+node_modules
+dist
+dist-ssr
+*.local
+
+# Editor directories and files
+.vscode/*
+!.vscode/extensions.json
+.idea
+.DS_Store
+*.suo
+*.ntvs*
+*.njsproj
+*.sln
+*.sw?

src/DataStreamSuite/Bts.DataStreamWebApp.Client/Bts.DataStreamWebApp.Client.esproj

@@ -0,0 +1,11 @@
+<Project Sdk="Microsoft.VisualStudio.JavaScript.Sdk/0.5.128-alpha">
+  <PropertyGroup>
+    <StartupCommand>npm run dev</StartupCommand>
+    <JavaScriptTestRoot>src\</JavaScriptTestRoot>
+    <JavaScriptTestFramework>Jest</JavaScriptTestFramework>
+    <!-- Allows the build (or compile) script located on package.json to run on Build -->
+    <ShouldRunBuildScript>false</ShouldRunBuildScript>
+    <!-- Folder where production build objects will be placed -->
+    <PublishAssetsDirectory>$(MSBuildProjectDirectory)\dist</PublishAssetsDirectory>
+  </PropertyGroup>
+</Project>

src/DataStreamSuite/Bts.DataStreamWebApp.Client/README.md

@@ -0,0 +1,8 @@
+# React + Vite
+
+This template provides a minimal setup to get React working in Vite with HMR and some ESLint rules.
+
+Currently, two official plugins are available:
+
+- [@vitejs/plugin-react](https://github.com/vitejs/vite-plugin-react/blob/main/packages/plugin-react/README.md) uses [Babel](https://babeljs.io/) for Fast Refresh
+- [@vitejs/plugin-react-swc](https://github.com/vitejs/vite-plugin-react-swc) uses [SWC](https://swc.rs/) for Fast Refresh

src/DataStreamSuite/Bts.DataStreamWebApp.Client/index.html

@@ -0,0 +1,13 @@
+<!doctype html>
+<html lang="en">
+  <head>
+    <meta charset="UTF-8" />
+    <link rel="icon" type="image/svg+xml" href="/vite.svg" />
+    <meta name="viewport" content="width=device-width, initial-scale=1.0" />
+    <title>Vite + React</title>
+  </head>
+  <body>
+    <div id="root"></div>
+    <script type="module" src="/src/main.jsx"></script>
+  </body>
+</html>

src/DataStreamSuite/Bts.DataStreamWebApp.Client/nuget.config

@@ -0,0 +1,10 @@
+<?xml version="1.0" encoding="utf-8"?>
+<configuration>
+  <packageSources>
+    <clear />
+    <add key="nuget.org" value="https://api.nuget.org/v3/index.json" />
+  </packageSources>
+  <disabledPackageSources>
+    <clear />
+  </disabledPackageSources>
+</configuration>

src/DataStreamSuite/Bts.DataStreamWebApp.Client/package.json

@@ -0,0 +1,27 @@
+{
+  "name": "bts.datastreamwebapp.client",
+  "private": true,
+  "version": "0.0.0",
+  "type": "module",
+  "scripts": {
+    "dev": "vite",
+    "build": "vite build",
+    "lint": "eslint . --ext js,jsx --report-unused-disable-directives --max-warnings 0",
+    "preview": "vite preview"
+  },
+  "dependencies": {
+    "react": "^18.2.0",
+    "react-dom": "^18.2.0",
+    "react-use-websocket": "^4.5.0"
+  },
+  "devDependencies": {
+    "@types/react": "^18.2.43",
+    "@types/react-dom": "^18.2.17",
+    "@vitejs/plugin-react": "^4.2.1",
+    "eslint": "^8.55.0",
+    "eslint-plugin-react": "^7.33.2",
+    "eslint-plugin-react-hooks": "^4.6.0",
+    "eslint-plugin-react-refresh": "^0.4.5",
+    "vite": "^5.0.8"
+  }
+}

src/DataStreamSuite/Bts.DataStreamWebApp.Client/public/vite.svg

@@ -0,0 +1 @@
+<svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" class="iconify iconify--logos" width="31.88" height="32" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 257"><defs><linearGradient id="IconifyId1813088fe1fbc01fb466" x1="-.828%" x2="57.636%" y1="7.652%" y2="78.411%"><stop offset="0%" stop-color="#41D1FF"></stop><stop offset="100%" stop-color="#BD34FE"></stop></linearGradient><linearGradient id="IconifyId1813088fe1fbc01fb467" x1="43.376%" x2="50.316%" y1="2.242%" y2="89.03%"><stop offset="0%" stop-color="#FFEA83"></stop><stop offset="8.333%" stop-color="#FFDD35"></stop><stop offset="100%" stop-color="#FFA800"></stop></linearGradient></defs><path fill="url(#IconifyId1813088fe1fbc01fb466)" d="M255.153 37.938L134.897 252.976c-2.483 4.44-8.862 4.466-11.382.048L.875 37.958c-2.746-4.814 1.371-10.646 6.827-9.67l120.385 21.517a6.537 6.537 0 0 0 2.322-.004l117.867-21.483c5.438-.991 9.574 4.796 6.877 9.62Z"></path><path fill="url(#IconifyId1813088fe1fbc01fb467)" d="M185.432.063L96.44 17.501a3.268 3.268 0 0 0-2.634 3.014l-5.474 92.456a3.268 3.268 0 0 0 3.997 3.378l24.777-5.718c2.318-.535 4.413 1.507 3.936 3.838l-7.361 36.047c-.495 2.426 1.782 4.5 4.151 3.78l15.304-4.649c2.372-.72 4.652 1.36 4.15 3.788l-11.698 56.621c-.732 3.542 3.979 5.473 5.943 2.437l1.313-2.028l72.516-144.72c1.215-2.423-.88-5.186-3.54-4.672l-25.505 4.922c-2.396.462-4.435-1.77-3.759-4.114l16.646-57.705c.677-2.35-1.37-4.583-3.769-4.113Z"></path></svg>

src/DataStreamSuite/Bts.DataStreamWebApp.Client/src/App.css

@@ -0,0 +1,44 @@
+#root {
+    max-width: 1280px;
+    margin: 0 auto;
+    padding: 2rem;
+    text-align: center;
+    --cell-background-color: #293136;
+    --cell-color: #cbe3ef;
+}
+
+tr:nth-child(even) {
+    background: #F2F2F2;
+}
+
+tr:nth-child(odd) {
+    background: #FFF;
+}
+
+th, td {
+    padding-left: 1rem;
+    padding-right: 1rem;
+    background-color: var(--cell-background-color);
+    color: var(--cell-color);
+}
+
+textarea#query-input {
+  height: 5em;
+  width: 50em;
+}
+
+button#query-button {
+  height: 5em;
+  vertical-align: top;
+}
+
+table#query-output {
+  float: none;
+  clear: both;
+}
+
+p#footer {
+  float: none;
+  clear: both;
+  vertical-align: bottom;
+}

src/DataStreamSuite/Bts.DataStreamWebApp.Client/src/App.jsx

@@ -0,0 +1,93 @@
+import { useEffect, useState, Component } from "react";
+import { useWebSocket } from "react-use-websocket";
+import "./App.css";
+
+export class Home extends Component {
+  static displayName = Home.name;
+  // static sparkdHost = '192.168.50.94';
+  static sparkdHost = "localhost";
+
+  // static websocket_url = 'wss://markets.api.uat.cmegroup.com/marketdatastream/v1';
+  // static access_token = 'eyJhbGciOiJSUzI1NiIsImtpZCI6IjF0al9MTVFrT2pGVzJWQTZvdkZzOEZHUWViMCIsInBpLmF0bSI6IjEifQ.eyJzY29wZSI6W10sImNsaWVudF9pZCI6ImFwaV90aGlua2FscGhhMzIxIiwiYXVkIjoiTnIiLCJqdGkiOiIweERtb3NnU2NHcE5GUEFaIiwic3ViIjoiYXBpX3RoaW5rYWxwaGEzMjEiLCJleHAiOjE2OTgzNTY5OTJ9.uJJEhWpujGiyJ5_lypiIPB8xR0GkuXqydtCugp5xD0hyTuX6VkQdNoY_oaRXXGRCUPCvfsXpU-HfL7pRkcXCjMLTrVIqf3RjD4_gar_ExbIb180nbYMG87E4rOILi5bIin51KOcMNBdHW7pTKwsS221XGyIAKIj65WxD-K8QSVr1HN6s7KDfYdez5dstWWC6xGg_3IZkrJjx8Df1HNT73K9PwUaAl7gcK4_2RSr8L636QbF3_y4Tv26g5IEv90Mc1mnAFPvUGI-nDR4s5Vso0398DlYl_tw-zbQQmn-QuQ6rS_ZIcZKgyoHKwD-3lgV-J1_--ndHdh8qwJC7Hk8ZKg'
+
+  render() {
+    let socket = null;
+    const wsUrl = `ws://localhost:5268/ws`;
+    function connectToSparkd() {
+      // const wsUrl = Home.websocket_url;
+      if (socket == null) {
+        socket = new WebSocket(wsUrl);
+      }
+      if (socket && socket.readyState === WebSocket.OPEN) {
+        return;
+      }
+
+      const output = document.getElementById("query-output");
+      socket.onopen = (event) => {
+        console.log(`Connected to ${wsUrl}`);
+      };
+
+      socket.onmessage = (event) => {
+        console.log(`Received message: ${event.data}`);
+        output.innerHTML = event.data;
+      };
+
+      socket.onerror = (error) => {
+        console.log(`Error on websocket ${wsUrl}: ${error}`);
+      };
+
+      socket.onclose = (event) => {
+        if (event.wasClean) {
+          const msg = `Connection to ${wsUrl} closed cleanly, code=${event.code}, reason=${event.reason}`;
+          console.log(msg);
+        } else {
+          const msg = `Connection to ${wsUrl} abruptly closed`;
+          console.log(msg);
+        }
+      };
+    }
+
+    function executeQuery() {
+      if (socket == null || !socket) {
+        connectToSparkd();
+      }
+      if (socket == null || !socket) {
+        console.log("Failed to connect to websocket server");
+        return;
+      }
+      if (socket.readyState === WebSocket.OPEN) {
+        const queryInput = document.getElementById("query-input");
+        const rpcReq = {
+          jsonrpc: "2.0",
+          method: "subscribe",
+          params: {
+            query: queryInput.value,
+          },
+          id: 1,
+        };
+        socket.send(JSON.stringify(rpcReq));
+      }
+    }
+
+    return (
+      <div>
+        <h1>Data Stream Query Analyzer</h1>
+        <p>Run queries against the BTS Cloud</p>
+        <div>
+          <textarea id="query-input" placeholder="Enter Query"></textarea>
+          <button id="query-button" onClick={executeQuery}>
+            Execute
+          </button>
+        </div>
+        <table id="query-output"></table>
+        <p id="footer">
+          The <code>ClientApp</code> subdirectory is a standard React
+          application based on the <code>create-react-app</code> template. If
+          you open a command prompt in that directory, you can run{" "}
+          <code>npm</code> commands such as <code>npm test</code> or{" "}
+          <code>npm install</code>.
+        </p>
+      </div>
+    );
+  }
+}

src/DataStreamSuite/Bts.DataStreamWebApp.Client/src/assets/react.svg

@@ -0,0 +1 @@
+<svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" class="iconify iconify--logos" width="35.93" height="32" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 228"><path fill="#00D8FF" d="M210.483 73.824a171.49 171.49 0 0 0-8.24-2.597c.465-1.9.893-3.777 1.273-5.621c6.238-30.281 2.16-54.676-11.769-62.708c-13.355-7.7-35.196.329-57.254 19.526a171.23 171.23 0 0 0-6.375 5.848a155.866 155.866 0 0 0-4.241-3.917C100.759 3.829 77.587-4.822 63.673 3.233C50.33 10.957 46.379 33.89 51.995 62.588a170.974 170.974 0 0 0 1.892 8.48c-3.28.932-6.445 1.924-9.474 2.98C17.309 83.498 0 98.307 0 113.668c0 15.865 18.582 31.778 46.812 41.427a145.52 145.52 0 0 0 6.921 2.165a167.467 167.467 0 0 0-2.01 9.138c-5.354 28.2-1.173 50.591 12.134 58.266c13.744 7.926 36.812-.22 59.273-19.855a145.567 145.567 0 0 0 5.342-4.923a168.064 168.064 0 0 0 6.92 6.314c21.758 18.722 43.246 26.282 56.54 18.586c13.731-7.949 18.194-32.003 12.4-61.268a145.016 145.016 0 0 0-1.535-6.842c1.62-.48 3.21-.974 4.76-1.488c29.348-9.723 48.443-25.443 48.443-41.52c0-15.417-17.868-30.326-45.517-39.844Zm-6.365 70.984c-1.4.463-2.836.91-4.3 1.345c-3.24-10.257-7.612-21.163-12.963-32.432c5.106-11 9.31-21.767 12.459-31.957c2.619.758 5.16 1.557 7.61 2.4c23.69 8.156 38.14 20.213 38.14 29.504c0 9.896-15.606 22.743-40.946 31.14Zm-10.514 20.834c2.562 12.94 2.927 24.64 1.23 33.787c-1.524 8.219-4.59 13.698-8.382 15.893c-8.067 4.67-25.32-1.4-43.927-17.412a156.726 156.726 0 0 1-6.437-5.87c7.214-7.889 14.423-17.06 21.459-27.246c12.376-1.098 24.068-2.894 34.671-5.345a134.17 134.17 0 0 1 1.386 6.193ZM87.276 214.515c-7.882 2.783-14.16 2.863-17.955.675c-8.075-4.657-11.432-22.636-6.853-46.752a156.923 156.923 0 0 1 1.869-8.499c10.486 2.32 22.093 3.988 34.498 4.994c7.084 9.967 14.501 19.128 21.976 27.15a134.668 134.668 0 0 1-4.877 4.492c-9.933 8.682-19.886 14.842-28.658 17.94ZM50.35 144.747c-12.483-4.267-22.792-9.812-29.858-15.863c-6.35-5.437-9.555-10.836-9.555-15.216c0-9.322 13.897-21.212 37.076-29.293c2.813-.98 5.757-1.905 8.812-2.773c3.204 10.42 7.406 21.315 12.477 32.332c-5.137 11.18-9.399 22.249-12.634 32.792a134.718 134.718 0 0 1-6.318-1.979Zm12.378-84.26c-4.811-24.587-1.616-43.134 6.425-47.789c8.564-4.958 27.502 2.111 47.463 19.835a144.318 144.318 0 0 1 3.841 3.545c-7.438 7.987-14.787 17.08-21.808 26.988c-12.04 1.116-23.565 2.908-34.161 5.309a160.342 160.342 0 0 1-1.76-7.887Zm110.427 27.268a347.8 347.8 0 0 0-7.785-12.803c8.168 1.033 15.994 2.404 23.343 4.08c-2.206 7.072-4.956 14.465-8.193 22.045a381.151 381.151 0 0 0-7.365-13.322Zm-45.032-43.861c5.044 5.465 10.096 11.566 15.065 18.186a322.04 322.04 0 0 0-30.257-.006c4.974-6.559 10.069-12.652 15.192-18.18ZM82.802 87.83a323.167 323.167 0 0 0-7.227 13.238c-3.184-7.553-5.909-14.98-8.134-22.152c7.304-1.634 15.093-2.97 23.209-3.984a321.524 321.524 0 0 0-7.848 12.897Zm8.081 65.352c-8.385-.936-16.291-2.203-23.593-3.793c2.26-7.3 5.045-14.885 8.298-22.6a321.187 321.187 0 0 0 7.257 13.246c2.594 4.48 5.28 8.868 8.038 13.147Zm37.542 31.03c-5.184-5.592-10.354-11.779-15.403-18.433c4.902.192 9.899.29 14.978.29c5.218 0 10.376-.117 15.453-.343c-4.985 6.774-10.018 12.97-15.028 18.486Zm52.198-57.817c3.422 7.8 6.306 15.345 8.596 22.52c-7.422 1.694-15.436 3.058-23.88 4.071a382.417 382.417 0 0 0 7.859-13.026a347.403 347.403 0 0 0 7.425-13.565Zm-16.898 8.101a358.557 358.557 0 0 1-12.281 19.815a329.4 329.4 0 0 1-23.444.823c-7.967 0-15.716-.248-23.178-.732a310.202 310.202 0 0 1-12.513-19.846h.001a307.41 307.41 0 0 1-10.923-20.627a310.278 310.278 0 0 1 10.89-20.637l-.001.001a307.318 307.318 0 0 1 12.413-19.761c7.613-.576 15.42-.876 23.31-.876H128c7.926 0 15.743.303 23.354.883a329.357 329.357 0 0 1 12.335 19.695a358.489 358.489 0 0 1 11.036 20.54a329.472 329.472 0 0 1-11 20.722Zm22.56-122.124c8.572 4.944 11.906 24.881 6.52 51.026c-.344 1.668-.73 3.367-1.15 5.09c-10.622-2.452-22.155-4.275-34.23-5.408c-7.034-10.017-14.323-19.124-21.64-27.008a160.789 160.789 0 0 1 5.888-5.4c18.9-16.447 36.564-22.941 44.612-18.3ZM128 90.808c12.625 0 22.86 10.235 22.86 22.86s-10.235 22.86-22.86 22.86s-22.86-10.235-22.86-22.86s10.235-22.86 22.86-22.86Z"></path></svg>

src/DataStreamSuite/Bts.DataStreamWebApp.Client/src/index.css

@@ -0,0 +1,68 @@
+:root {
+  font-family: Inter, system-ui, Avenir, Helvetica, Arial, sans-serif;
+  line-height: 1.5;
+  font-weight: 400;
+
+  color-scheme: light dark;
+  color: rgba(255, 255, 255, 0.87);
+  background-color: #242424;
+
+  font-synthesis: none;
+  text-rendering: optimizeLegibility;
+  -webkit-font-smoothing: antialiased;
+  -moz-osx-font-smoothing: grayscale;
+}
+
+a {
+  font-weight: 500;
+  color: #646cff;
+  text-decoration: inherit;
+}
+a:hover {
+  color: #535bf2;
+}
+
+body {
+  margin: 0;
+  display: flex;
+  place-items: center;
+  min-width: 320px;
+  min-height: 100vh;
+}
+
+h1 {
+  font-size: 3.2em;
+  line-height: 1.1;
+}
+
+button {
+  border-radius: 8px;
+  border: 1px solid transparent;
+  padding: 0.6em 1.2em;
+  font-size: 1em;
+  font-weight: 500;
+  font-family: inherit;
+  background-color: #1a1a1a;
+  cursor: pointer;
+  transition: border-color 0.25s;
+}
+button:hover {
+  border-color: #646cff;
+}
+button:focus,
+button:focus-visible {
+  outline: 4px auto -webkit-focus-ring-color;
+}
+
+@media (prefers-color-scheme: light) {
+  :root {
+    color: #213547;
+    background-color: #ffffff;
+  }
+  a:hover {
+    color: #747bff;
+  }
+  button {
+    background-color: #f9f9f9;
+  }
+}

src/DataStreamSuite/Bts.DataStreamWebApp.Client/src/main.jsx

@@ -0,0 +1,10 @@
+import React from 'react'
+import ReactDOM from 'react-dom/client'
+import {Home} from './App.jsx'
+import './index.css'
+
+ReactDOM.createRoot(document.getElementById('root')).render(
+  <React.StrictMode>
+    <Home />
+  </React.StrictMode>,
+)

src/DataStreamSuite/Bts.DataStreamWebApp.Client/vite.config.js

@@ -0,0 +1,60 @@
+import { fileURLToPath, URL } from 'node:url';
+
+import { defineConfig } from 'vite';
+import plugin from '@vitejs/plugin-react';
+import fs from 'fs';
+import path from 'path';
+import child_process from 'child_process';
+
+const baseFolder =
+    process.env.APPDATA !== undefined && process.env.APPDATA !== ''
+        ? `${process.env.APPDATA}/ASP.NET/https`
+        : `${process.env.HOME}/.aspnet/https`;
+
+const certificateArg = process.argv.map(arg => arg.match(/--name=(?<value>.+)/i)).filter(Boolean)[0];
+const certificateName = certificateArg ? certificateArg.groups.value : "bts.datastreamwebapp.client";
+
+if (!certificateName) {
+    console.error('Invalid certificate name. Run this script in the context of an npm/yarn script or pass --name=<<app>> explicitly.')
+    process.exit(-1);
+}
+
+const certFilePath = path.join(baseFolder, `${certificateName}.pem`);
+const keyFilePath = path.join(baseFolder, `${certificateName}.key`);
+
+if (!fs.existsSync(certFilePath) || !fs.existsSync(keyFilePath)) {
+    if (0 !== child_process.spawnSync('dotnet', [
+        'dev-certs',
+        'https',
+        '--export-path',
+        certFilePath,
+        '--format',
+        'Pem',
+        '--no-password',
+    ], { stdio: 'inherit', }).status) {
+        throw new Error("Could not create certificate.");
+    }
+}
+
+// https://vitejs.dev/config/
+export default defineConfig({
+    plugins: [plugin()],
+    resolve: {
+        alias: {
+            '@': fileURLToPath(new URL('./src', import.meta.url))
+        }
+    },
+    server: {
+        proxy: {
+            '^/weatherforecast': {
+                target: 'https://localhost:7128/',
+                secure: false
+            }
+        },
+        port: 5173,
+        https: {
+            key: fs.readFileSync(keyFilePath),
+            cert: fs.readFileSync(certFilePath),
+        }
+    }
+})

src/DataStreamSuite/Bts.DataStreamWebApp.Server/Bts.DataStreamWebApp.Server.csproj

@@ -0,0 +1,35 @@
+<Project Sdk="Microsoft.NET.Sdk.Web">
+
+  <PropertyGroup>
+    <TargetFramework>net8.0</TargetFramework>
+    <Nullable>enable</Nullable>
+    <ImplicitUsings>enable</ImplicitUsings>
+    <InvariantGlobalization>true</InvariantGlobalization>
+    <SpaRoot>..\Bts.DataStreamWebApp.Client</SpaRoot>
+    <SpaProxyLaunchCommand>npm run dev</SpaProxyLaunchCommand>
+    <SpaProxyServerUrl>https://localhost:5173</SpaProxyServerUrl>
+  </PropertyGroup>
+
+  <ItemGroup>
+    <PackageReference Include="CommandLineParser" Version="2.9.1" />
+    <PackageReference Include="Microsoft.AspNetCore.SpaProxy">
+      <Version>8.*-*</Version>
+    </PackageReference>
+    <PackageReference Include="Newtonsoft.Json" Version="13.0.3" />
+    <PackageReference Include="Swashbuckle.AspNetCore" Version="6.4.0" />
+  </ItemGroup>
+
+  <ItemGroup>
+    <ProjectReference Include="..\Bts.DataStreamWebApp.Client\Bts.DataStreamWebApp.Client.esproj">
+      <ReferenceOutputAssembly>false</ReferenceOutputAssembly>
+    </ProjectReference>
+    <ProjectReference Include="..\DataStream\DataStream.csproj" />
+    <ProjectReference Include="..\SharedStreams\SharedStreams.csproj" />
+    <ProjectReference Include="..\..\client\spark_api\spark_api.csproj" />
+    <ProjectReference Include="..\..\client\spark_websocket\spark_websocket.csproj" />
+  </ItemGroup>
+
+  <!-- <ItemGroup>
+    <Content Update="wwwroot\alt-index.html" CopyToPublishDirectory="Never" />
+  </ItemGroup> -->
+</Project>

src/DataStreamSuite/Bts.DataStreamWebApp.Server/Bts.DataStreamWebApp.Server.http

@@ -0,0 +1,6 @@
+@Bts.DataStreamWebApp.Server_HostAddress = http://localhost:5268
+
+GET {{Bts.DataStreamWebApp.Server_HostAddress}}/weatherforecast/
+Accept: application/json
+
+###

src/DataStreamSuite/Bts.DataStreamWebApp.Server/Controllers/DataStreamSubscriptionController.cs

@@ -0,0 +1,117 @@
+namespace Bts.DataStreamWebApp.Server.Controllers;
+
+using BCG;
+
+using bts.api.core;
+
+using Bts.DataStreamExpressions;
+
+using Microsoft.AspNetCore.Mvc;
+
+using Spark.Core.Disposables;
+
+using System.Net.WebSockets;
+using System.Text;
+using System.Text.Json;
+
+// [Route("[controller]")]
+[Route("/ws")]
+public class DataStreamSubscriptionController(ILogger<DataStreamSubscriptionController> logger) : ControllerBase {
+   readonly ILogger<DataStreamSubscriptionController> _logger = logger;
+   IDisposable? streamSubscription;
+
+   [Route("/ws")]
+   public async Task Get() {
+      if (!HttpContext.WebSockets.IsWebSocketRequest) {
+         _logger.LogError("Expected websocket request on path '{}'", HttpContext.Request.Path);
+         return;
+      }
+      Console.WriteLine("Establishing websocket connection...");
+      using var webSocket = await HttpContext.WebSockets.AcceptWebSocketAsync();
+      Console.WriteLine($"Websocket connection established.");
+      await runWebSocketServer(null, "bts", string.Empty, webSocket);
+   }
+
+   public static async Task Echo(WebSocket webSocket) {
+      var buffer = new byte[1024 * 4];
+      var receiveResult = await webSocket.ReceiveAsync(
+          new ArraySegment<byte>(buffer), CancellationToken.None);
+
+      while (!receiveResult.CloseStatus.HasValue) {
+         await webSocket.SendAsync(
+             new ArraySegment<byte>(buffer, 0, receiveResult.Count),
+             receiveResult.MessageType,
+             receiveResult.EndOfMessage,
+             CancellationToken.None);
+
+         receiveResult = await webSocket.ReceiveAsync(
+             new ArraySegment<byte>(buffer), CancellationToken.None);
+      }
+
+      await webSocket.CloseAsync(
+          receiveResult.CloseStatus.Value,
+          receiveResult.CloseStatusDescription,
+          CancellationToken.None);
+   }
+
+   async Task<int> runWebSocketServer(string? sparkd, string user, string password, WebSocket webSocket) {
+      var context = ServiceContext.create(sparkd);
+      await context.login(user, password);
+      var buffer = new byte[1024 * 4];
+      var receiveResult = await webSocket.ReceiveAsync(
+          new ArraySegment<byte>(buffer), CancellationToken.None);
+
+      try {
+         while (!receiveResult.CloseStatus.HasValue) {
+            receiveResult = await webSocket.ReceiveAsync(
+                new ArraySegment<byte>(buffer), CancellationToken.None);
+            var msg = Encoding.UTF8.GetString(buffer, 0, receiveResult.Count);
+            // var queryArgs = new QueryArguments { Query = msg };
+            var rpcReq = JsonSerializer.Deserialize<Bts.Websocket.JsonRpcRequest<DataStreamSubscription>>(msg);
+            var queryArgs = rpcReq!.Params;
+            // if (queryArgs is null) {
+            //    Console.WriteLine("Invalid queryArgs.");
+            //    continue;
+            // }
+            var exp = DataStreamExpressionsInterpreter.Evaluate(queryArgs.Query!);
+            if (exp is null) {
+               Console.WriteLine($"Invalid query expression: {queryArgs.Query}.");
+               continue;
+            }
+            var vs = exp.getView(new Dictionary<string, StatementExpression>(0));
+            Disposable.dispose(ref streamSubscription);
+            vs.onChange += onStreamChanged;
+            streamSubscription = Disposable.Create(() => vs.onChange -= onStreamChanged);
+            // await webSocket.SendAsync(
+            //     new ArraySegment<byte>(buffer, 0, receiveResult.Count),
+            //     receiveResult.MessageType,
+            //     receiveResult.EndOfMessage,
+            //     CancellationToken.None);
+         }
+      } catch (Exception ex) {
+         Console.WriteLine($"Exception on WebsocketServer thread: {ex}");
+         return 1;
+      }
+
+      return 0;
+
+      async void onStreamChanged(object newValue) {
+         if (newValue is not DataPacket dp) {
+            Console.WriteLine(
+               $"OnStreamChanged delegate expected DataPacket (actually received {newValue?.GetType()}.");
+            return;
+         }
+         Console.WriteLine("Received stream update.");
+         var added_rows_str = dp.addedRows.get_printed_rows(val => val.Length);
+         var msg = $"Received new rows:\n{added_rows_str}";
+         // var payload = new ReadOnlyMemory<byte>([.. msg]);
+         var encoded = Encoding.UTF8.GetBytes(msg);
+         var payload = new ArraySegment<byte>(encoded, 0, encoded.Length);
+         await webSocket.SendAsync(
+             payload,
+             receiveResult.MessageType,
+             receiveResult.EndOfMessage,
+             CancellationToken.None);
+      }
+   }
+}

src/DataStreamSuite/Bts.DataStreamWebApp.Server/Controllers/WeatherForecastController.cs

@@ -0,0 +1,22 @@
+namespace Bts.DataStreamWebApp.Server.Controllers;
+using Microsoft.AspNetCore.Mvc;
+
+[ApiController]
+[Route("[controller]")]
+public class WeatherForecastController(ILogger<WeatherForecastController> logger) : ControllerBase {
+   static readonly string[] Summaries = [
+      "Freezing", "Bracing", "Chilly", "Cool", "Mild", "Warm", "Balmy", "Hot", "Sweltering", "Scorching"
+   ];
+
+   readonly ILogger<WeatherForecastController> _logger = logger;
+
+   [HttpGet(Name = "GetWeatherForecast")]
+   public IEnumerable<WeatherForecast> Get() {
+      return Enumerable.Range(1, 5).Select(index => new WeatherForecast {
+         Date = DateOnly.FromDateTime(DateTime.Now.AddDays(index)),
+         TemperatureC = Random.Shared.Next(-20, 55),
+         Summary = Summaries[Random.Shared.Next(Summaries.Length)]
+      })
+      .ToArray();
+   }
+}

src/DataStreamSuite/Bts.DataStreamWebApp.Server/DataStreamSubscription.cs

@@ -0,0 +1,8 @@
+namespace Bts.DataStreamWebApp.Server;
+
+using System.Text.Json.Serialization;
+
+public class DataStreamSubscription {
+   [JsonPropertyName("query")]
+   public string? Query { get; set; }
+}

src/DataStreamSuite/Bts.DataStreamWebApp.Server/DataStreamUpdate.cs

@@ -0,0 +1,8 @@
+namespace Bts.DataStreamWebApp.Server;
+
+using BCG;
+
+class DataStreamUpdate {
+   public List<IBlueRow> addedRows { get; set; } = new(0);
+   public List<IBlueRow> deletedRows { get; set; } = new(0);
+}

src/DataStreamSuite/Bts.DataStreamWebApp.Server/DataStreamWebAppServerOptions.cs

@@ -0,0 +1,14 @@
+namespace Bts.DataStreamWebApp.Server;
+
+using CommandLine;
+
+class DataStreamWebAppServerOptions {
+   [Option("sparkd")]
+   public string? sparkd { get; set; }
+
+   [Option("user", Default = "bts")]
+   public string? user { get; set; }
+
+   [Option("password", Default = "")]
+   public string? password { get; set; }
+}

src/DataStreamSuite/Bts.DataStreamWebApp.Server/DataStreamWebsocketHandler.cs

@@ -0,0 +1,58 @@
+using Microsoft.AspNetCore.Http;
+using Microsoft.Extensions.Logging;
+
+using System;
+using System.Net.WebSockets;
+using System.Threading;
+using System.Threading.Tasks;
+
+namespace Bts.DataStreamWebApp.Server {
+   // public class DataStreamWebSocketHandler {
+   //    readonly WebSocketConnectionManager _connectionManager = new();
+   //    readonly ILogger<DataStreamWebSocketHandler> _logger;
+
+   //    public DataStreamWebSocketHandler(
+   //       WebSocketConnectionManager connectionManager,
+   //       ILogger<DataStreamWebSocketHandler> logger
+   //    ) {
+   //       _connectionManager = connectionManager;
+   //       _logger = logger;
+   //    }
+
+   //    public async Task HandleWebSocket(HttpContext context, WebSocket webSocket) {
+   //       var socketId = _connectionManager.AddSocket(webSocket);
+
+   //       _logger.LogInformation($"WebSocket connection established with ID {socketId}");
+
+   //       while (webSocket.State == WebSocketState.Open) {
+   //          var message = await ReceiveMessageAsync(webSocket);
+   //          if (message != null) {
+   //             _logger.LogInformation($"Received message from ID {socketId}: {message}");
+   //             await BroadcastMessageAsync(message);
+   //          }
+   //       }
+
+   //       _connectionManager.RemoveSocket(socketId);
+   //       _logger.LogInformation($"WebSocket connection closed with ID {socketId}");
+   //    }
+
+   //    async Task<string?> ReceiveMessageAsync(WebSocket webSocket) {
+   //       var buffer = new byte[1024];
+   //       var result = await webSocket.ReceiveAsync(new ArraySegment<byte>(buffer), CancellationToken.None);
+
+   //       if (result.CloseStatus.HasValue) {
+   //          return null;
+   //       }
+
+   //       return System.Text.Encoding.UTF8.GetString(buffer, 0, result.Count);
+   //    }
+
+   //    async Task BroadcastMessageAsync(string message) {
+   //       foreach (var socket in _connectionManager.GetAllSockets()) {
+   //          if (socket.Value.State == WebSocketState.Open) {
+   //             await socket.Value.SendAsync(System.Text.Encoding.UTF8.GetBytes(message), WebSocketMessageType.Text, true, CancellationToken.None);
+   //          }
+   //       }
+   //    }
+   // }
+}

src/DataStreamSuite/Bts.DataStreamWebApp.Server/Program.cs

@@ -0,0 +1,166 @@
+using BCG;
+
+using bts;
+using bts.api.core;
+using bts.utils;
+
+using Bts.DataStreamExpressions;
+using Bts.DataStreamWebApp.Server;
+using Bts.DataStreamWebApp.Server.Controllers;
+
+using CommandLine;
+
+using mbts.config;
+
+using System.Net.WebSockets;
+using System.Reactive.Disposables;
+using System.Reactive.Linq;
+using System.Text;
+using System.Text.Json;
+using System.Text.Json.Serialization;
+
+static string fmtReq(HttpRequest req) =>
+   $"{req.Protocol}://{req.Host}/{req.Path}?{req.QueryString}";
+
+static string fmtDataPacket(DataPacket dp) {
+   var sb = new StringBuilder();
+   var rows = dp.addedRows.Cast<IBlueRow>().ToList();
+   var colWidths = dp.schema.columnInfo.Select(c => rows.Max(r => r[c.colNo]?.ToString()?.Length ?? 0)).ToArray();
+   foreach (IBlueRow r in dp.addedRows) {
+      for (int i = 0; i < colWidths.Length; i++) {
+         var str = r[i]?.ToString() ?? string.Empty;
+         if (i > 0) {
+            sb.Append(" | ");
+         }
+         sb.Append(str);
+         var width = colWidths[i];
+         if (str.Length < width) {
+            sb.Append(new string(' ', width - str.Length));
+         }
+      }
+      sb.AppendLine();
+   }
+   return sb.ToString();
+   // var upd = new DataStreamUpdate();
+   // upd.addedRows.AddRange(dp.addedRows.Cast<IBlueRow>());
+   // upd.deletedRows.AddRange(dp.deletedRows.Cast<IBlueRow>());
+   // return fmtDataStreamUpdate(upd);
+}
+
+static string fmtDataStreamUpdate(DataStreamUpdate rows) {
+   return JsonSerializer.Serialize(rows);
+}
+
+var parser = new Parser(ps => {
+   ps.CaseInsensitiveEnumValues = true;
+   ps.HelpWriter = Console.Out;
+});
+
+await parser.ParseArguments<DataStreamWebAppServerOptions>(args).
+   MapResult(run, errors => Task.FromResult(1));
+
+async Task<int> run(DataStreamWebAppServerOptions opts) {
+   BTSNativeMethods.provide_command_line(args, args.Length);
+   Configuration.initialize();
+   var context = ServiceContext.create(opts.sparkd);
+   await context.login(opts.user!, opts.password!);
+   var vsWorkLoop = Observable.Interval(TimeSpan.FromMilliseconds(200)).Subscribe(_ => {
+      ViewStreamDatabase.instance.doWork();
+      MasterWorker.getInstance(false).doWork();
+   });
+   var builder = WebApplication.CreateBuilder(args);
+   builder.Services.AddDirectoryBrowser();
+   var app = builder.Build();
+   var wsOpts = new WebSocketOptions {
+      KeepAliveInterval = TimeSpan.FromMinutes(2)
+   };
+   app.UseWebSockets(wsOpts);
+   app.UseDefaultFiles();
+   app.UseStaticFiles();
+   app.MapFallbackToFile("/index.html");
+
+   app.Use(async (context, next) => {
+      var reqDesc = fmtReq(context.Request);
+      Console.WriteLine($"Webserver received: {reqDesc}");
+      if (context.Request.Path == "/ws") {
+         if (context.WebSockets.IsWebSocketRequest) {
+            Console.WriteLine("Accepting websocket connection...");
+            using var webSocket = await context.WebSockets.AcceptWebSocketAsync();
+            Console.WriteLine("Websocket connection accepted.");
+            await runWebSocketServer(
+               opts.sparkd, opts.user!, opts.password!, webSocket);
+         } else {
+            context.Response.StatusCode = StatusCodes.Status400BadRequest;
+         }
+      } else {
+         Console.WriteLine($"Webserver ignoring req: {reqDesc}");
+         await next(context);
+      }
+   });
+
+   try {
+      await app.RunAsync();
+   } catch (Exception ex) {
+      Console.WriteLine($"Exception on DataStreamWebsocketServer thread: {ex}");
+      return 1;
+   }
+   Spark.Core.Disposables.Disposable.dispose(ref vsWorkLoop);
+   return 0;
+}
+
+async Task<int> runWebSocketServer(string? sparkd, string user, string password, WebSocket webSocket) {
+   var buffer = new byte[1024 * 4];
+   var receiveResult = await webSocket.ReceiveAsync(
+       new ArraySegment<byte>(buffer), CancellationToken.None);
+
+   try {
+      while (!receiveResult.CloseStatus.HasValue) {
+         Console.WriteLine("Awaiting websocket message from client");
+         receiveResult = await webSocket.ReceiveAsync(
+             new ArraySegment<byte>(buffer), CancellationToken.None);
+         var msg = Encoding.UTF8.GetString(buffer, 0, receiveResult.Count);
+         var rpcReq = JsonSerializer.Deserialize<Bts.Websocket.JsonRpcRequest<DataStreamSubscription>>(msg);
+         var queryArgs = rpcReq!.Params!;
+         // var queryArgs = new DataStreamSubscription { Query = msg };
+         var exp = DataStreamExpressionsInterpreter.Evaluate(queryArgs.Query!);
+         if (exp is null) {
+            Console.WriteLine($"Invalid query expression: {queryArgs.Query}.");
+            continue;
+         }
+         var vs = exp.getView(new Dictionary<string, StatementExpression>(0));
+         Console.WriteLine("Subscribing to stream...");
+         vs.onChange += onStreamChanged;
+
+         await webSocket.SendAsync(
+             new ArraySegment<byte>(buffer, 0, receiveResult.Count),
+             receiveResult.MessageType,
+             receiveResult.EndOfMessage,
+             CancellationToken.None);
+      }
+   } catch (Exception ex) {
+      Console.WriteLine($"Exception on WebsocketServer thread: {ex}");
+      return 1;
+   }
+
+   return 0;
+
+   async void onStreamChanged(object newValue) {
+      if (newValue is not DataPacket dp) {
+         Console.WriteLine(
+            $"OnStreamChanged delegate expected DataPacket (actually received {newValue?.GetType()}.");
+         return;
+      }
+      Console.WriteLine("Received stream update.");
+      var sb = new StringBuilder();
+      var updateDesc = fmtDataPacket(dp);
+      var msg = $"Received new rows:\n{updateDesc}";
+      // var payload = new ReadOnlyMemory<byte>([.. msg]);
+      var encoded = Encoding.UTF8.GetBytes(msg);
+      var payload = new ArraySegment<byte>(encoded, 0, encoded.Length);
+      await webSocket.SendAsync(
+          payload,
+          receiveResult.MessageType,
+          receiveResult.EndOfMessage,
+          CancellationToken.None);
+   }
+}

src/DataStreamSuite/Bts.DataStreamWebApp.Server/Properties/launchSettings.json

@@ -0,0 +1,48 @@
+{
+  "profiles": {
+    "default": {
+      "commandName": "Project",
+      "launchBrowser": true,
+      "applicationUrl": "http://localhost:5400"
+    },
+    "http": {
+      "commandName": "Project",
+      "launchBrowser": true,
+      "launchUrl": "swagger",
+      "environmentVariables": {
+        "ASPNETCORE_ENVIRONMENT": "Development",
+        "ASPNETCORE_HOSTINGSTARTUPASSEMBLIES": "Microsoft.AspNetCore.SpaProxy"
+      },
+      "dotnetRunMessages": true,
+      "applicationUrl": "http://localhost:5268"
+    },
+    "https": {
+      "commandName": "Project",
+      "launchUrl": "swagger",
+      "environmentVariables": {
+        "ASPNETCORE_ENVIRONMENT": "Development",
+        "ASPNETCORE_HOSTINGSTARTUPASSEMBLIES": "Microsoft.AspNetCore.SpaProxy"
+      },
+      "dotnetRunMessages": true,
+      "applicationUrl": "https://localhost:7128;http://localhost:5268"
+    },
+    "IIS Express": {
+      "commandName": "IISExpress",
+      "launchBrowser": true,
+      "launchUrl": "swagger",
+      "environmentVariables": {
+        "ASPNETCORE_ENVIRONMENT": "Development",
+        "ASPNETCORE_HOSTINGSTARTUPASSEMBLIES": "Microsoft.AspNetCore.SpaProxy"
+      }
+    }
+  },
+  "$schema": "http://json.schemastore.org/launchsettings.json",
+  "iisSettings": {
+    "windowsAuthentication": false,
+    "anonymousAuthentication": true,
+    "iisExpress": {
+      "applicationUrl": "http://localhost:2083",
+      "sslPort": 44383
+    }
+  }
+}

src/DataStreamSuite/Bts.DataStreamWebApp.Server/WeatherForecast.cs

@@ -0,0 +1,12 @@
+namespace Bts.DataStreamWebApp.Server;
+
+public class WeatherForecast
+{
+   public DateOnly Date { get; set; }
+
+   public int TemperatureC { get; set; }
+
+   public int TemperatureF => 32 + (int)(TemperatureC / 0.5556);
+
+   public string? Summary { get; set; }
+}

src/DataStreamSuite/Bts.DataStreamWebApp.Server/WebsocketConnectionManager.cs

@@ -0,0 +1,37 @@
+using System;
+using System.Collections.Concurrent;
+using System.Net.WebSockets;
+
+namespace Bts.DataStreamWebApp.Server {
+   // public class WebSocketConnectionManager {
+   //    private readonly ConcurrentDictionary<Guid, WebSocket> _sockets = new ConcurrentDictionary<Guid, WebSocket>();
+
+   //    public Guid AddSocket(WebSocket socket) {
+   //       var socketId = Guid.NewGuid();
+   //       _sockets.TryAdd(socketId, socket);
+   //       return socketId;
+   //    }
+
+   //    public WebSocket? GetSocket(Guid socketId) {
+   //       _sockets.TryGetValue(socketId, out var socket);
+   //       return socket;
+   //    }
+
+   //    public ConcurrentDictionary<Guid, WebSocket>.ValueCollection GetAllSockets() {
+   //       return _sockets.Values;
+   //    }
+
+   //    public Guid? GetSocketId(WebSocket socket) {
+   //       foreach (var (key, value) in _sockets) {
+   //          if (value == socket) {
+   //             return key;
+   //          }
+   //       }
+   //       return null;
+   //    }
+
+   //    public void RemoveSocket(Guid socketId) {
+   //       _sockets.TryRemove(socketId, out _);
+   //    }
+   // }
+}

src/DataStreamSuite/Bts.DataStreamWebApp.Server/appsettings.Development.json

@@ -0,0 +1,8 @@
+{
+  "Logging": {
+    "LogLevel": {
+      "Default": "Information",
+      "Microsoft.AspNetCore": "Warning"
+    }
+  }
+}

src/DataStreamSuite/Bts.DataStreamWebApp.Server/appsettings.json

@@ -0,0 +1,9 @@
+{
+  "Logging": {
+    "LogLevel": {
+      "Default": "Information",
+      "Microsoft.AspNetCore": "Warning"
+    }
+  },
+  "AllowedHosts": "*"
+}

src/DataStreamSuite/Bts.DataStreamWebApp.Server/wwwroot/alt-index.html

@@ -0,0 +1,168 @@
+<!DOCTYPE html>
+<html>
+<head>
+    <meta charset="utf-8" />
+    <title></title>
+    <style>
+        table {
+            border: 0
+        }
+
+        .commslog-data {
+            font-family: Consolas, Courier New, Courier, monospace;
+        }
+
+        .commslog-server {
+            background-color: red;
+            color: white
+        }
+
+        .commslog-client {
+            background-color: green;
+            color: white
+        }
+    </style>
+</head>
+<body>
+    <h1>ALT WebSocket Sample Application</h1>
+    <p id="stateLabel">Ready to connect...</p>
+    <div>
+        <label for="connectionUrl">WebSocket Server URL:</label>
+        <input id="connectionUrl" />
+        <button id="connectButton" type="submit">Connect</button>
+    </div>
+    <p></p>
+    <div>
+        <label for="sendMessage">Message to send:</label>
+        <input id="sendMessage" disabled />
+        <button id="sendButton" type="submit" disabled>Send</button>
+        <button id="closeButton" disabled>Close Socket</button>
+    </div>
+
+    <h2>Communication Log</h2>
+    <table style="width: 800px">
+        <thead>
+            <tr>
+                <td style="width: 100px">From</td>
+                <td style="width: 100px">To</td>
+                <td>Data</td>
+            </tr>
+        </thead>
+        <tbody id="commsLog">
+        </tbody>
+    </table>
+
+    <script>
+        var connectionUrl = document.getElementById("connectionUrl");
+        var connectButton = document.getElementById("connectButton");
+        var stateLabel = document.getElementById("stateLabel");
+        var sendMessage = document.getElementById("sendMessage");
+        var sendButton = document.getElementById("sendButton");
+        var commsLog = document.getElementById("commsLog");
+        var closeButton = document.getElementById("closeButton");
+        var socket;
+
+        var scheme = document.location.protocol === "https:" ? "wss" : "ws";
+        var port = document.location.port ? (":" + document.location.port) : "";
+
+        connectionUrl.value = scheme + "://" + document.location.hostname + port + "/ws" ;
+
+        function updateState() {
+            function disable() {
+                sendMessage.disabled = true;
+                sendButton.disabled = true;
+                closeButton.disabled = true;
+            }
+            function enable() {
+                sendMessage.disabled = false;
+                sendButton.disabled = false;
+                closeButton.disabled = false;
+            }
+
+            connectionUrl.disabled = true;
+            connectButton.disabled = true;
+
+            if (!socket) {
+                disable();
+            } else {
+                switch (socket.readyState) {
+                    case WebSocket.CLOSED:
+                        stateLabel.innerHTML = "Closed";
+                        disable();
+                        connectionUrl.disabled = false;
+                        connectButton.disabled = false;
+                        break;
+                    case WebSocket.CLOSING:
+                        stateLabel.innerHTML = "Closing...";
+                        disable();
+                        break;
+                    case WebSocket.CONNECTING:
+                        stateLabel.innerHTML = "Connecting...";
+                        disable();
+                        break;
+                    case WebSocket.OPEN:
+                        stateLabel.innerHTML = "Open";
+                        enable();
+                        break;
+                    default:
+                        stateLabel.innerHTML = "Unknown WebSocket State: " + htmlEscape(socket.readyState);
+                        disable();
+                        break;
+                }
+            }
+        }
+
+        closeButton.onclick = function () {
+            if (!socket || socket.readyState !== WebSocket.OPEN) {
+                alert("socket not connected");
+            }
+            socket.close(1000, "Closing from client");
+        };
+
+        sendButton.onclick = function () {
+            if (!socket || socket.readyState !== WebSocket.OPEN) {
+                alert("socket not connected");
+            }
+            var data = sendMessage.value;
+            socket.send(data);
+            commsLog.innerHTML += '<tr>' +
+                '<td class="commslog-client">Client</td>' +
+                '<td class="commslog-server">Server</td>' +
+                '<td class="commslog-data">' + htmlEscape(data) + '</td></tr>';
+        };
+
+        connectButton.onclick = function() {
+            stateLabel.innerHTML = "Connecting";
+            socket = new WebSocket(connectionUrl.value);
+            socket.onopen = function (event) {
+                updateState();
+                commsLog.innerHTML += '<tr>' +
+                    '<td colspan="3" class="commslog-data">Connection opened</td>' +
+                '</tr>';
+            };
+            socket.onclose = function (event) {
+                updateState();
+                commsLog.innerHTML += '<tr>' +
+                    '<td colspan="3" class="commslog-data">Connection closed. Code: ' + htmlEscape(event.code) + '. Reason: ' + htmlEscape(event.reason) + '</td>' +
+                '</tr>';
+            };
+            socket.onerror = updateState;
+            socket.onmessage = function (event) {
+                commsLog.innerHTML += '<tr>' +
+                    '<td class="commslog-server">Server</td>' +
+                    '<td class="commslog-client">Client</td>' +
+                    '<td class="commslog-data">' + htmlEscape(event.data) + '</td></tr>';
+            };
+        };
+
+        function htmlEscape(str) {
+            return str.toString()
+                .replace(/&/g, '&amp;')
+                .replace(/"/g, '&quot;')
+                .replace(/'/g, '&#39;')
+                .replace(/</g, '&lt;')
+                .replace(/>/g, '&gt;');
+        }
+    </script>
+</body>
+</html>

src/DataStreamSuite/Bts.DataStreamWebApp.Server/wwwroot/index.html

@@ -0,0 +1,176 @@
+<!DOCTYPE html>
+<html>
+<head>
+    <meta charset="utf-8" />
+    <title></title>
+    <style>
+        table {
+            border: 0
+        }
+
+        .commslog-data {
+            font-family: Consolas, Courier New, Courier, monospace;
+        }
+
+        .commslog-server {
+            background-color: red;
+            color: white
+        }
+
+        .commslog-client {
+            background-color: green;
+            color: white
+        }
+    </style>
+</head>
+<body>
+    <h1>DataStreams via WebSocket</h1>
+    <p id="stateLabel">Ready to connect...</p>
+    <div>
+        <label for="connectionUrl">WebSocket Server URL:</label>
+        <input id="connectionUrl" />
+        <button id="connectButton" type="submit">Connect</button>
+    </div>
+    <p></p>
+    <div>
+        <label for="sendMessage">Query:</label>
+        <input id="sendMessage" disabled />
+        <button id="sendButton" type="submit" disabled>Send</button>
+        <button id="closeButton" disabled>Close Socket</button>
+    </div>
+
+    <h2>Communication Log</h2>
+    <table style="width: 800px">
+        <thead>
+            <tr>
+                <td style="width: 100px">From</td>
+                <td style="width: 100px">To</td>
+                <td>Data</td>
+            </tr>
+        </thead>
+        <tbody id="commsLog">
+        </tbody>
+    </table>
+
+    <script>
+        var connectionUrl = document.getElementById("connectionUrl");
+        var connectButton = document.getElementById("connectButton");
+        var stateLabel = document.getElementById("stateLabel");
+        var sendMessage = document.getElementById("sendMessage");
+        var sendButton = document.getElementById("sendButton");
+        var commsLog = document.getElementById("commsLog");
+        var closeButton = document.getElementById("closeButton");
+        var socket;
+
+        var scheme = document.location.protocol === "https:" ? "wss" : "ws";
+        var port = document.location.port ? (":" + document.location.port) : "";
+
+        connectionUrl.value = scheme + "://" + document.location.hostname + port + "/ws" ;
+
+        function updateState() {
+            function disable() {
+                sendMessage.disabled = true;
+                sendButton.disabled = true;
+                closeButton.disabled = true;
+            }
+            function enable() {
+                sendMessage.disabled = false;
+                sendButton.disabled = false;
+                closeButton.disabled = false;
+            }
+
+            connectionUrl.disabled = true;
+            connectButton.disabled = true;
+
+            if (!socket) {
+                disable();
+            } else {
+                switch (socket.readyState) {
+                    case WebSocket.CLOSED:
+                        stateLabel.innerHTML = "Closed";
+                        disable();
+                        connectionUrl.disabled = false;
+                        connectButton.disabled = false;
+                        break;
+                    case WebSocket.CLOSING:
+                        stateLabel.innerHTML = "Closing...";
+                        disable();
+                        break;
+                    case WebSocket.CONNECTING:
+                        stateLabel.innerHTML = "Connecting...";
+                        disable();
+                        break;
+                    case WebSocket.OPEN:
+                        stateLabel.innerHTML = "Open";
+                        enable();
+                        break;
+                    default:
+                        stateLabel.innerHTML = "Unknown WebSocket State: " + htmlEscape(socket.readyState);
+                        disable();
+                        break;
+                }
+            }
+        }
+
+        closeButton.onclick = function () {
+            if (!socket || socket.readyState !== WebSocket.OPEN) {
+                alert("socket not connected");
+            }
+            socket.close(1000, "Closing from client");
+        };
+
+        sendButton.onclick = function () {
+            if (!socket || socket.readyState !== WebSocket.OPEN) {
+                alert("socket not connected");
+            }
+            const rpcReq = {
+              jsonrpc: "2.0",
+              method: "subscribe",
+              params: {
+                query: sendMessage.value,
+              },
+              id: 1,
+            };
+            const rpcReqStr = JSON.stringify(rpcReq);
+            socket.send(rpcReqStr);
+            commsLog.innerHTML += '<tr>' +
+                '<td class="commslog-client">Client</td>' +
+                '<td class="commslog-server">Server</td>' +
+                '<td class="commslog-data">' + htmlEscape(rpcReqStr) + '</td></tr>';
+        };
+
+        connectButton.onclick = function() {
+            stateLabel.innerHTML = "Connecting";
+            socket = new WebSocket(connectionUrl.value);
+            socket.onopen = function (event) {
+                updateState();
+                commsLog.innerHTML += '<tr>' +
+                    '<td colspan="3" class="commslog-data">Connection opened</td>' +
+                '</tr>';
+            };
+            socket.onclose = function (event) {
+                updateState();
+                commsLog.innerHTML += '<tr>' +
+                    '<td colspan="3" class="commslog-data">Connection closed. Code: ' + htmlEscape(event.code) + '. Reason: ' + htmlEscape(event.reason) + '</td>' +
+                '</tr>';
+            };
+            socket.onerror = updateState;
+            socket.onmessage = function (event) {
+                commsLog.innerHTML += '<tr>' +
+                    '<td class="commslog-server">Server</td>' +
+                    '<td class="commslog-client">Client</td>' +
+                    '<td class="commslog-data">' + htmlEscape(event.data) + '</td></tr>';
+            };
+        };
+
+        function htmlEscape(str) {
+            return str.toString()
+                .replace(/&/g, '&amp;')
+                .replace(/"/g, '&quot;')
+                .replace(/'/g, '&#39;')
+                .replace(/</g, '&lt;')
+                .replace(/>/g, '&gt;');
+        }
+    </script>
+</body>
+</html>

src/DataStreamSuite/ColumnMapExpressions/ColumnMapExpressions.csproj

@@ -1,7 +1,7 @@
 <Project Sdk="Microsoft.NET.Sdk">
 
   <PropertyGroup>
-    <TargetFramework>net7.0</TargetFramework>
+    <TargetFramework>net8.0</TargetFramework>
     <ImplicitUsings>enable</ImplicitUsings>
     <Nullable>enable</Nullable>
   </PropertyGroup>

src/DataStreamSuite/DataStream.Console/DataStream.Console.csproj

@@ -7,7 +7,7 @@
 
   <PropertyGroup>
     <OutputType>Exe</OutputType>
-    <TargetFramework>net7.0</TargetFramework>
+    <TargetFramework>net8.0</TargetFramework>
     <ImplicitUsings>enable</ImplicitUsings>
     <Nullable>enable</Nullable>
   </PropertyGroup>

src/DataStreamSuite/DataStream/DataStream.csproj

@@ -1,7 +1,7 @@
 <Project Sdk="Microsoft.NET.Sdk">
 
   <PropertyGroup>
-    <TargetFramework>net7.0</TargetFramework>
+    <TargetFramework>net8.0</TargetFramework>
     <Nullable>enable</Nullable>
   </PropertyGroup>
 
@@ -15,6 +15,7 @@
   </ItemGroup>
 
   <ItemGroup>
+    <PackageReference Include="HtmlAgilityPack" Version="1.11.57" />
     <PackageReference Include="MongoDB.Bson" Version="2.23.1" />
     <PackageReference Include="MongoDB.Driver" Version="2.23.1" />
   </ItemGroup>

src/DataStreamSuite/DataStream/Primitives/FileStreams/LoaderStream.cs

@@ -30,8 +30,8 @@ namespace BCG
            {
               using (FileStream fs = new FileStream(fileName, FileMode.Open, FileAccess.Read))
               {
-                  BinaryFormatter bf = new BinaryFormatter();
 #pragma warning disable SYSLIB0011
+                  BinaryFormatter bf = new BinaryFormatter();
                   BlueTable bt = (BlueTable)bf.Deserialize(fs);
 #pragma warning restore SYSLIB0011
                   mySetRealSource(new BlueRowCollectionStaticStream(bt));

src/DataStreamSuite/DataStream/Primitives/KeyValue/KeyValueToRelationalStream.cs

@@ -0,0 +1,31 @@
+namespace BCG {
+    using System.Collections.Generic;
+
+    public class KeyValueToRelationalStream : ViewStream {
+        private readonly ViewStream _kv_str;
+        private readonly string _doc_id_col, _key_col, _val_col;
+        public KeyValueToRelationalStream(ViewStream kv_str, string doc_id_col, string key_col, string val_col) {
+            _kv_str = kv_str;
+            _doc_id_col = doc_id_col;
+            _key_col = key_col;
+            _val_col = val_col;
+        }
+
+        protected override void myOnFirstSubscriber() {
+            _kv_str.onChange += _on_str_update;
+        }
+
+        protected override void myOnLastSubscriber() {
+            _kv_str.onChange -= _on_str_update;
+        }
+
+        private void _on_str_update(object dp_obj) {
+            DataPacket dp = dp_obj as DataPacket;
+            var rows = dp.newRows;
+            var rv = BlueDataUtility.keyvalue_to_relational<string>(rows, _doc_id_col, _key_col, _val_col);
+            var dp_out = BlueDataUtility.build_dp_inc(rv, myLastValue as DataPacket, true);
+            myFireEvent(dp_out);
+        }
+
+    }
+}

src/DataStreamSuite/DataStream/Primitives/Misc/WebScrapingStream.cs

@@ -0,0 +1,121 @@
+using System;
+using System.Collections.Generic;
+using System.IO;
+using System.Security.Cryptography.X509Certificates;
+
+using BCG;
+using BCG.Utilities.General;
+
+using HtmlAgilityPack;
+
+namespace BCG {
+    class Program {
+        static void Main() {
+            string url = "http://quotes.toscrape.com";
+            List<string> quotes = ScrapeQuotes(url);
+
+            Console.WriteLine("Quotes from http://quotes.toscrape.com:");
+            foreach (var quote in quotes) {
+                Console.WriteLine(quote);
+            }
+        }
+
+        static List<string> ScrapeQuotes(string url) {
+            List<string> quotes = new List<string>();
+
+            // Create a new HtmlWeb instance
+            HtmlWeb web = new HtmlWeb();
+
+            try {
+                // Load the HTML document from the specified URL
+                HtmlDocument document = web.Load(url);
+
+                // Extract quotes from the HTML document
+                var quoteNodes = document.DocumentNode.SelectNodes("//span[@class='text']");
+                if (quoteNodes != null) {
+                    foreach (var quoteNode in quoteNodes) {
+                        string quote = quoteNode.InnerText.Trim();
+                        quotes.Add(quote);
+                    }
+                }
+            } catch (Exception ex) {
+                Console.WriteLine($"Error: {ex.Message}");
+            }
+
+            return quotes;
+        }
+    }
+
+    public class WebScapperStream : ViewStream, IWorker {
+        private readonly string _url;
+        private readonly string _xpath;
+        private readonly ViewStreamDatabase _vsd;
+        private IDictionary<string, int> _line_id_map = new Dictionary<string, int>();
+        private readonly IBlueTableSchema _schema;
+
+        public WebScapperStream()
+        :
+        this("http://quotes.toscrape.com", "//span[@class='text']", ViewStreamDatabase.instance) { }
+        public WebScapperStream(string url, string xpath)
+        : this(url, xpath, ViewStreamDatabase.instance) { }
+        public WebScapperStream(string url, string xpath, ViewStreamDatabase vsd) {
+            bts.Log.Gen.info($"WebScapperStream cons url: {url} xpath: {xpath}");
+            _schema = BlueDataUtility.form_schema($"{url}_{xpath}",
+                                                Tuple.Create("id", TypeCode.Int32),
+                                                Tuple.Create("url", TypeCode.UInt32),
+                                                Tuple.Create("line", TypeCode.String));
+            _url = url;
+            _xpath = xpath;
+            _vsd = vsd;
+        }
+
+        protected override void myOnFirstSubscriber() {
+            MasterWorker.getInstance(false).addWorker(this, TimeSpan.FromSeconds(1));
+        }
+
+        protected override void myOnLastSubscriber() {
+            MasterWorker.getInstance(false).removeWorker(this);
+        }
+
+        public void doWork() {
+            var lines = scrape_lines(_url, _xpath);
+            Func<string, int> get_id = (line) => {
+                int rv = 0;
+                if (!_line_id_map.TryGetValue(line, out rv))
+                    _line_id_map.Add(line, _line_id_map.Count);
+                return rv;
+            };
+            BlueTable nrows = new BlueTable(_schema);
+            foreach (var l in lines) {
+                bts.Log.Gen.info($"line: {l}");
+                var id = get_id(l);
+                BlueRow nr = new BlueRow(_schema, get_id(l), _url, l);
+                nrows.addRow(nr);
+            }
+            var dp = BlueDataUtility.build_dp_inc(nrows, myLastValue as DataPacket, true);
+            myFireEvent(dp);
+        }
+
+        static List<string> scrape_lines(string url, string xpath) {
+            List<string> quotes = new List<string>();
+            // Create a new HtmlWeb instance
+            HtmlWeb web = new HtmlWeb();
+            try {
+                // Load the HTML document from the specified URL
+                HtmlDocument document = web.Load(url);
+                // Extract quotes from the HTML document
+                var quote_nodes = document.DocumentNode.SelectNodes(xpath);
+                if (quote_nodes != null) {
+                    foreach (var qn in quote_nodes) {
+                        string quote = qn.InnerText.Trim();
+                        quotes.Add(quote);
+                    }
+                }
+            } catch (Exception ex) {
+                Console.WriteLine($"Error: {ex.Message}");
+            }
+
+            return quotes;
+        }
+    }
+}

src/DataStreamSuite/DataStream/Primitives/MongoDB/MongoDBStream.cs

@@ -1,37 +1,212 @@
 using System;
-
+using System.Collections.Generic;
 using MongoDB.Driver;
 using MongoDB.Bson;
+using BCG;
+using BCG.Utilities.General;
+using ZstdSharp.Unsafe;
+using System.Runtime.CompilerServices;
+using mbts.spark;
 
 
-class Program {
-    static void Main() {
-        // Set the connection string and database name
-        string connectionString = "mongodb://localhost:27017";
-        string databaseName = "your_database_name";
-        string collectionName = "your_collection_name";
-
-        // Create a MongoClient
-        MongoClient client = new MongoClient(connectionString);
+namespace BCG {
+    [DataStreamUserRespresentation("mongodb_sql")]
+    public class MongoDBSQLStream : ViewStream, IWorker {
+        private ViewStreamDatabase _vsd;
+        private readonly string _connection_string, _db_name, _collection_name;
+        private MongoClient _client;
+        private BlueTable _table;
 
         // Access the database
-        IMongoDatabase database = client.GetDatabase(databaseName);
+        private IMongoDatabase _database;
+        private IDictionary<string, int> _id_map = new Dictionary<string, int>();
+        // mongodb://192.168.1.80:27017
 
-        // Access the collection
-        IMongoCollection<BsonDocument> collection = database.GetCollection<BsonDocument>(collectionName);
+        public MongoDBSQLStream(string connection_string, string db_name, string collection_name)
+        : this(connection_string, db_name, collection_name, ViewStreamDatabase.instance) {
 
-        // Fetch all documents from the collection
-        var documents = collection.Find(Builders<BsonDocument>.Filter.Empty).ToCursor();
+        }
+        public MongoDBSQLStream(string connection_string, string db_name, string collection_name, ViewStreamDatabase vsd) {
+            bts.Log.Gen.info($"MongoDBStream constructed connection_string: {connection_string} " +
+                                    $"db_name: {db_name}");
+            _connection_string = connection_string;
+            _db_name = db_name;
+            _collection_name = collection_name;
+            _vsd = vsd;
+            _client = new MongoClient(_connection_string);
+            _database = _client.GetDatabase(_db_name);
+        }
 
-        // Iterate through the documents and print values
-        foreach (var document in documents.ToEnumerable()) {
-            // Assuming your document structure has fields named "Field1", "Field2", etc.
-            Console.WriteLine("Field1: " + document["Field1"]);
-            Console.WriteLine("Field2: " + document["Field2"]);
-            // Add more fields as needed
+        protected override void myOnFirstSubscriber() {
+            MasterWorker.getInstance(false).addWorker(this, TimeSpan.FromSeconds(1));
+        }
 
-            // Print a separator between documents
-            Console.WriteLine(new string('-', 20));
+        protected override void myOnLastSubscriber() {
+            MasterWorker.getInstance(false).removeWorker(this);
+        }
+
+        public void doWork() {
+            _read_mogodb();
+        }
+
+        private void _read_mogodb() {
+            HighPerformanceTimer timer = new HighPerformanceTimer();
+            timer.start();
+            // Access the collection
+            IMongoCollection<BsonDocument> collection = _database.GetCollection<BsonDocument>(name: _collection_name);
+
+            // Fetch all documents from the collection
+            var documents = collection.Find(Builders<BsonDocument>.Filter.Empty).ToCursor();
+            _table = _table == null ? null : new BlueTable(_table.schema);
+            // Iterate through the documents and print values
+            foreach (var document in documents.ToEnumerable()) {
+                bts.Log.Gen.info($"document: {document} json: {document.ToJson()}");
+                if (_table == null) {
+                    _init_table(document);
+                }
+                _add_row(document);
+            }
+            timer.stop();
+            bts.Log.Gen.info($"MongoDBStream time to read {_collection_name} {timer.lastIntervalTime} msecs");
+            timer.start();
+            var dp = BlueDataUtility.build_dp_inc(_table, myLastValue == null ? null : myLastValue as DataPacket, true);
+            timer.stop();
+            bts.Log.Gen.info($"MongoDBStream time to compare tables :{timer.lastIntervalTime} msecs");
+            if (dp.addedRows.Count == 0 && dp.deletedRows.Count == 0)
+                return;
+            myFireEvent(dp);
+        }
+
+        private void _init_table(BsonDocument doc_row) {
+            bts.Log.assert(_table == null);
+            List<ColumnInfo> cis = new List<ColumnInfo>();
+            cis.Add((new ColumnInfo("id", 0, typeof(Int32))));
+            foreach (var n in doc_row.Names) {
+                cis.Add(new ColumnInfo(n, cis.Count, typeof(string)));
+            }
+            IBlueTableSchema schema = new BlueTableSchema(cis.ToArray(), _collection_name);
+            _table = new BlueTable(schema);
+        }
+
+        private void _add_row(BsonDocument doc) {
+            var key_val = doc["_id"];
+            var key = key_val.ToString();
+            int id;
+            if (!_id_map.TryGetValue(key, out id)) {
+                id = _id_map.Count;
+                _id_map.Add(key, id);
+            }
+            MutableBlueRow nr = new MutableBlueRow(_table.schema);
+            nr.setValue("id", id);
+            foreach (var n in doc.Names) {
+                var val = doc[n].ToString();
+                nr.setValue(n, val);
+            }
+            _table.addRow(nr);
         }
     }
-}
+
+    [DataStreamUserRespresentation("mongodb")]
+    public class MongoDBStream : ViewStream, IWorker {
+        private ViewStreamDatabase _vsd;
+        private readonly string _connection_string, _db_name, _collection_name;
+        private MongoClient _client;
+        private BlueTable _table;
+
+        // Access the database
+        private IMongoDatabase _database;
+        private IDictionary<string, int> _id_map = new Dictionary<string, int>();
+        // mongodb://192.168.1.80:27017
+
+        public MongoDBStream(string connection_string, string db_name, string collection_name)
+        : this(connection_string, db_name, collection_name, ViewStreamDatabase.instance) {
+
+        }
+        public MongoDBStream(string connection_string, string db_name, string collection_name, ViewStreamDatabase vsd) {
+            bts.Log.Gen.info($"MongoDBStream constructed connection_string: {connection_string} " +
+                                    $"db_name: {db_name}");
+            _connection_string = connection_string;
+            _db_name = db_name;
+            _collection_name = collection_name;
+            _vsd = vsd;
+            _client = new MongoClient(_connection_string);
+            _database = _client.GetDatabase(_db_name);
+        }
+
+        protected override void myOnFirstSubscriber() {
+            MasterWorker.getInstance(false).addWorker(this, TimeSpan.FromSeconds(1));
+        }
+
+        protected override void myOnLastSubscriber() {
+            MasterWorker.getInstance(false).removeWorker(this);
+        }
+
+        public void doWork() {
+            _read_mogodb();
+        }
+
+        private void _read_mogodb() {
+            HighPerformanceTimer timer = new HighPerformanceTimer();
+            timer.start();
+            // Access the collection
+            IMongoCollection<BsonDocument> collection = _database.GetCollection<BsonDocument>(name: _collection_name);
+
+            // Fetch all documents from the collection
+            var documents = collection.Find(Builders<BsonDocument>.Filter.Empty).ToCursor();
+            _table = _table == null ? null : new BlueTable(_table.schema);
+            // Iterate through the documents and print values
+            foreach (var document in documents.ToEnumerable()) {
+                bts.Log.Gen.info($"document: {document} json: {document.ToJson()}");
+                if (_table == null) {
+                    _init_table(document);
+                }
+                _add_doc(document);
+            }
+            timer.stop();
+            bts.Log.Gen.info($"MongoDBStream time to read {_collection_name} {timer.lastIntervalTime} msecs");
+            timer.start();
+            var dp = BlueDataUtility.build_dp_inc(_table, myLastValue == null ? null : myLastValue as DataPacket, true);
+            timer.stop();
+            bts.Log.Gen.info($"MongoDBStream time to compare tables :{timer.lastIntervalTime} msecs");
+            if (dp.addedRows.Count == 0 && dp.deletedRows.Count == 0)
+                return;
+            myFireEvent(dp);
+        }
+
+        private void _init_table(BsonDocument doc_row) {
+            bts.Log.assert(_table == null);
+            var schema = BlueDataUtility.form_schema($"{_database}_{_table}",
+            Tuple.Create("id", TypeCode.Int32),
+            Tuple.Create("doc_id", TypeCode.String),
+            Tuple.Create("key", TypeCode.String),
+            Tuple.Create("value", TypeCode.String));
+            _table = new BlueTable(schema);
+        }
+
+        private void _add_doc(BsonDocument doc) {
+            var doc_id_bson = doc["_id"];
+            var doc_id = doc_id_bson.ToString();
+            Func<string, string, int> get_id = (string doc_id, string key) => {
+                string hash_key = $"{doc_id}_{key}";
+                int id = 0;
+                if (!_id_map.TryGetValue(hash_key, out id)) {
+                    id = _id_map.Count;
+                    _id_map.Add(hash_key, id);
+                }
+                return id;
+            };
+            foreach (var n in doc.Names) {
+                string key = n;
+                var id = get_id(doc_id, key);
+                MutableBlueRow nr = new MutableBlueRow(_table.schema);
+                var val = doc[n].ToString();
+                nr.setValue("id", id);
+                nr.setValue("doc_id", doc_id);
+                nr.setValue("key", key);
+                nr.setValue("value", val);
+                _table.addRow(nr);
+            }
+        }
+    }
+
+}

src/DataStreamSuite/DataStreamCore/BlueData/ColumnInfo.cs

@@ -1,95 +1,103 @@
-using System;
-
-namespace BCG
-{
-   /// <summary>
-   /// Resprsents information about a column schema, ie its ordinal, the type, the name and constant value or expression. where applicable
-   /// </summary>
-   [Serializable]
-   public class ColumnInfo : IComparable
-   {
-      public static bool isAllowedType (Type t)
-      {
-         return 
-            t.Equals (typeof (System.Int32)) ||
-            t.Equals (typeof (System.Int64)) ||
-            t.Equals (typeof (System.Double)) ||
-            t.Equals (typeof (System.String)) ||
-            t.Equals (typeof (System.DateTime)) ? true :
-            false;
-      }
-
-      //the default column info does not cach values in a lookup table, therefore it is in group 0
-      public ColumnInfo(string colName, int colNo, Type colType)
-         : this(colName, colNo, colType, 0)
-      {
-
-
-      }
-
-      public ColumnInfo(string colName, int colNo, Type colType, short colGroup)
-      {
-         myColName = colName;
-         myColNo = colNo;
-         myColType = colType;
-         myLookupTableGroup = colGroup;
-         myColValue = null;
-      }
-
-
-      public ColumnInfo (string colName, int colNo, object colValue)
-         : this (colName, colNo, colValue.GetType())  {myColValue = colValue == null ? DBNull.Value : colValue;}
-
-      public ColumnInfo (string colName, int colNo, Type colType, object colValue)
-         : this (colName, colNo, colType) 
-      {
-
-         myColValue = colValue == null ? DBNull.Value : colValue;
-      
-      }
-
-
-      public int CompareTo (object obj)
-      {
-         int otherColNo = ((ColumnInfo)(obj)).colNo;
-         if (myColNo < otherColNo)
-            return -1;
-         else if (myColNo > otherColNo)
-            return 1;
-         return 0;
-      }
-
-      //public bool isCalculated{get{return myExpression != null;}}
-
-      public bool isConstant {get{return myColValue != null;}}
-
-      private int myColNo;
-      private string myColName;
-      private Type myColType;
-      //private string myExpression;
-      private object myColValue;
-
-      //no table will ever have more than 65,536 columns
-      //if it does, we are in serious trouble
-      private short myLookupTableGroup;
-
-      public int colNo{get{return myColNo;}}
-      public string colName {get{return myColName;}}
-      public Type colType {get{return myColType;}}
-
-
-      //this value corresponds to how the data in the column is serialized. 
-      //if the group number is 0, the values are not stored in a table
-      //if the vlaue is non-zero, all columns with the the same lookup table group are stored in the same lookup table together
-      //are stored together in a lookup table
-
-      
-      public short lookupTableGroup
-      {
-         get { return myLookupTableGroup; }
-         set { myLookupTableGroup = value; }
-      }
-      //public string expression {get{return myExpression;}}
-      public object colValue {get{return myColValue;}}
-   }
-}
+using System;
+using System.Text.Json.Serialization;
+
+namespace BCG
+{
+   /// <summary>
+   /// Resprsents information about a column schema, ie its ordinal, the type, the name and constant value or expression. where applicable
+   /// </summary>
+   [Serializable]
+   public class ColumnInfo : IComparable
+   {
+      public static bool isAllowedType (Type t)
+      {
+         return 
+            t.Equals (typeof (System.Int32)) ||
+            t.Equals (typeof (System.Int64)) ||
+            t.Equals (typeof (System.Double)) ||
+            t.Equals (typeof (System.String)) ||
+            t.Equals (typeof (System.DateTime)) ? true :
+            false;
+      }
+
+      //the default column info does not cach values in a lookup table, therefore it is in group 0
+      public ColumnInfo(string colName, int colNo, Type colType)
+         : this(colName, colNo, colType, 0)
+      {
+
+
+      }
+
+      public ColumnInfo(string colName, int colNo, Type colType, short colGroup)
+      {
+         myColName = colName;
+         myColNo = colNo;
+         myColType = colType;
+         myLookupTableGroup = colGroup;
+         myColValue = null;
+      }
+
+
+      public ColumnInfo (string colName, int colNo, object colValue)
+         : this (colName, colNo, colValue.GetType())  {myColValue = colValue == null ? DBNull.Value : colValue;}
+
+      public ColumnInfo (string colName, int colNo, Type colType, object colValue)
+         : this (colName, colNo, colType) 
+      {
+
+         myColValue = colValue == null ? DBNull.Value : colValue;
+      
+      }
+
+
+      public int CompareTo (object obj)
+      {
+         int otherColNo = ((ColumnInfo)(obj)).colNo;
+         if (myColNo < otherColNo)
+            return -1;
+         else if (myColNo > otherColNo)
+            return 1;
+         return 0;
+      }
+
+      //public bool isCalculated{get{return myExpression != null;}}
+
+      public bool isConstant {get{return myColValue != null;}}
+
+      private int myColNo;
+      private string myColName;
+      private Type myColType;
+      //private string myExpression;
+      private object myColValue;
+
+      //no table will ever have more than 65,536 columns
+      //if it does, we are in serious trouble
+      private short myLookupTableGroup;
+
+      [JsonPropertyName("colNo")]
+      public int colNo{get{return myColNo;}}
+
+      [JsonPropertyName("colName")]
+      public string colName {get{return myColName;}}
+
+      [JsonPropertyName("colType")]
+      public Type colType {get{return myColType;}}
+
+
+      //this value corresponds to how the data in the column is serialized. 
+      //if the group number is 0, the values are not stored in a table
+      //if the vlaue is non-zero, all columns with the the same lookup table group are stored in the same lookup table together
+      //are stored together in a lookup table
+
+      
+      [JsonIgnore]
+      public short lookupTableGroup
+      {
+         get { return myLookupTableGroup; }
+         set { myLookupTableGroup = value; }
+      }
+      //public string expression {get{return myExpression;}}
+      [JsonIgnore]
+      public object colValue {get{return myColValue;}}
+   }
+}

src/DataStreamSuite/DataStreamCore/BlueData/MetaDataField.cs

@@ -3,8 +3,7 @@ using System.Collections.Generic;
 using System.Linq;
 using System.Text;
 
-namespace BCG
-{
+namespace BCG {
    /*
       These fields are first going to be used by  BlueRay to help it determine what differnet tags mean, but eventually
     * the goal is that they will be used for all services in this system. 
@@ -12,11 +11,12 @@ namespace BCG
     * not all metadata fields are present in all metadata; it depends upon the class. For example, a bluepages service
     * probably won't have a 'symbol' or 'exchange' field populated, but it'll probably have the 'type' of service populated
     */
-   public enum MetaDataField {   Protocol,  //is this a bluepages service, a pubsub tag, a blueray tag, a 29 west thing, or what
-                                 Type,   //if it's a multicast, is it option, fireprice, whatever. If it's bluepages, is it persistence, pubsub, whatever
-                                 Symbol, 
-                                 Exchange,
-                                 Platform,
-                                 MarketDataType, // BAM (bid, ask, mid), last, etc
-                                 RealName }; //does this thing have a better name we could call it
+   public enum MetaDataField {
+      Protocol,       // is this a bluepages service, a pubsub tag, a blueray tag, a 29 west thing, or what
+      Type,           // if it's a multicast, is it option, fireprice, whatever. If it's bluepages, is it persistence, pubsub, whatever
+      Symbol,
+      Exchange,
+      MarketDataType, // BAM (bid, ask, mid), last, etc
+      RealName
+   }; //does this thing have a better name we could call it
 }

src/DataStreamSuite/DataStreamCore/BlueRowJsonConverter.cs

@@ -0,0 +1,61 @@
+#nullable enable
+using BCG;
+
+using System;
+using System.Text.Json;
+using System.Text.Json.Serialization;
+
+namespace Bts.DataStreamCore;
+
+class BlueRowJsonConverter : JsonConverter<IBlueRow> {
+   IBlueTableSchema? schema = null;
+
+   public override IBlueRow Read(
+      ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options) {
+      if (schema is null) {
+         throw new InvalidOperationException("Cannot read row from json because schema is null");
+      }
+      var mbr = new MutableBlueRow(schema);
+      int colNum = 0;
+      switch (reader.TokenType) {
+         case JsonTokenType.StartArray:
+            while (reader.Read() && reader.TokenType != JsonTokenType.EndArray) {
+               mbr.setValue(colNum++, readCellValue(ref reader));
+               colNum++;
+            }
+            break;
+         case JsonTokenType.StartObject:
+            while (reader.Read() && reader.TokenType != JsonTokenType.EndObject) {
+               if (reader.TokenType != JsonTokenType.PropertyName) {
+                  throw new InvalidOperationException(
+                     "Expected 'PropertyName' immediately after 'StartObject' token in row json");
+               }
+               var colName = reader.GetString();
+               if (!reader.Read()) {
+                  throw new InvalidOperationException(
+                     "Failed to read token immediately after 'PropertyName' token in row json");
+               }
+               mbr.setValue(colName, readCellValue(ref reader));
+            }
+            break;
+      }
+      if (mbr.Count != schema.columnInfo.Length) {
+         throw new InvalidOperationException(
+            $"Row json array length '{mbr.Count}' does not match the schema's " +
+            $"column length '{schema.columnInfo.Length}'.");
+      }
+      return mbr;
+
+      static object? readCellValue(ref Utf8JsonReader reader) =>
+         reader.TokenType switch {
+            JsonTokenType.String => reader.GetString(),
+            JsonTokenType.Number => reader.GetDouble(),
+            JsonTokenType.True or JsonTokenType.False => reader.GetBoolean(),
+            _ => throw new InvalidOperationException("Failed to read cell value"),
+         };
+   }
+
+   public override void Write(Utf8JsonWriter writer, IBlueRow value, JsonSerializerOptions options) {
+      throw new NotImplementedException();
+   }
+}

src/DataStreamSuite/DataStreamCore/DataPacketJsonConverter.cs

@@ -0,0 +1,154 @@
+#nullable enable
+using BCG;
+
+using Spark.Core.Json;
+
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text.Json;
+using System.Text.Json.Serialization;
+
+namespace Bts.DataStreamCore;
+
+class DataPacketJsonConverter : JsonConverter<DataPacket> {
+   /// <summary>
+   /// Whether or not to include a field for current rows (i.e., snapshot of current state). By
+   /// default, only incremental changes are included ('added' and 'deleted').
+   /// </summary>
+   public bool IncludeCurrent { get; set; }
+
+   /// <summary>
+   /// Whether to treat rows as objects (name/value pairs), or arrays (list of values). Arrays
+   /// result in smaller payloads, but aren't as easy to read as name/value pairs (for mere humans).
+   /// </summary>
+   public bool UseObjectRows { get; set; }
+
+   public override DataPacket? Read(ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options) {
+      if (!reader.ReadObject()) {
+         return null;
+      }
+      if (!reader.ReadPropertyName(out var prop)) {
+         return null;
+      }
+      if (!reader.Read()) {
+         return null;
+      }
+      if (prop == "added") {
+         var added = new List<MutableBlueRow>();
+         if (reader.TokenType != JsonTokenType.StartArray) {
+            return null;
+         }
+         while (true) {
+            if (!reader.Read()) {
+               break;
+            }
+            if (reader.TokenType != JsonTokenType.StartArray || !reader.Read()) {
+               return null;
+            }
+            switch (reader.TokenType) {
+               case JsonTokenType.String:
+                  // DAMNIT! Need the schema.
+                  break;
+            }
+         }
+      }
+   }
+
+   public override void Write(Utf8JsonWriter writer, DataPacket value, JsonSerializerOptions options) {
+      using var _ = writer.WriteObject();
+      using (writer.WriteArray("schema")) {
+         foreach (var col in value.schema.columnInfo) {
+            using (writer.WriteObject()) {
+               writer.WriteString("name", col.colName);
+               writer.WriteNumber("index", col.colNo);
+               // TODO: use language agnostic type names (e.g., 'string', 'integer', 'float', 'date' etc.)
+               writer.WriteString("type", col.colType.ToString());
+            }
+         }
+      }
+
+      var cellWriters = value.schema.columnInfo.Select(getCellWriter).ToList();
+      using (writer.WriteArray("added")) {
+         foreach (IBlueRow row in value.addedRows) {
+            using (writer.WriteArray()) {
+               for (int i = 0; i < cellWriters.Count; i++) {
+                  cellWriters[i](row[i]);
+               }
+            }
+         }
+      }
+      using (writer.WriteArray("deleted")) {
+         foreach (IBlueRow row in value.deletedRows) {
+            using (writer.WriteArray()) {
+               for (int i = 0; i < cellWriters.Count; i++) {
+                  cellWriters[i](row[i]);
+               }
+            }
+         }
+      }
+      if (IncludeCurrent) {
+         using (writer.WriteArray("current")) {
+            foreach (IBlueRow row in value.newRows) {
+               using (writer.WriteArray()) {
+                  for (int i = 0; i < cellWriters.Count; i++) {
+                     cellWriters[i](row[i]);
+                  }
+               }
+            }
+         }
+      }
+
+      Action<object?> getCellWriter(ColumnInfo colInfo) {
+         var writeVal = getCellValueWriter(colInfo);
+         if (!UseObjectRows) {
+            return writeVal;
+         }
+         return v => {
+            writer.WritePropertyName(colInfo.colName);
+            writeVal(v);
+         };
+      }
+
+      Action<object?> getCellValueWriter(ColumnInfo colInfo) {
+         var t = colInfo.colType;
+         if (t == typeof(int)) {
+            return nullCheckWrapper(v => writer.WriteNumberValue((int)v));
+         }
+         if (t == typeof(uint)) {
+            return nullCheckWrapper(v => writer.WriteNumberValue((uint)v));
+         }
+         if (t == typeof(long)) {
+            return nullCheckWrapper(v => writer.WriteNumberValue((long)v));
+         }
+         if (t == typeof(ulong)) {
+            return nullCheckWrapper(v => writer.WriteNumberValue((ulong)v));
+         }
+         if (t == typeof(short)) {
+            return nullCheckWrapper(v => writer.WriteNumberValue((short)v));
+         }
+         if (t == typeof(ushort)) {
+            return nullCheckWrapper(v => writer.WriteNumberValue((ushort)v));
+         }
+         if (t == typeof(double)) {
+            return nullCheckWrapper(v => writer.WriteNumberValue((double)v));
+         }
+         if (t == typeof(float)) {
+            return nullCheckWrapper(v => writer.WriteNumberValue((float)v));
+         }
+         if (t == typeof(bool)) {
+            return nullCheckWrapper(v => writer.WriteBooleanValue((bool)v));
+         }
+         return nullCheckWrapper(v => writer.WriteStringValue(v?.ToString()));
+
+         Action<object?> nullCheckWrapper(Action<object> writeNonNull) =>
+            v => {
+               if (v is null) {
+                  writer.WriteNullValue();
+               } else {
+                  writeNonNull(v);
+               }
+            };
+      }
+   }
+}

src/DataStreamSuite/DataStreamCore/DataStreamCore.csproj

@@ -2,7 +2,7 @@
 
   <PropertyGroup>
     <AllowUnsafeBlocks>true</AllowUnsafeBlocks>
-    <TargetFramework>net7.0</TargetFramework>
+    <TargetFramework>net8.0</TargetFramework>
     <Nullable>disable</Nullable>
   </PropertyGroup>
 

src/DataStreamSuite/DataStreamExpressions/DataStreamExpressions.csproj

@@ -1,7 +1,7 @@
 <Project Sdk="Microsoft.NET.Sdk">
 
   <PropertyGroup>
-    <TargetFramework>net7.0</TargetFramework>
+    <TargetFramework>net8.0</TargetFramework>
     <ImplicitUsings>enable</ImplicitUsings>
     <Nullable>enable</Nullable>
   </PropertyGroup>

src/DataStreamSuite/DataStreamPublisher/DataStreamPublisher.csproj

@@ -11,7 +11,7 @@
 
   <PropertyGroup>
     <OutputType>Exe</OutputType>
-    <TargetFramework>net7.0</TargetFramework>
+    <TargetFramework>net8.0</TargetFramework>
     <Nullable>enable</Nullable>
     <AllowUnsafeBlocks>true</AllowUnsafeBlocks>
   </PropertyGroup>

src/DataStreamSuite/DataStreamPublisher/Program.cs

@@ -11,7 +11,7 @@ using System.Threading;
 
 bts.Log.Gen.info($"Starting DSPublisher");
 Console.WriteLine("Starting DSPublisher...");
-bool run_echo_tcp_client_only = true;
+bool run_echo_tcp_client_only = false;
 static void init(string[] args) {
    Func<string, string, Task> login = async (l, p) => {
       var login = l;
@@ -31,6 +31,7 @@ else
 
 static ViewStream build_stream(string stream_expression) {
    try {
+      bts.Log.Gen.info($"build_stream stream_expression: {stream_expression}");
       var exp = DataStreamExpressionsInterpreter.Evaluate(stream_expression);
       var rv = exp.getView(new Dictionary<string, StatementExpression>(0));
       return rv;
@@ -58,14 +59,16 @@ void _publish_streams() {
    CloudStreamManager csm = new CloudStreamManager();
    // get it from config
    Dictionary<string, string> streams_to_publish = new Dictionary<string, string>() {
-        {"bbo","bbo()"},
-        {"instruments","instruments()"},
+        //{"bbo","bbo()"},
+        //{"instruments","instruments()"},
         //{"timesales","timesales(10000)"},
-        {"books","books()"},
-      // {"seconds_1_10","second(1,10)"},
+        //{"books","books()"},
+       {"seconds_1_10","second(1,10)"},
         {"literal_420","literal(\"x\", \"420.0\")"},
-      //  {"bbo_ts","timeSeries(bbo())"}
-   };
+        {"mongodb_testcollection", "mongodb(\"mongodb://192.168.1.80:27017\",\"local\",\"test_collection\")"}
+        };
+   //  {"bbo_ts","timeSeries(bbo())"}
+
    foreach (var kv in streams_to_publish) {
       var name = kv.Key;
       var exp = kv.Value;
@@ -77,6 +80,7 @@ void _publish_streams() {
       System.Threading.Thread.Sleep(TimeSpan.FromSeconds(1));
       ViewStreamDatabase.instance.doWork();
       csm.doWork();
+      MasterWorker.getInstance(false).doWork();
    }
 }
 

src/DataStreamSuite/DataStreamQueryAnalyzer/DataStreamQueryAnalyzer.csproj

@@ -2,7 +2,7 @@
 
   <PropertyGroup>
     <OutputType>Exe</OutputType>
-    <TargetFramework>net7.0-windows</TargetFramework>
+    <TargetFramework>net8.0-windows</TargetFramework>
     <Nullable>enable</Nullable>
     <UseWindowsForms>true</UseWindowsForms>
   </PropertyGroup>

src/DataStreamSuite/DataStreamSuite.sln

@@ -38,6 +38,8 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "DataStreamTests", "DataStre
 EndProject
 Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "SharedStreams", "SharedStreams\SharedStreams.csproj", "{C1EBC423-4668-43D4-B7E5-A291086BA823}"
 EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Bts.DataStreamWebApp.Server", "Bts.DataStreamWebApp.Server\Bts.DataStreamWebApp.Server.csproj", "{6DE5E4CE-423E-4571-81F7-539D268D6059}"
+EndProject
 Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Bts.DataStream.Websocket", "Bts.DataStream.Websocket\Bts.DataStream.Websocket.csproj", "{088655DA-3126-4F8A-9AC7-9061FDA891FF}"
 EndProject
 Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Bts.DataStream.WebsocketServer", "Bts.DataStream.WebsocketServer\Bts.DataStream.WebsocketServer.csproj", "{35823BE0-C883-476A-8B30-D6097CBA1A8D}"
@@ -131,6 +133,10 @@ Global
 		{C1EBC423-4668-43D4-B7E5-A291086BA823}.Debug|Any CPU.Build.0 = Debug|Any CPU
 		{C1EBC423-4668-43D4-B7E5-A291086BA823}.Release|Any CPU.ActiveCfg = Release|Any CPU
 		{C1EBC423-4668-43D4-B7E5-A291086BA823}.Release|Any CPU.Build.0 = Release|Any CPU
+		{6DE5E4CE-423E-4571-81F7-539D268D6059}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{6DE5E4CE-423E-4571-81F7-539D268D6059}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{6DE5E4CE-423E-4571-81F7-539D268D6059}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{6DE5E4CE-423E-4571-81F7-539D268D6059}.Release|Any CPU.Build.0 = Release|Any CPU
 		{088655DA-3126-4F8A-9AC7-9061FDA891FF}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
 		{088655DA-3126-4F8A-9AC7-9061FDA891FF}.Debug|Any CPU.Build.0 = Debug|Any CPU
 		{088655DA-3126-4F8A-9AC7-9061FDA891FF}.Release|Any CPU.ActiveCfg = Release|Any CPU

src/DataStreamSuite/DataStreamTests/DataStreamTests.csproj

@@ -1,7 +1,7 @@
 <Project Sdk="Microsoft.NET.Sdk">
 
   <PropertyGroup>
-    <TargetFramework>net7.0</TargetFramework>
+    <TargetFramework>net8.0</TargetFramework>
     <ImplicitUsings>enable</ImplicitUsings>
     <Nullable>enable</Nullable>
 

src/DataStreamSuite/ExpressionCore/ExpressionCore.csproj

@@ -1,7 +1,7 @@
 <Project Sdk="Microsoft.NET.Sdk">
 
   <PropertyGroup>
-    <TargetFramework>net7.0</TargetFramework>
+    <TargetFramework>net8.0</TargetFramework>
     <ImplicitUsings>enable</ImplicitUsings>
     <Nullable>enable</Nullable>
   </PropertyGroup>

src/DataStreamSuite/ExpressionTester/ExpressionTester.csproj

@@ -6,7 +6,7 @@
 
   <PropertyGroup>
     <OutputType>Exe</OutputType>
-    <TargetFramework>net7.0</TargetFramework>
+    <TargetFramework>net8.0</TargetFramework>
     <ImplicitUsings>enable</ImplicitUsings>
     <Nullable>enable</Nullable>
   </PropertyGroup>

src/DataStreamSuite/ExpressionTests/ExpressionTests.csproj

@@ -1,7 +1,7 @@
 <Project Sdk="Microsoft.NET.Sdk">
 
   <PropertyGroup>
-    <TargetFramework>net7.0</TargetFramework>
+    <TargetFramework>net8.0</TargetFramework>
     <ImplicitUsings>enable</ImplicitUsings>
     <Nullable>enable</Nullable>
 

src/DataStreamSuite/GuiUtilities/GuiUtilities.csproj

@@ -1,7 +1,7 @@
 <Project Sdk="Microsoft.NET.Sdk">
 
   <PropertyGroup>
-    <TargetFramework>net7.0-windows</TargetFramework>
+    <TargetFramework>net8.0-windows</TargetFramework>
     <ImplicitUsings>enable</ImplicitUsings>
     <Nullable>enable</Nullable>
     <UseWindowsForms>true</UseWindowsForms>

src/DataStreamSuite/PubSub/Client/Communication/IPubSubServerConnection.cs

@@ -47,30 +47,30 @@ namespace BCG {
       /// of Int32.Max would return all the messages it has
       /// received between last getUpdates and this call.
       /// </summary
-      IList<byte[]> getUpdates(TimeSpan timeout, int maxElements);
+      IList<byte[]> get_updates(TimeSpan timeout, int maxElements);
       /// <summary>
       /// server URL to which the connection is made
       /// </summary>
-      string serverURL { get; }
+      string server_url { get; }
       /// <summary>
       /// client URL as seen by the server. if a connection has not
       /// been established (as can happen in TCP) an empty string 
       /// is returned
       /// </summary>
-      string clientURL { get; }
+      string client_url { get; }
 
       /// <summary>
       /// registers the callback to receive 
       /// connection notifications
       /// </summary>
-      void registerCallback(ConnectionStatusChange del);
+      void register_connection_callback(PubSubServerConnectionStatusChange del);
 
       /// <summary>
       /// registers callbacks to reveive data
       /// </summary>
       /// <param name="del"></param>
-      void registerDataCallback(DataDelegate del);
+      void register_data_callback(PubSubServerDataDelegate del);
 
-      void registerSendConfirmationCallback(SendConfirmationDelegate del);
+      void register_send_confirmation_callback(SendConfirmationDelegate del);
    }
 }

src/DataStreamSuite/PubSub/Client/Communication/ManagedTCP/PubSubServerManagedTCPConnection.cs

@@ -8,6 +8,7 @@ using System.IO;
 //using BCG.Utilities.General;
 using System.Threading;
 using mbts;
+using bts;
 
 namespace BCG {
    /// <summary>
@@ -20,26 +21,27 @@ namespace BCG {
    /// ITCPConnectionHandler.onConnection will be called on any arbitrary thread
    /// 
    /// </summary>
-   public class PubSubServerManagedTCPConnection : IPubSubServerConnection,
-                                                   ITCPMessageHandler,
-                                                   ITCPConnectionHandler {
+   public class PubSubServerManagedTCPConnection : IPubSubServerConnection,// external facing interface to communicate msgs to Pubsubserver   
+                                                   ITCPMessageHandler,     // to get messages from ITCPCOnnection
+                                                   ITCPConnectionHandler { // to get connections from AsyncTCPConnector_APM
       private ITCPConnection _connection, _new_connection;
       private readonly BlutoSubAddress _server_address;
       // if a message(with confirm) can not be sent , its id is queued here, 
       // on reconnection, _on_send_onfirm_callback is fired with a negative on these ids, so the user
       //can re-send it. The re-send mechanism is not in implemented in this class, but by the user of this class
       private ThreadSafeQueue<string> _negative_unsent_confs = new ThreadSafeQueue<string>();
-      // in onData, data is queued here
+      // in onData, data is queued here, for polling
       private SyncQueue<byte[]> _in_data_queue = new SyncQueue<byte[]>();
       private DateTime _last_hb = DateTime.Now;
-      private ConnectionStatusChange _conn_callback;
-      private DataDelegate _data_callback;
-      private SendConfirmationDelegate _on_send_onfirm_callback;
+      private PubSubServerConnectionStatusChange _conn_callback;
+      private PubSubServerDataDelegate _data_callback;
+      private SendConfirmationDelegate _on_send_confirm_callback;
       private readonly TimeSpan _connection_drop_threshold =
-         new TimeSpan(0, 0, 0, 0, PubSub3Config.instance.PS3_HeartBeatInterval * PubSub3Config.instance.PS3_MissingHeartBeatsForGC);
+         new TimeSpan(0, 0, 0, 0,
+                     PubSub3Config.instance.PS3_HeartBeatInterval * PubSub3Config.instance.PS3_MissingHeartBeatsForGC);
 
-      public PubSubServerManagedTCPConnection(string serverURL)
-         : this(new BlutoSubAddress(serverURL)) { }
+      public PubSubServerManagedTCPConnection(string server_url)
+         : this(new BlutoSubAddress(server_url)) { }
 
       public PubSubServerManagedTCPConnection(BlutoSubAddress server_address) {
          _server_address = server_address;
@@ -48,17 +50,17 @@ namespace BCG {
 
       private void _connect_to_ps_server() {
          bts.Log.Gen.info($"PubSubServerManagedTCPConnection._connect_to_ps_server server: {_server_address}");
-         ClientToServerManagedTCPConnector.connect_to(_server_address, this, TimeSpan.FromSeconds(1));
+         AsyncTCPConnector_APM.connect_to(_server_address, this);
       }
 
       #region IPubSubServerConnection Members
 
-      public void sendWithConfirmation(byte[] data, ref string conf_id) {
-         bts.Log.Gen.info($"PubSubServerManagedTCPConnection.sendWithConfirmation server: {_server_address} data: {data.Length} myConnection : {_connection != null}");
+      public void send_with_confirmation(byte[] data, ref string conf_id) {
+         bts.Log.Gen.info($"PubSubServerManagedTCPConnection.sendWithConfirmation server: {_server_address} data: {data.Length} _connection : {_connection != null}");
          if (_connection != null) {
             _connection.sendWithConfirmation(data, ref conf_id, (msg_id, confirmed) => {
-               if (_on_send_onfirm_callback != null)
-                  _on_send_onfirm_callback(msg_id, confirmed);
+               if (_on_send_confirm_callback != null)
+                  _on_send_confirm_callback(msg_id, confirmed);
             });
          } else {
             bts.Log.Gen.warn($"PubSubServerManagedTCPConnection.sendWithConfirmation server: {_server_address} no connection, buffering send");
@@ -86,18 +88,18 @@ namespace BCG {
 
       public void send_with_confirmation(PubSubMessage msg, ref string conf_id) {
          var bytes = msg.to_bytes();
-         sendWithConfirmation(bytes, ref conf_id);
+         send_with_confirmation(bytes, ref conf_id);
       }
 
-      public IList<byte[]> getUpdates(TimeSpan timeout, int maxElements) {
+      public IList<byte[]> get_updates(TimeSpan timeout, int maxElements) {
          return _in_data_queue.dequeueMultipleElements(Convert.ToInt32(timeout.TotalMilliseconds), maxElements);
       }
 
-      public string serverURL {
+      public string server_url {
          get { return _server_address.ToString(); }
       }
 
-      public string clientURL {
+      public string client_url {
          get {
             string rv = "";
             lock (this) {
@@ -118,16 +120,16 @@ namespace BCG {
          }
       }
 
-      public void registerCallback(ConnectionStatusChange del) {
+      public void register_connection_callback(PubSubServerConnectionStatusChange del) {
          _conn_callback = del;
       }
 
-      public void registerDataCallback(DataDelegate del) {
+      public void register_data_callback(PubSubServerDataDelegate del) {
          _data_callback = del;
       }
 
-      public void registerSendConfirmationCallback(SendConfirmationDelegate del) {
-         _on_send_onfirm_callback = del;
+      public void register_send_confirmation_callback(SendConfirmationDelegate del) {
+         _on_send_confirm_callback = del;
       }
 
       #endregion
@@ -185,18 +187,6 @@ namespace BCG {
                break;
 
          }
-         // this stupid shit, caused duplicating og messages and 2 days of wild goose chase
-         // if (message.messageType == RemoteMessageType.Data)
-         // {
-         //    if (_data_callback != null)
-         //       _data_callback(message.remoteConnectionInfo.ToString(), message.message);
-         //    else
-         //       _in_data_queue.enqueue(message.message);
-         // }
-         // else
-         // {
-         //    _last_hb = DateTime.Now;
-         // }
       }
 
       #endregion
@@ -204,8 +194,7 @@ namespace BCG {
       #region ITCPConnectionHandler Members
 
       /// <summary>
-      /// This function is called on the main thread (as all other functions
-      /// except onData)
+      ///  will be called on ana rbitrary .Net thread
       /// </summary>
       /// <param name="connection"></param>
       public void onConnection(ITCPConnection connection) {
@@ -246,10 +235,10 @@ namespace BCG {
             _connection.register(this);
             if (_conn_callback != null)
                _conn_callback(_server_address.ToString(), true);
-            if (_on_send_onfirm_callback != null) {
+            if (_on_send_confirm_callback != null) {
                var msg_ids = _negative_unsent_confs.dequeueAll();
                foreach (var id in msg_ids)
-                  _on_send_onfirm_callback(id, false);
+                  _on_send_confirm_callback(id, false);
             }
          }
       }
@@ -264,47 +253,6 @@ namespace BCG {
       public void doWork() {
          //_dispose_and_reconnect_if_missing_hb();
          _check_for_new_connection();
-         ///check for heartbeats and let clients know in 
-         ///case of missing hb (connection breaking)
-         // TimeSpan hb_interval = DateTime.Now - myLastHB;
-         // bool hb_missed = hb_interval > myConnectionDropThreshold;
-         // bool conn_disposed = myConnection != null || myConnection.isDisposed;
-         // if (conn_disposed || hb_missed)
-         // {
-         //    if (hb_missed)
-         //    {
-         //       // since conn breakage was diagnosed logically, lets physically close the connection
-         //       myConnection.Dispose();
-         //       string dispose_msg = $"PubSubServerManagedTCPConnection.doWork server: {myServerAddress} Disposing connection conn_disposed: {conn_disposed} hb_missed: {hb_missed}";
-         //       Console.WriteLine(dispose_msg);
-         //    }
-         //    myConnection = null; // this is used as a marker for a dead connection
-         //    if (myConnCallback != null)
-         //       myConnCallback(myServerAddress.ToString(), false);
-         //    string reconn_msg = $"PubSubServerManagedTCPConnection.doWork disconnecting client from server: {myServerAddress} conn_disposed: {conn_disposed} hb_missed: {hb_missed} " +
-         //    $"because of missing hbs last_hb: {myLastHB}, now: {DateTime.Now} curr_hb_interval: {hb_interval.TotalSeconds} threshold: {myConnectionDropThreshold.TotalSeconds} conn_disposed: {conn_disposed}";
-         //    ClientToServerManagedTCPConnector.connectTo(myServerAddress, this);
-         // }
-         // if (myTempNewConnection != null)
-         // {
-         //    if (myConnection != null)
-         //    {
-         //       // TODOds
-         //       // log error and close it
-         //       // BlueEventLog.logAssert(myConnection == null, "Connection has to be nulled out and disposed "
-         //       //                                                 + " before a new connection is established");
-         //    }
-         //    Console.WriteLine("PubSubServerManagedTCPConnection hooking new connection");
-         //    myConnection = myTempNewConnection;
-         //    myTempNewConnection = null;
-         //    myConnection.register(this);
-         //    myConfs.dequeueAll();
-         //    if (myConnCallback != null)
-         //       myConnCallback(myServerAddress.ToString(), true);
-         // }
-         // foreach (Pair<string, bool> p in myConfs.dequeueAll())
-         //    if (myOnSendConfirmCallback != null)
-         //       myOnSendConfirmCallback(p.first, p.second);
       }
 
       #endregion

src/DataStreamSuite/PubSub/Client/MetaData/MetaData.cs

@@ -16,6 +16,7 @@ namespace BCG {
    // SubMetaData
    // TransactionMetaData
    // UnSubMetaData
+   // 
    public class MetaData : ICloneable {
       private IDictionary _name_value_map = new Dictionary<string, string>();
       int mySize = -1;

src/DataStreamSuite/PubSub/Client/MetaData/SubMetaData.cs

@@ -1,10 +1,8 @@
 using System;
 
-namespace BCG
-{
-   [Serializable]
-   public class SubMetaData 
-   {
+namespace BCG {
+   //[Serializable]
+   public class SubMetaData {
       /// <summary>
       /// connection URL on which the connection will receive data
       /// </summary>
@@ -13,20 +11,18 @@ namespace BCG
       /// identification of the client and the connection on which to receive data
       /// machine; ip; port; clientURL
       /// </summary>
-      public string subscribingApp;
+      public readonly string subscribing_app = ClientInfo.thisClient.processString;
       /// <summary>
       /// tags that are to be subscribed
       /// </summary>
       public string[] tags;
 
-      public SubMetaData()
-      { }
+      public SubMetaData() { }
 
-      public SubMetaData(string theConnection, params string[] theTags)
-      {
-         connection = theConnection;
-         tags = theTags;
-         subscribingApp = ClientInfo.thisClient.processString;
+      public SubMetaData(string the_connection, params string[] the_tags) {
+         connection = the_connection;
+         tags = the_tags;
+         //subscribing_app = ClientInfo.thisClient.processString;
       }
    }
 }

src/DataStreamSuite/PubSub/Client/MetaData/UnSubMetaData.cs

@@ -1,21 +1,17 @@
 using System;
 
-namespace BCG
-{
-   [Serializable]
-   public class UnSubMetaData
-   {
+namespace BCG {
+   //[Serializable]
+   public class UnSubMetaData {
       public string connection;
-      public string[] tags;
-      public string unsubscribingApp = ClientInfo.thisClient.processString;
+      public readonly string[] tags;
+      public readonly string unsubscribing_app = ClientInfo.thisClient.processString;
 
-      public UnSubMetaData()
-      {}
+      public UnSubMetaData() { }
 
-      public UnSubMetaData(string theConnection, params string[] theTags)
-      {
-         connection = theConnection;
-         tags = theTags;
+      public UnSubMetaData(string the_connection, params string[] the_tags) {
+         connection = the_connection;
+         tags = the_tags;
       }
    }
 }

src/DataStreamSuite/PubSub/Client/PublishingUtilities/IPublisher.cs

@@ -6,12 +6,18 @@ namespace BCG
 	public interface IPublisher
 	{
       void publish( IPubItem pi );
+#pragma warning disable SYSLIB0011 // Type or member is obsolete
       void publish( IPubItem pi, IFormatter f );
+#pragma warning restore SYSLIB0011 // Type or member is obsolete
       void publish( IPubItem[] pi );
+#pragma warning disable SYSLIB0011 // Type or member is obsolete
       void publish( IPubItem[] pi, IFormatter f );
+#pragma warning restore SYSLIB0011 // Type or member is obsolete
    }
 
+#pragma warning disable SYSLIB0011 // Type or member is obsolete
    public interface IPublisher<T> where T : IFormatter
+#pragma warning restore SYSLIB0011 // Type or member is obsolete
    {
       void publish (IPubItem pi);
       void publish (IPubItem[] pi);

src/DataStreamSuite/PubSub/Client/PublishingUtilities/Publisher.cs

@@ -8,7 +8,9 @@ namespace BCG
 	public class Publisher : IPublisher
 	{
       private PublisherCore myPubCore = new PublisherCore ();
+#pragma warning disable SYSLIB0011 // Type or member is obsolete
       private BinaryFormatter myBinFormatter = new BinaryFormatter ();
+#pragma warning restore SYSLIB0011 // Type or member is obsolete
       private const string myTagType = "DefaultTag";
 
       public Publisher()   {}
@@ -21,7 +23,9 @@ namespace BCG
       public void publish (IPubItem[] pis)
       { publish( pis, myBinFormatter ); }
 
+#pragma warning disable SYSLIB0011 // Type or member is obsolete
       public void publish( IPubItem pi, IFormatter f )
+#pragma warning restore SYSLIB0011 // Type or member is obsolete
       {
          MemoryStream ms = new MemoryStream();
          #pragma warning disable 
@@ -31,7 +35,9 @@ namespace BCG
          myPubCore.publish( myTagType, pi.getTag(), ms );
       }
 
+#pragma warning disable SYSLIB0011 // Type or member is obsolete
       public void publish( IPubItem[] pis, IFormatter f )
+#pragma warning restore SYSLIB0011 // Type or member is obsolete
       {
          myPubCore.beginTransaction();
          foreach ( IPubItem pi in pis )
@@ -42,10 +48,14 @@ namespace BCG
       #endregion
    }
 
+#pragma warning disable SYSLIB0011 // Type or member is obsolete
    public class Publisher<T> : IPublisher<T> where T : IFormatter, new ()
+#pragma warning restore SYSLIB0012 // Type or member is obsolete
    {
       private PublisherCore myPubCore = new PublisherCore ();
+#pragma warning disable SYSLIB0011 // Type or member is obsolete
       private IFormatter myFormatter = new T ();
+#pragma warning restore SYSLIB0011 // Type or member is obsolete
       private const string myTagType = "DefaultTag";
 
       public Publisher () { }
@@ -55,9 +65,9 @@ namespace BCG
       public void publish (IPubItem pi)
       {
          MemoryStream ms = new MemoryStream ();
-         #pragma warning disable 
+#pragma warning disable 
          myFormatter.Serialize (ms, pi);
-         #pragma warning restore SYSLIB0011
+#pragma warning restore SYSLIB0011
          ms.Position = 0;
          myPubCore.publish (myTagType, pi.getTag (), ms);
       }

src/DataStreamSuite/PubSub/ManagedTCP/AsyncTCPConnection_APM.cs

@@ -328,7 +328,8 @@ namespace bts {
             return;
          }
          _send_key_map.Add(message.key);
-         bts.Log.Gen.info($"AsyncTCPConnection_APM._on_send: pre endSend this: {ToString()} tid: {_tid()} res: {_cb_txt(res)} message: {res.AsyncState} _is_alive: {_is_alive}");
+         bts.Log.Gen.info($"AsyncTCPConnection_APM._on_send: pre endSend this: {ToString()} tid: {_tid()} " +
+                                 $"res: {_cb_txt(res)} message: {res.AsyncState} _is_alive: {_is_alive}");
          if (_is_alive) {
             try {
                SocketError send_err;

src/DataStreamSuite/PubSub/ManagedTCP/ClientToServerManagedTCPConnector.cs

@@ -1,82 +1,82 @@
-using System;
-using System.Collections.Generic;
-using System.Text;
-using System.Net;
-using System.Net.Sockets;
-using System.Threading;
-using System.Threading.Tasks;
-//using BCG.Utilities;
-//using BCG.Utilities.General;
+// using System;
+// using System.Collections.Generic;
+// using System.Text;
+// using System.Net;
+// using System.Net.Sockets;
+// using System.Threading;
+// using System.Threading.Tasks;
+// //using BCG.Utilities;
+// //using BCG.Utilities.General;
 
-namespace BCG {
-   internal class SocketConnectionArgs {
-      public EndPoint ep;
-      public Socket sock;
-      public ITCPConnectionHandler handler;
-      public string serverURL;
+// namespace BCG {
+//    internal class SocketConnectionArgs {
+//       public EndPoint ep;
+//       public Socket sock;
+//       public ITCPConnectionHandler handler;
+//       public string serverURL;
 
-      internal SocketConnectionArgs(EndPoint theEp, Socket theSock, ITCPConnectionHandler theHandler,
-         string theServerURL) {
-         ep = theEp;
-         sock = theSock;
-         handler = theHandler;
-         serverURL = theServerURL;
-      }
-   }
+//       internal SocketConnectionArgs(EndPoint theEp, Socket theSock, ITCPConnectionHandler theHandler,
+//          string theServerURL) {
+//          ep = theEp;
+//          sock = theSock;
+//          handler = theHandler;
+//          serverURL = theServerURL;
+//       }
+//    }
 
-   public class ClientToServerManagedTCPConnector {
-      private static Random myRnd = new Random(DateTime.Now.Millisecond);
-      private static TimeSpan myHBTimeSpan = new TimeSpan(0, 0, 0, 0, PubSub3Config.instance.PS3_HeartBeatInterval);
-      private static int myWaitTimeBeforeRetryingUpperLimit = 5000,
-                           myWaitTimeBeforeRetryingLowerLimit = 1000;
+//    public class ClientToServerManagedTCPConnector {
+//       private static Random myRnd = new Random(DateTime.Now.Millisecond);
+//       private static TimeSpan myHBTimeSpan = new TimeSpan(0, 0, 0, 0, PubSub3Config.instance.PS3_HeartBeatInterval);
+//       private static int myWaitTimeBeforeRetryingUpperLimit = 5000,
+//                            myWaitTimeBeforeRetryingLowerLimit = 1000;
 
-      public static void connectTo(BlutoSubAddress serverAddress, ITCPConnectionHandler handler) {
-         Console.WriteLine($"ClientToServerManagedTCPConnector.connectTo server: {serverAddress}");
-         Socket sock = new Socket(AddressFamily.InterNetwork, SocketType.Stream, ProtocolType.Tcp);
-         EndPoint ep = new IPEndPoint(IPAddress.Parse(serverAddress.ip), serverAddress.port);
-         sock.BeginConnect(ep, new AsyncCallback(myOnConnect), new SocketConnectionArgs(ep, sock, handler, serverAddress.ToString()));
-      }
+//       public static void connectTo(BlutoSubAddress serverAddress, ITCPConnectionHandler handler) {
+//          Console.WriteLine($"ClientToServerManagedTCPConnector.connectTo server: {serverAddress}");
+//          Socket sock = new Socket(AddressFamily.InterNetwork, SocketType.Stream, ProtocolType.Tcp);
+//          EndPoint ep = new IPEndPoint(IPAddress.Parse(serverAddress.ip), serverAddress.port);
+//          sock.BeginConnect(ep, new AsyncCallback(myOnConnect), new SocketConnectionArgs(ep, sock, handler, serverAddress.ToString()));
+//       }
 
-      public static async Task connect_to_async(BlutoSubAddress serverAddress, ITCPConnectionHandler handler, TimeSpan wait_before_retrying, int max_n_attempts = 1024) {
-         bts.Log.Gen.info($"ClientToServerManagedTCPConnector.connectTo server: {serverAddress}");
-         await TCPSocketUtils.connect_to_server_async(serverAddress.ip, serverAddress.port, handler, wait_before_retrying, max_n_attempts);
-      }
+//       public static async Task connect_to_async(BlutoSubAddress serverAddress, ITCPConnectionHandler handler, TimeSpan wait_before_retrying, int max_n_attempts = 1024) {
+//          bts.Log.Gen.info($"ClientToServerManagedTCPConnector.connectTo server: {serverAddress}");
+//          await TCPSocketUtils.connect_to_server_async(serverAddress.ip, serverAddress.port, handler, wait_before_retrying, max_n_attempts);
+//       }
 
-      public static void connect_to(BlutoSubAddress serverAddress, ITCPConnectionHandler handler, TimeSpan wait_before_retrying, int max_n_attempts = 1024) {
-         bts.Log.Gen.info($"ClientToServerManagedTCPConnector.connectTo server: {serverAddress}");
-         Task.Run(async () => {
-            await TCPSocketUtils.connect_to_server_async(serverAddress.ip, serverAddress.port, handler, wait_before_retrying, max_n_attempts);
-         });
-      }
+//       public static void connect_to(BlutoSubAddress serverAddress, ITCPConnectionHandler handler, TimeSpan wait_before_retrying, int max_n_attempts = 1024) {
+//          bts.Log.Gen.info($"ClientToServerManagedTCPConnector.connectTo server: {serverAddress}");
+//          Task.Run(async () => {
+//             await TCPSocketUtils.connect_to_server_async(serverAddress.ip, serverAddress.port, handler, wait_before_retrying, max_n_attempts);
+//          });
+//       }
 
-      private static void myOnConnect(IAsyncResult res) {
-         Console.WriteLine("ClientToServerManagedTCPConnector.myOnConnect " + res.ToString() + " " + res.AsyncState.ToString());
-         SocketConnectionArgs args = res.AsyncState as SocketConnectionArgs;
-         try {
-            args.sock.EndConnect(res);
-         } catch (SocketException sex) {
-            string errMsg = String.Format("{0}: Connection to {1} failed.", DateTime.Now.ToString(), args.serverURL.ToString());
-            //BlueEventLog.logEvent(sex.Message + ";" + errMsg, PS3LogLevels.disconnection);
-            //Console.WriteLine(errMsg);
-            if (args.handler.isDisposed) {
-               string doneMsg = String.Format("{0}: Connection to {1} was disposed. Not Retrying.", DateTime.Now.ToString(), args.serverURL.ToString());
-               //Console.WriteLine(doneMsg);
-               //               BlueEventLog.logEvent(doneMsg, PS3LogLevels.disconnection);
-               return;
-            } else {
-               string retryMsg = String.Format("{0}: Retrying Connection to {1}.", DateTime.Now.ToString(), args.serverURL.ToString());
-               //Console.WriteLine(retryMsg);
-               //BlueEventLog.logEvent(retryMsg, PS3LogLevels.disconnection);
+//       private static void myOnConnect(IAsyncResult res) {
+//          Console.WriteLine("ClientToServerManagedTCPConnector.myOnConnect " + res.ToString() + " " + res.AsyncState.ToString());
+//          SocketConnectionArgs args = res.AsyncState as SocketConnectionArgs;
+//          try {
+//             args.sock.EndConnect(res);
+//          } catch (SocketException sex) {
+//             string errMsg = String.Format("{0}: Connection to {1} failed.", DateTime.Now.ToString(), args.serverURL.ToString());
+//             //BlueEventLog.logEvent(sex.Message + ";" + errMsg, PS3LogLevels.disconnection);
+//             //Console.WriteLine(errMsg);
+//             if (args.handler.isDisposed) {
+//                string doneMsg = String.Format("{0}: Connection to {1} was disposed. Not Retrying.", DateTime.Now.ToString(), args.serverURL.ToString());
+//                //Console.WriteLine(doneMsg);
+//                //               BlueEventLog.logEvent(doneMsg, PS3LogLevels.disconnection);
+//                return;
+//             } else {
+//                string retryMsg = String.Format("{0}: Retrying Connection to {1}.", DateTime.Now.ToString(), args.serverURL.ToString());
+//                //Console.WriteLine(retryMsg);
+//                //BlueEventLog.logEvent(retryMsg, PS3LogLevels.disconnection);
 
-               int waitTime = myRnd.Next(myWaitTimeBeforeRetryingLowerLimit, myWaitTimeBeforeRetryingUpperLimit);
-               Thread.Sleep(waitTime);
-               args.sock.BeginConnect(args.ep, new AsyncCallback(myOnConnect), args);
-               return;
-            }
-         }
-         //BlueEventLog.logEvent(String.Format("Successful connection made to {0}.", args.serverURL.ToString()), PS3LogLevels.connection);
-         args.handler.onConnection(new bts.AsyncTCPConnection_APM(args.sock, myHBTimeSpan, ITCPConnection.ConnectionSide.Client));
-      }
+//                int waitTime = myRnd.Next(myWaitTimeBeforeRetryingLowerLimit, myWaitTimeBeforeRetryingUpperLimit);
+//                Thread.Sleep(waitTime);
+//                args.sock.BeginConnect(args.ep, new AsyncCallback(myOnConnect), args);
+//                return;
+//             }
+//          }
+//          //BlueEventLog.logEvent(String.Format("Successful connection made to {0}.", args.serverURL.ToString()), PS3LogLevels.connection);
+//          args.handler.onConnection(new bts.AsyncTCPConnection_APM(args.sock, myHBTimeSpan, ITCPConnection.ConnectionSide.Client));
+//       }
 
-   }
-}
+//    }
+// }

src/DataStreamSuite/PubSub/Misc/GenericFormatter.cs

@@ -5,23 +5,21 @@ using System.Runtime.Serialization.Formatters.Binary;
 
 namespace BCG
 {
+#pragma warning disable SYSLIB0011 // Type or member is obsolete
    public class GenericFormatter<F> : IGenericFormatter where F : IFormatter, new ()
    {
       IFormatter myFormatter = new F ();
 
       public T Deserialize<T> (Stream serializationStream)
       {
-         #pragma warning disable 
          return (T) myFormatter.Deserialize (serializationStream);
-         #pragma warning restore SYSLIB0011
       }
       public void Serialize<T> (Stream serializationStream, T graph)
       {
-         #pragma warning disable 
          myFormatter.Serialize (serializationStream, graph);
-         #pragma warning restore SYSLIB0011
       }
    }
 
    public class GenericBinaryFormatter : GenericFormatter<BinaryFormatter> { }
+#pragma warning restore SYSLIB0011 // Type or member is obsolete
 }

src/DataStreamSuite/PubSub/PubSub.csproj

@@ -6,7 +6,7 @@
 
   <PropertyGroup>
       <AllowUnsafeBlocks>true</AllowUnsafeBlocks>
-    <TargetFramework>net7.0</TargetFramework>
+    <TargetFramework>net8.0</TargetFramework>
     <Nullable>enable</Nullable>
   </PropertyGroup>
 

src/DataStreamSuite/PubSub/PubSubUpdate.cs

@@ -4,19 +4,22 @@ using System.IO;
 
 namespace BCG {
    /// <summary>
-   /// this is the data returned from the
-   /// pubsub server
+   /// this is the data returned from the pubsub server
    /// </summary>
    public class PubSubUpdate {
 
-      public PubSubUpdate(string tag, string tagType, Stream body) { this.tag = tag; this.body = body; this.tagType = tagType; }
+      public PubSubUpdate(string tag, string the_tag_type, Stream body) {
+         this.tag = tag;
+         this.body = body;
+         this.tag_type = the_tag_type;
+      }
 
       public string tag;
       public Stream body;
-      public string tagType;
+      public string tag_type;
 
       public override string ToString() {
-         return $"PubSubUpdate tag: {tag} tag_type: {tagType} len: {body.Position}";
+         return $"PubSubUpdate tag: {tag} tag_type: {tag_type} len: {body.Position}";
       }
    };
 }

src/DataStreamSuite/PubSub/Server/Communication/IPubsubMessageHubBase.cs

@@ -5,18 +5,16 @@ using System.IO;
 using System.Collections.Generic;
 using mbts;
 
-namespace BCG
-{
-   public delegate void ConnectionStatusChange(string peerURL, bool connection);
-   public delegate void DataDelegate(string sender, byte[] data);
+namespace BCG {
+   public delegate void PubSubServerConnectionStatusChange(string peerURL, bool connection);
+   public delegate void PubSubServerDataDelegate(string sender, byte[] data);
    /// <summary>
    /// Encapsulates PubsubServer's communication with the
    /// rest of the world. Has send and get methods, and also
    /// an event to tell if a client connects or disconnects.
    /// this event is prompted by the doWork.
    /// </summary>
-   public interface IPubsubMessageHub : IWorker
-   {
+   public interface IPubsubMessageHub : IWorker {
 
       /// <summary>
       /// The port that the message hub is listening on.  Return -1 if you're not listening on a port, i.e. for hte case of multicast
@@ -29,14 +27,14 @@ namespace BCG
       /// getUpdates. The delegate will be fired on an arbitrary thread.
       /// </summary>
       /// <param name="?"></param>
-      void registerDataCallback(DataDelegate del);
+      void registerDataCallback(PubSubServerDataDelegate del);
       /// <summary>
       /// registers a connection callback. this callback
       /// will be fired within the doWork call.
       /// </summary>
       /// <param name="del"></param>
       /// <param name="register"></param>
-      void registerConnectionCallback(ConnectionStatusChange del);
+      void registerConnectionCallback(PubSubServerConnectionStatusChange del);
       /// <summary>
       /// this is where this interface starts communication with the
       /// rest of the world. Obvioulsy, the resgiters should be

src/DataStreamSuite/PubSub/Server/Communication/ManagedTCP/ManagedTCPMessageHub.cs

@@ -22,8 +22,8 @@ namespace BCG {
 
       }
       private bool myActivated = false;
-      private ConnectionStatusChange myConnCb;
-      private DataDelegate myDataCb;
+      private PubSubServerConnectionStatusChange myConnCb;
+      private PubSubServerDataDelegate myDataCb;
 
       /// <summary>
       /// client to connection
@@ -77,12 +77,12 @@ namespace BCG {
 
       #region IPubsubMessageHub Members
 
-      public void registerDataCallback(DataDelegate del) {
+      public void registerDataCallback(PubSubServerDataDelegate del) {
          // BlueEventLog.logAssert(!myActivated, "connection callbacks should be registered before the call to activate");
          myDataCb = del;
       }
 
-      public void registerConnectionCallback(ConnectionStatusChange del) {
+      public void registerConnectionCallback(PubSubServerConnectionStatusChange del) {
          //BlueEventLog.logAssert(!myActivated, "data callbacks should be registered before the call to activate");
          myConnCb = del;
       }

src/DataStreamSuite/PubSub/Server/Communication/ServerMessageHub.cs

@@ -4,25 +4,22 @@ using System.Threading;
 using System.IO;
 using mbts;
 
-namespace BCG
-{
+namespace BCG {
    /// <summary>
    /// Wraps around all the IPubsubMessageHub's that pubsub supports.
    /// ie MSMQ and TCP to begin with.
    /// </summary>
-   public class ServerMessageHub : IPubsubMessageHub
-   {
+   public class ServerMessageHub : IPubsubMessageHub {
       private static ServerMessageHub mySelf = null;
       private static object myInstanceLock = new object();
       private Dictionary<string, IPubsubMessageHub> myHubs = new Dictionary<string, IPubsubMessageHub>();
       private SyncQueue<Pair<string, byte[]>> myReceivingQueue = new SyncQueue<Pair<string, byte[]>>();
-      private ConnectionStatusChange myConnectionCb;
-      private DataDelegate myDataCb;
+      private PubSubServerConnectionStatusChange myConnectionCb;
+      private PubSubServerDataDelegate myDataCb;
       private MasterWorker myMasterWorker = MasterWorker.getInstance(false);
 
       protected ServerMessageHub() : this(PubSub3Config.instance.PS3_ServerPort) { }
-      protected ServerMessageHub(int portNumber)
-      {
+      protected ServerMessageHub(int portNumber) {
          // TODO: no more MSMQ
          // if (!PubSub3Config.instance.PS3_DisableMSMQ)
          //    myHubs.Add(MSMQSubAddress.prefix, new MSMQMessageHub());
@@ -35,19 +32,14 @@ namespace BCG
                                                       as IPubsubMessageHub);
       }
 
-      public int port
-      {
+      public int port {
          get { return myHubs[BlutoSubAddress.prefix].port; }
 
       }
-      public static ServerMessageHub instance
-      {
-         get
-         {
-            if (mySelf == null)
-            {
-               lock (myInstanceLock)
-               {
+      public static ServerMessageHub instance {
+         get {
+            if (mySelf == null) {
+               lock (myInstanceLock) {
                   if (mySelf == null)
                      mySelf = new ServerMessageHub();
                }
@@ -60,47 +52,39 @@ namespace BCG
 
       /* Builds the server message hub with a specified port number. Only works if the server message hub has not already been created. 
        * */
-      public static ServerMessageHub getInstance(int portNumber)
-      {
-         if (mySelf == null)
-         {
-            lock (myInstanceLock)
-            {
+      public static ServerMessageHub getInstance(int portNumber) {
+         if (mySelf == null) {
+            lock (myInstanceLock) {
                if (mySelf == null)
                   mySelf = new ServerMessageHub(portNumber);
             }
 
-         }
-         else
+         } else
             throw new ArgumentException("Error: Cannot specify a port number for the ServerMessageHub instance if it has already been created.");
          return mySelf;
       }
       #region IPubsubMessageHub Members
 
-      public void registerDataCallback(DataDelegate del) { myDataCb = del; }
+      public void registerDataCallback(PubSubServerDataDelegate del) { myDataCb = del; }
 
-      public void registerConnectionCallback(ConnectionStatusChange del) { myConnectionCb = del; }
+      public void registerConnectionCallback(PubSubServerConnectionStatusChange del) { myConnectionCb = del; }
 
-      public void activate()
-      {
+      public void activate() {
          string msg = "PubSubServer started .. initializing TCP transport components .. ";
          Console.WriteLine(msg);
-         foreach (IPubsubMessageHub hub in myHubs.Values)
-         {
-            hub.registerDataCallback(new DataDelegate(myOnData));
-            hub.registerConnectionCallback(new ConnectionStatusChange(myOnConnection));
+         foreach (IPubsubMessageHub hub in myHubs.Values) {
+            hub.registerDataCallback(new PubSubServerDataDelegate(myOnData));
+            hub.registerConnectionCallback(new PubSubServerConnectionStatusChange(myOnConnection));
             hub.activate();
          }
          myMasterWorker.addWorker(this, new TimeSpan(0, 0, 0, 0, 250));
       }
 
-      public List<Pair<string, byte[]>> getUpdates(TimeSpan timeout, int maxElements)
-      {
+      public List<Pair<string, byte[]>> getUpdates(TimeSpan timeout, int maxElements) {
          return myReceivingQueue.dequeueMultipleElements(timeout, maxElements);
       }
 
-      public bool send(string receiver, byte[] data)
-      {
+      public bool send(string receiver, byte[] data) {
          if (receiver.StartsWith(BlutoSubAddress.prefix))
             return myHubs[BlutoSubAddress.prefix].send(receiver, data);
          else if (receiver.StartsWith(MSMQSubAddress.prefix))
@@ -109,8 +93,7 @@ namespace BCG
             return false;
       }
 
-      public bool send(string receiver, Stream str)
-      {
+      public bool send(string receiver, Stream str) {
          if (receiver.StartsWith(BlutoSubAddress.prefix))
             return myHubs[BlutoSubAddress.prefix].send(receiver, str);
          else if (receiver.StartsWith(MSMQSubAddress.prefix))
@@ -126,8 +109,7 @@ namespace BCG
       /// <summary>
       /// connection callback will be called within this doWork
       /// </summary>
-      public void doWork()
-      {
+      public void doWork() {
          foreach (IPubsubMessageHub hub in myHubs.Values)
             hub.doWork();
       }
@@ -138,21 +120,18 @@ namespace BCG
       public static string internalPublisherConnection = "InternalPublisher";
 
 
-      public void internalSend(byte[] data)
-      {
+      public void internalSend(byte[] data) {
          myReceivingQueue.enqueue(new Pair<string, byte[]>(internalPublisherConnection, data));
       }
 
-      private void myOnData(string sender, byte[] msg)
-      {
+      private void myOnData(string sender, byte[] msg) {
          if (myDataCb != null)
             myOnData(sender, msg);
          else
             myReceivingQueue.enqueue(new Pair<string, byte[]>(sender, msg));
       }
 
-      private void myOnConnection(string clientURL, bool connected)
-      {
+      private void myOnConnection(string clientURL, bool connected) {
          if (myConnectionCb != null)
             myConnectionCb(clientURL, connected);
       }

src/DataStreamSuite/PubSub/Server/PubSubMessageReactor.cs

@@ -58,7 +58,7 @@ namespace BCG {
 
          myReceivingProcThread = new Thread(new ThreadStart(myReceivingThreadProc));
          myMessageHub = ServerMessageHub.instance;
-         myMessageHub.registerConnectionCallback(new ConnectionStatusChange(myHandleConnections));
+         myMessageHub.registerConnectionCallback(new PubSubServerConnectionStatusChange(myHandleConnections));
 
 
       }

src/DataStreamSuite/PubSub/Server/PubSubState.cs

@@ -55,7 +55,7 @@ namespace BCG {
             myTagToPubsubMap[tag].onSub(sub);
 
             if (onSingleTagChange != null)
-               onSingleTagChange(new PubSubSingleTagStateEvent(tag, PubSubSingleTagStateEvent.EventType.subscription, sub.connection, sub.subscribingApp));
+               onSingleTagChange(new PubSubSingleTagStateEvent(tag, PubSubSingleTagStateEvent.EventType.subscription, sub.connection, sub.subscribing_app));
          }
          if (onTagMapChanged != null)
             onTagMapChanged(sub.tags);
@@ -68,10 +68,10 @@ namespace BCG {
                ///and publishers for the tag then remove it.
                myTagToPubsubMap[tag].onUnsub(unsub);
                if (onSingleTagChange != null)
-                  onSingleTagChange(new PubSubSingleTagStateEvent(tag, PubSubSingleTagStateEvent.EventType.unsubscription, unsub.connection, unsub.unsubscribingApp));
+                  onSingleTagChange(new PubSubSingleTagStateEvent(tag, PubSubSingleTagStateEvent.EventType.unsubscription, unsub.connection, unsub.unsubscribing_app));
             } else {
                string warningMsg = String.Format("{0} was never published or subscribed on this pubsub, hence" +
-                  " the unsub call by app {1} is voided", tag, unsub.unsubscribingApp);
+                  " the unsub call by app {1} is voided", tag, unsub.unsubscribing_app);
                //BlueEventLog.logEvent(warningMsg, BlueEventLevel.Warning);
             }
          }

src/DataStreamSuite/PubSub/Server/SingleTagPubSubMap.cs

@@ -1,19 +1,19 @@
 using System;
 using System.Collections.Generic;
 using System.Text;
+
 using BCG.Utilities;
+
 using mbts;
 
-namespace BCG
-{
+namespace BCG {
    /// <summary>
    /// SingleTagPubSubMap consists of all the state w.r.t to
    /// publishing and subscription of a single tag. this information
    /// consists of the publisher, and a map of the subscriber
    /// connection vs the apps they're coming from 
    /// </summary>
-   internal class SingleTagPubSubMap
-   {
+   internal class SingleTagPubSubMap {
       private string myTag;
       /// <summary>
       /// map of clientURL to client app (machine;app;pid;clientURL)
@@ -29,8 +29,7 @@ namespace BCG
       /// </summary>
       List<Pair<string, string>> myDeadPubConnToApp = new List<Pair<string, string>>();
 
-      public SingleTagPubSubMap(string tag)
-      {
+      public SingleTagPubSubMap(string tag) {
          myTag = tag;
       }
 
@@ -38,8 +37,7 @@ namespace BCG
       /// Returns a list all the active connections that 
       /// have subscribed to this tag
       /// </summary>
-      public ICollection<string> subscriberConnections
-      {
+      public ICollection<string> subscriberConnections {
          get { return mySubConnToApp.Keys; }
       }
 
@@ -47,8 +45,7 @@ namespace BCG
       /// Returns a list of all the applications that have
       /// subscribed to this tag
       /// </summary>
-      public ICollection<string> subscripingApps
-      {
+      public ICollection<string> subscripingApps {
          get { return mySubConnToApp.Values; }
       }
 
@@ -57,8 +54,7 @@ namespace BCG
       /// connections as key and the application
       /// as value
       /// </summary>
-      public Dictionary<string, string> subscribers
-      {
+      public Dictionary<string, string> subscribers {
          get { return mySubConnToApp; }
       }
 
@@ -66,8 +62,7 @@ namespace BCG
       /// Returns a dictionary for all active publishing
       /// connection as key and the application as value
       /// </summary>
-      public Dictionary<string, string> activePublishers
-      {
+      public Dictionary<string, string> activePublishers {
          get { return myActivePubConnToApp; }
       }
 
@@ -75,8 +70,7 @@ namespace BCG
       /// Returns a list of pair of dead connections and
       /// theur corresponding applications
       /// </summary>
-      public ICollection<Pair<string, string>> deadPublishers
-      {
+      public ICollection<Pair<string, string>> deadPublishers {
          get { return myDeadPubConnToApp; }
       }
 
@@ -87,18 +81,14 @@ namespace BCG
       /// </summary>
       /// <param name="sub"></param>
       /// <returns></returns>
-      public bool onSub(SubMetaData sub)
-      {
-         if (mySubConnToApp.ContainsKey(sub.connection))
-         {
+      public bool onSub(SubMetaData sub) {
+         if (mySubConnToApp.ContainsKey(sub.connection)) {
             string warningMsg = String.Format("tag {0} has already been subscribed by app {0}, subscriber {1}"
-               + "it should unsubscribe before resubscribing", myTag, sub.connection, sub.subscribingApp);
+               + "it should unsubscribe before resubscribing", myTag, sub.connection, sub.subscribing_app);
             //BlueEventLog.logEvent(warningMsg, BlueEventLevel.Warning);
             return false;
-         }
-         else
-         {
-            mySubConnToApp.Add(sub.connection, sub.subscribingApp);
+         } else {
+            mySubConnToApp.Add(sub.connection, sub.subscribing_app);
             return true;
          }
       }
@@ -110,17 +100,13 @@ namespace BCG
       /// </summary>
       /// <param name="unsub"></param>
       /// <returns></returns>
-      public bool onUnsub(UnSubMetaData unsub)
-      {
-         if (mySubConnToApp.ContainsKey(unsub.connection))
-         {
+      public bool onUnsub(UnSubMetaData unsub) {
+         if (mySubConnToApp.ContainsKey(unsub.connection)) {
             mySubConnToApp.Remove(unsub.connection);
             return true;
-         }
-         else
-         {
+         } else {
             string warningMsg = String.Format("tag {0} has already was either unsubscribed or never subscribed by app {0}, subscriber {1}",
-               myTag, unsub.connection, unsub.unsubscribingApp);
+               myTag, unsub.connection, unsub.unsubscribing_app);
             //BlueEventLog.logEvent(warningMsg, BlueEventLevel.Warning);
             return false;
          }
@@ -133,25 +119,20 @@ namespace BCG
       /// </summary>
       /// <param name="subscriber"></param>
       /// <returns></returns>
-      public bool onConnectionRemoveIfExists(string connection, out bool wasPublisher, out bool wasSubscriber)
-      {
+      public bool onConnectionRemoveIfExists(string connection, out bool wasPublisher, out bool wasSubscriber) {
          bool rv = false;
-         if (mySubConnToApp.ContainsKey(connection))
-         {
+         if (mySubConnToApp.ContainsKey(connection)) {
             mySubConnToApp.Remove(connection);
             wasSubscriber = true;
             rv = true;
-         }
-         else
+         } else
             wasSubscriber = false;
-         if (myActivePubConnToApp.ContainsKey(connection))
-         {
+         if (myActivePubConnToApp.ContainsKey(connection)) {
             myDeadPubConnToApp.Add(new Pair<string, string>(connection, myActivePubConnToApp[connection]));
             myActivePubConnToApp.Remove(connection);
             rv = true;
             wasPublisher = true;
-         }
-         else
+         } else
             wasPublisher = false;
          return rv;
       }
@@ -162,14 +143,10 @@ namespace BCG
       /// </summary>
       /// <param name="pub"></param>
       /// <param name="publisherChanged"></param>
-      public void onPub(PubMetaData pub, out bool newPublisher)
-      {
-         if (myActivePubConnToApp.ContainsKey(pub.connection))
-         {
+      public void onPub(PubMetaData pub, out bool newPublisher) {
+         if (myActivePubConnToApp.ContainsKey(pub.connection)) {
             newPublisher = false;
-         }
-         else
-         {
+         } else {
             myActivePubConnToApp.Add(pub.connection, pub.publishingApp);
             newPublisher = true;
          }

src/DataStreamSuite/PubSub/SingleServerSubscriberCore.cs

@@ -6,92 +6,79 @@ using BCG.Utilities;
 using BCG.Utilities.General;
 using mbts;
 using mbts.spark;
+using System.Linq;
 
 namespace BCG {
    /// <summary>
-   /// Deals with the subscriptions (and getting data) from one
-   /// pubsub server. Once again, theres no sharing of conenctions
+   /// Deals with the subscriptions (and getting data) from one sole
+   /// pubsub server. Theres no sharing of conenctions
    /// from this point. Every new instance of SingleServerSubscriberCore
    /// will open its new dedicated connection to the serverURL that
    /// its concerned with (passed as a conctructor arg).
    /// </summary>
 
    internal interface ISingleServerSubscriberCore : IDisposable {
-      IList<PubSubUpdate> getUpdates(int timeout);
+      IList<PubSubUpdate> get_updates(int timeout);
       event PubSubConnectionChange onPubSubConnectionChange;
       void subscribe(params string[] tags);
       void unsubscribe(params string[] tags);
    }
+
    internal class SingleServerSubscriberCore : IDisposable, ISingleServerSubscriberCore {
-      private HighPerformanceTimer myTimer = new HighPerformanceTimer();
-      private PubSubServerConnectionFactory myConnectionFactory;
-      private IPubSubServerConnection myConnection;
-      private List<string> myTags = new List<string>();
+      private readonly IPubSubServerConnection _connection;
+      private HashSet<string> _tags = new HashSet<string>();
 
       public event PubSubConnectionChange onPubSubConnectionChange;
 
-      public SingleServerSubscriberCore(string serverURL, PubSubServerConnectionFactory connectionFactory) {
-         myConnectionFactory = connectionFactory;
-         myConnection = myConnectionFactory.getConnection(serverURL);
-         myConnection.registerCallback(new ConnectionStatusChange(myHandleServerReboot));
+      public SingleServerSubscriberCore(string server_url, PubSubServerConnectionFactory connection_factory) {
+         _connection = connection_factory.getConnection(server_url);
+         _connection.register_connection_callback(new PubSubServerConnectionStatusChange(_handle_server_reboot));
       }
 
-      public IList<PubSubUpdate> getUpdates() {
-         return getUpdates(1);
-      }
-
-      /// <summary>
-      /// gets as many data items that it can get in
-      /// timeout milliseconds
-      /// </summary>
-      /// <param name="timeout"></param>
-      /// <returns></returns>
-      public IList<PubSubUpdate> getUpdates(int timeout) {
-         myConnection.doWork();
-         List<PubSubUpdate> retVal = new List<PubSubUpdate>();
-         IList<byte[]> dataItems = myConnection.getUpdates(new TimeSpan(0, 0, 0, 0, timeout), Int32.MaxValue);
-         foreach (byte[] data in dataItems) {
-            processMessage(data, retVal);
+      // gets as many data items that it can get in
+      // timeout milliseconds
+      public IList<PubSubUpdate> get_updates(int timeout) {
+         _connection.doWork();
+         List<PubSubUpdate> rv = new List<PubSubUpdate>();
+         IList<byte[]> data_items = _connection.get_updates(new TimeSpan(0, 0, 0, 0, timeout), Int32.MaxValue);
+         foreach (byte[] data in data_items) {
+            process_message(data, ref rv);
          }
-         return retVal;
+         return rv;
 
       }
 
-      public List<string> getSubscribedTags() {
-         return myTags;
-      }
-
       public void subscribe(params string[] tags) {
+         HashSet<string> this_call_tags = new HashSet<string>();
          foreach (string tag in tags) {
-            if (myTags.Contains(tag)) {
-               string errMsg = String.Format("tag {0} was already subscribed from server {1}",
-                  tag, myConnection.serverURL);
-               // BlueEventLog.logEvent(errMsg, BlueEventLevel.Warning);
+            if (_tags.Contains(tag)) {
+               bts.Log.Gen.warn($"SingleServerSubscriberCore.subscribe tag: {tag} has already " +
+                                 $"been subscribed to server: {_connection.server_url}");
             } else {
-               myTags.Add(tag);
+               _tags.Add(tag);
+               this_call_tags.Add(tag);
             }
          }
-         mySubscribe(tags);
+         _subscribe(this_call_tags.ToArray());
       }
 
       public void unsubscribe(params string[] tags) {
-         List<string> unsubTags = new List<string>();
+         List<string> unsub_tags = new List<string>();
          foreach (string tag in tags) {
-            if (myTags.Contains(tag)) {
-               myTags.Remove(tag);
-               unsubTags.Add(tag);
+            if (_tags.Contains(tag)) {
+               _tags.Remove(tag);
+               unsub_tags.Add(tag);
             } else {
-               string errMsg = String.Format("tag {0} is being unsubscribed from server {1}, eventhough " +
-                  "it was never subscribed", tag, myConnection.serverURL);
-               // BlueEventLog.logEvent(errMsg, BlueEventLevel.Warning);
+               bts.Log.Gen.warn($"tag {tag} is being unsubscribed from server {_connection.server_url}, eventhough " +
+                                 "it was never subscribed");
             }
          }
-         UnSubMetaData unsub = new UnSubMetaData(myConnection.clientURL, unsubTags.ToArray());
+         UnSubMetaData unsub = new UnSubMetaData(_connection.client_url, unsub_tags.ToArray());
          Stream str = new MemoryStream();
          new MetaData(unsub).save(str);
          str.Position = 0;
          // this is where data is actually sent out
-         myConnection.send(str);
+         _connection.send(str);
       }
 
       #region IDisposable Members
@@ -99,44 +86,40 @@ namespace BCG {
       public void Dispose() {
          ///remove the subscriber
          //myConnection.send(new PubSubMessage(new MetaData(new RemoveSubscriberMetaData(myConnection.clientURL))).toStream());
-         myConnection.Dispose();
+         _connection.Dispose();
 
          //clear the list of tags you've subscribed to. This way, if you're ever used again, 
          //you have to send all the subscription metadata out again.
-         myTags.Clear();
+         _tags.Clear();
       }
 
       #endregion
 
-      private void mySubscribe(params string[] tags) {
-         SubMetaData sub = new SubMetaData(myConnection.clientURL, tags);
+      // The only place where a tag is subsribed from the Pubsub server. the subscriptions are wrapped into SubMetaData,
+      // which has a to_stream method
+      private void _subscribe(params string[] tags) {
+         SubMetaData sub = new SubMetaData(_connection.client_url, tags);
          Stream str = new MemoryStream();
          new MetaData(sub).save(str);
          str.Position = 0;
-         myConnection.send(str);
+         _connection.send(str);
       }
 
-      private void myHandleServerReboot(string serverURL, bool connection) {
+      private void _handle_server_reboot(string server_url, bool connection) {
+         bts.Log.Gen.info($"SingleServerSubscriberCore._handle_server_reboot connection: {connection} server_url: {server_url}");
          if (connection) {
-            //ie connection established again, so has to
-            //resubscribe to all the tags
-            // BlueEventLog.logEvent("Connection to " + serverURL + " established....resubscribing to all the tags",
-            //    PS3LogLevels.disconnection);
-            mySubscribe(myTags.ToArray());
-
-
+            _subscribe(_tags.ToArray());
          } else {
-            // BlueEventLog.logEvent("Connection to " + serverURL + " broke....attempting to reconnect...",
-            //    PS3LogLevels.disconnection);
+            bts.Log.Gen.warn($"SingleServerSubscriberCore._handle_server_reboot connection to {server_url}  broke....attempting to reconnect...");
          }
-
          //let anyone watching you know that your connection has changed. 
          if (onPubSubConnectionChange != null)
-            onPubSubConnectionChange(connection, serverURL);
+            onPubSubConnectionChange(connection, server_url);
       }
 
-      internal static void processMessage(byte[] data, List<PubSubUpdate> updates) {
-         bts.Log.Gen.info($"SingleServerSubscriberCore.processMessage data_len: {data.Length}");
+      // processes one message
+      internal static void process_message(byte[] data, ref List<PubSubUpdate> updates) {
+         bts.Log.Gen.info($"SingleServerSubscriberCore.process_message data_len: {data.Length}");
          if (data != null && data.Length > 0) {
             MemoryStream str = new MemoryStream(data);
             MetaData md = new MetaData(str);
@@ -156,9 +139,8 @@ namespace BCG {
                if (p.isCompressed) {
                   long compressedSize = bodyStream.Length - bodyStream.Position;
                   if (compressedSize == 0) {
-                     string errorMsg = "Error: tag " + p.tag + " had compressed size of 0 bytes. This shouldnt' happen. Not decompressing tag or sending out update.";
-                     Console.WriteLine(errorMsg);
-                     //BlueEventLog.logEvent(errorMsg, BlueEventLevel.Error);
+                     string error_msg = "Error: tag " + p.tag + " had compressed size of 0 bytes. This shouldnt' happen. Not decompressing tag or sending out update.";
+                     bts.Log.Gen.error(error_msg);
                      return;
                   }
                   realBody = StreamDecompressor.decompressStream(bodyStream);
@@ -174,8 +156,9 @@ namespace BCG {
                   bodyStream.Read(trConstituent, 0, trConstituent.Length);
                   updates.Add(new PubSubUpdate(pub.tag, pub.type, new MemoryStream(trConstituent)));
                }
-            } else { }
-            //BlueEventLog.logAssert(false, "Unknown message received: " + md.ToString() + " , only Pub and TransactionMetaData are legal");
+            } else {
+               bts.Log.assert_d(false, $"Unknown message received: {md} only Pub and TransactionMetaData are legal");
+            }
          }
       }
    }

src/DataStreamSuite/PubSub/StaticPublisherCore.cs

@@ -387,18 +387,18 @@ namespace BCG {
             SharpZipCompressor Kompressor = new SharpZipCompressor();
             Kompressor.compress(content, contentToPublish);
             contentToPublish.Position = 0;
-            pub_md = new PubMetaData(tagType, tag, (int)contentToPublish.Length, (int)content.Length, serverConn.serverURL, ClientInfo.thisClient.processString);
+            pub_md = new PubMetaData(tagType, tag, (int)contentToPublish.Length, (int)content.Length, serverConn.server_url, ClientInfo.thisClient.processString);
             if (PubSub3Config.instance.shouldReportTagsCompressionRatio(tag)) {
                string msg = String.Format("Compressed tag {0} from {1} to {2} bytes, for a ratio of {3}.", tag, content.Length, contentToPublish.Length, (float)content.Length / (float)contentToPublish.Length);
                // BlueEventLog.logEvent(msg, PS3LogLevels.compressionRatio);
             }
          } else
-            pub_md = new PubMetaData(tagType, tag, (int)content.Length, serverConn.serverURL, ClientInfo.thisClient.processString);
+            pub_md = new PubMetaData(tagType, tag, (int)content.Length, serverConn.server_url, ClientInfo.thisClient.processString);
 
          content.Position = initPos;
          if (CrushedTagManager.instance.is_tag_crushable(tag)) {
             string refId = String.Empty;
-            Console.WriteLine("StaticPublisherCore.mySend serverConn.sendWithConfirmation tag: " + tag + " server: " + serverConn.serverURL);
+            Console.WriteLine("StaticPublisherCore.mySend serverConn.sendWithConfirmation tag: " + tag + " server: " + serverConn.server_url);
             //  this is where the data is actually sent out (there are multiple sends)
             PubSubMessage ps_msg = new PubSubMessage(new MetaData(pub_md), contentToPublish);
             //Console.WriteLine($"serverConn.send_with_confirmation metadata: {ps_msg.meta_data}");
@@ -428,8 +428,8 @@ namespace BCG {
             myTagToSender.Add(tag, sender);
             if (!mySenders.Contains(sender)) {
                mySenders.Add(sender);
-               sender.registerCallback(new ConnectionStatusChange(myHandleServerReboot));
-               sender.registerSendConfirmationCallback(new SendConfirmationDelegate(myOnSendConfirm));
+               sender.register_connection_callback(new PubSubServerConnectionStatusChange(myHandleServerReboot));
+               sender.register_send_confirmation_callback(new SendConfirmationDelegate(myOnSendConfirm));
             }
 
          }
@@ -443,7 +443,7 @@ namespace BCG {
          if (connection) {
             string thisPublisherInfo = ClientInfo.thisClient.processString;
             foreach (KeyValuePair<string, IPubSubServerConnection> p in myTagToSender) {
-               if (p.Value.serverURL == serverURL) {
+               if (p.Value.server_url == serverURL) {
                   string tagType;
                   string tag = p.Key;
                   myCrushableTagsWaitingOnAck.Clear();

src/DataStreamSuite/PubSub/StaticSubscriberCore.cs

@@ -80,7 +80,7 @@ namespace BCG {
                int timeShare = timeout / myServerURLToSubCore.Count;
                List<PubSubUpdate> retVal = new List<PubSubUpdate>();
                foreach (ISingleServerSubscriberCore subCore in myServerURLToSubCore.Values)
-                  retVal.AddRange(subCore.getUpdates(timeShare));
+                  retVal.AddRange(subCore.get_updates(timeShare));
                _log_n_updates(retVal);
                myTimer.stop();
                //if (myTimer.lastIntervalTime > 10)

src/DataStreamSuite/PubSubServer/Program.cs

@@ -15,7 +15,7 @@ namespace BCG {
       /// </summary>
       [STAThread]
       public static void Main(string[] args) {
-         bool run_echo_tcp_server_only = true;
+         bool run_echo_tcp_server_only = false;
          bts.utils.BTSNativeMethods.provide_command_line(new string[] { }, 0);
          mbts.config.Configuration.initialize();
          if (run_echo_tcp_server_only) {

src/DataStreamSuite/PubSubServer/PubSubServer.csproj

@@ -8,7 +8,7 @@
   </ItemGroup>
   <PropertyGroup>
     <OutputType>Exe</OutputType>
-    <TargetFramework>net7.0</TargetFramework>
+    <TargetFramework>net8.0</TargetFramework>
     <Nullable>enable</Nullable>
   </PropertyGroup>
 

src/DataStreamSuite/SharedStreams/Business/MarketData/TimSeriesOperations/TimeSeriesGroupByStream.cs

@@ -1,26 +1,20 @@
-namespace BCG
-{
+namespace BCG {
    using bts.api.orderman;
 
-   enum TSGroupByOperation
-   {
+   enum TSGroupByOperation {
       OHLC = 0,
       Statistics
    }
 
    [DataStreamUserRespresentation("timeseriesanalysis")]
-   public class TimeSeriesGroupByStream : ViewStream
-   {
-      public TimeSeriesGroupByStream(ViewStream md_str, string timebucket_col, string output_col, string operation)
-      {
+   public class TimeSeriesGroupByStream : ViewStream {
+      public TimeSeriesGroupByStream(ViewStream md_str, string timebucket_col, string output_col, string operation) {
          ViewStream rs = new NonIncGroupByStream(md_str, new OHLCNonIncGroupByEvaluator(timebucket_col, output_col));
          mySetRealSource(rs);
       }
    }
-   public class OHLC
-   {
-      public OHLC(double open_args, double high_args, double low_args, double close_args)
-      {
+   public class OHLC {
+      public OHLC(double open_args, double high_args, double low_args, double close_args) {
          open = open_args;
          high = high_args;
          low = low_args;
@@ -28,47 +22,36 @@ namespace BCG
       }
       public readonly double open, high, low, close;
    }
-   public class OHLCNonIncGroupByEvaluator : INonIncGroupByEvaluator
-   {
+   public class OHLCNonIncGroupByEvaluator : INonIncGroupByEvaluator {
       private string _timestamp_col;
       private string _output_col;
       private IBlueTableSchema _out_schema;
-      public OHLCNonIncGroupByEvaluator(string timestamp_col, string output_col)
-      {
+      public OHLCNonIncGroupByEvaluator(string timestamp_col, string output_col) {
          _timestamp_col = timestamp_col;
          _output_col = output_col;
       }
 
-      string open_col
-      {
-         get
-         {
+      string open_col {
+         get {
             return $"{_output_col}_open"; ;
          }
       }
-      string high_col
-      {
-         get
-         {
+      string high_col {
+         get {
             return $"{_output_col}_high"; ;
          }
       }
-      string low_col
-      {
-         get
-         {
+      string low_col {
+         get {
             return $"{_output_col}_low"; ;
          }
       }
-      string close_col
-      {
-         get
-         {
+      string close_col {
+         get {
             return $"{_output_col}_close"; ;
          }
       }
-      private void _build_schema(IBlueTableSchema input_schema)
-      {
+      private void _build_schema(IBlueTableSchema input_schema) {
          List<Tuple<Type, string>> new_cols = new List<Tuple<Type, string>>();
          ColumnInfo ci = input_schema[_output_col];
          new_cols.Add(Tuple.Create(ci.colType, open_col));
@@ -79,32 +62,25 @@ namespace BCG
          _out_schema = BlueDataUtility.add_columns(input_schema, new_cols.ToArray());
       }
 
-      public IBlueRowCollection evaluate(IDictionary<int, List<IBlueRow>> grouped_rows)
-      {
-         if (_out_schema == null)
-         {
-            if (grouped_rows.Count > 0)
-            {
-               foreach (var kv in grouped_rows)
-               {
+      public IBlueRowCollection evaluate(IDictionary<int, List<IBlueRow>> grouped_rows) {
+         if (_out_schema == null) {
+            if (grouped_rows.Count > 0) {
+               foreach (var kv in grouped_rows) {
                   var rows = kv.Value;
-                  if (rows != null && rows.Count > 0)
-                  {
+                  if (rows != null && rows.Count > 0) {
                      _build_schema(rows.ElementAt(0).schema);
                   }
                }
             }
          }
          BlueTable rv = new BlueTable(_out_schema);
-         foreach (var kv in grouped_rows)
-         {
+         foreach (var kv in grouped_rows) {
             int id = kv.Key;
             IEnumerable<IBlueRow> rows = kv.Value;
             // sort rows on time to compute o,c correctly??
-            if (rows != null && rows.Count() > 0)
-            {
+            if (rows != null && rows.Count() > 0) {
                IBlueRow reprow = rows.ElementAt(0);
-               IEnumerable<double> vals = BlueDataUtility.get_covalues<double>(rows, _output_col);
+               IEnumerable<double> vals = BlueDataUtility.get_colvalues<double>(rows, _output_col);
                OHLC ohlc = compute_ohlc(vals);
                MutableBlueRow nr = new MutableBlueRow(_out_schema, reprow);
                nr.setValue("id", kv.Key);
@@ -118,13 +94,11 @@ namespace BCG
          return rv;
       }
 
-      public string[] get_group_by_cols()
-      {
+      public string[] get_group_by_cols() {
          return new string[] { _timestamp_col };
       }
 
-      static OHLC compute_ohlc(IEnumerable<double> prices)
-      {
+      static OHLC compute_ohlc(IEnumerable<double> prices) {
          double o = prices.ElementAt(0);
          double c = prices.ElementAt(prices.Count() - 1);
          double l = prices.Min();
@@ -133,28 +107,22 @@ namespace BCG
       }
    }
    [DataStreamUserRespresentation("RSI")]
-   public class RSIStream : ViewStream
-   {
+   public class RSIStream : ViewStream {
       // for ms_str expects a timeseries with a timestamp col and an output col (price col) to calculate the ohlc for
-      public RSIStream(ViewStream md_str, string timebucket_column, string output_col)
-      {
+      public RSIStream(ViewStream md_str, string timebucket_column, string output_col) {
 
       }
    }
    [DataStreamUserRespresentation("Bollinger")]
-   public class BollingerBandStream : ViewStream
-   {
+   public class BollingerBandStream : ViewStream {
       // for ms_str expects a timeseries with a timestamp col and an output col (price col) to calculate the ohlc for
-      public BollingerBandStream(ViewStream md_str, string timebucket_column, string output_col)
-      {
+      public BollingerBandStream(ViewStream md_str, string timebucket_column, string output_col) {
       }
    }
    [DataStreamUserRespresentation("OHLC")]
-   public class OHLCStream : ViewStream
-   {
+   public class OHLCStream : ViewStream {
       // for ms_str expects a timeseries with a timestamp col and an output col (price col) to calculate the ohlc for
-      public OHLCStream(ViewStream md_str, string timebucket_column, string output_col)
-      {
+      public OHLCStream(ViewStream md_str, string timebucket_column, string output_col) {
          ViewStream rs = new TimeSeriesGroupByStream(md_str, timebucket_column, output_col, TSGroupByOperation.OHLC.ToString());
          mySetRealSource(rs);
       }

src/DataStreamSuite/SharedStreams/SharedStreams.csproj

@@ -1,7 +1,7 @@
 <Project Sdk="Microsoft.NET.Sdk">
 
   <PropertyGroup>
-    <TargetFramework>net7.0</TargetFramework>
+    <TargetFramework>net8.0</TargetFramework>
     <ImplicitUsings>enable</ImplicitUsings>
     <Nullable>enable</Nullable>
   </PropertyGroup>

src/DataStreamSuite/TransportUtilities/TransportUtilities.csproj

@@ -1,7 +1,7 @@
 <Project Sdk="Microsoft.NET.Sdk">
 
   <PropertyGroup>
-    <TargetFramework>net7.0</TargetFramework>
+    <TargetFramework>net8.0</TargetFramework>
     <ImplicitUsings>enable</ImplicitUsings>
     <Nullable>enable</Nullable>
   </PropertyGroup>

src/Spark.Admin.WebApp/src/Spark.Admin.WebApp.fsproj

@@ -1,7 +1,7 @@
 <Project Sdk="Microsoft.NET.Sdk">
 
   <PropertyGroup>
-    <TargetFramework>net7.0</TargetFramework>
+    <TargetFramework>net8.0</TargetFramework>
     <Version>1.0.0</Version>
     <AssemblyVersion>1.0.0</AssemblyVersion>
   </PropertyGroup>