DataStreamCore Json stuff

* IsSocketStateOpen
* IsClosedProperly

.editorconfig

@@ -207,17 +207,23 @@ dotnet_naming_symbols.parameters.applicable_accessibilities = public,internal,pr
 dotnet_naming_symbols.private_symbols.applicable_kinds = property,method,field,event,delegate
 dotnet_naming_symbols.private_symbols.applicable_accessibilities = protected,private
 
+# local symbols
+dotnet_naming_symbols.local_symbols.applicable_kinds = local
+
+# parameter symbols
+dotnet_naming_symbols.parameter_symbols.applicable_kinds = parameter
+
 # Naming styles
 
-dotnet_naming_style.snake_case.required_prefix =
-dotnet_naming_style.snake_case.required_suffix =
-dotnet_naming_style.snake_case.word_separator = _
-dotnet_naming_style.snake_case.capitalization = all_lower
+dotnet_naming_style.pascal_case.required_prefix =
+dotnet_naming_style.pascal_case.required_suffix =
+dotnet_naming_style.pascal_case.word_separator =
+dotnet_naming_style.pascal_case.capitalization = pascal_case
 
-dotnet_naming_style.private_snake_case.required_prefix = _
-dotnet_naming_style.private_snake_case.required_suffix =
-dotnet_naming_style.private_snake_case.word_separator = _
-dotnet_naming_style.private_snake_case.capitalization = all_lower
+dotnet_naming_style.camel_case.required_prefix =
+dotnet_naming_style.camel_case.required_suffix =
+dotnet_naming_style.camel_case.word_separator =
+dotnet_naming_style.camel_case.capitalization = camel_case
 
 dotnet_naming_style.user_types.required_prefix =
 dotnet_naming_style.user_types.required_suffix =
@@ -229,6 +235,10 @@ dotnet_naming_style.interface.required_suffix =
 dotnet_naming_style.interface.word_separator =
 dotnet_naming_style.interface.capitalization = pascal_case
 
+dotnet_naming_style.local.capitalization = camel_case
+
+dotnet_naming_style.parameter.capitalization = camel_case
+
 # rules
 dotnet_naming_rule.user_types.symbols = user_types
 dotnet_naming_rule.user_types.style = user_types
@@ -238,14 +248,18 @@ dotnet_naming_rule.interfaces.symbols = interface
 dotnet_naming_rule.interfaces.style = interface_style
 dotnet_naming_rule.interfaces.severity = suggestion
 
-dotnet_naming_rule.public_members_snake_case.symbols = public_symbols
-dotnet_naming_rule.public_members_snake_case.style = snake_case
-dotnet_naming_rule.public_members_snake_case.severity = suggestion
+dotnet_naming_rule.public_members.symbols = public_symbols
+dotnet_naming_rule.public_members.style = pascal_case
+dotnet_naming_rule.public_members.severity = suggestion
 
-dotnet_naming_rule.private_members_snake_case.symbols = private_symbols
-dotnet_naming_rule.private_members_snake_case.style = private_snake_case
-dotnet_naming_rule.private_members_snake_case.severity = suggestion
+dotnet_naming_rule.private_members.symbols = private_symbols
+dotnet_naming_rule.private_members.style = camel_case
+dotnet_naming_rule.private_members.severity = suggestion
 
-dotnet_naming_rule.parameters.symbols = parameters
-dotnet_naming_rule.parameters.style = snake_case
+dotnet_naming_rule.variables_are_camel_case.severity = suggestion
+dotnet_naming_rule.variables_are_camel_case.symbols = local_symbols
+dotnet_naming_rule.variables_are_camel_case.style = local
+
+dotnet_naming_rule.parameters.symbols = parameter_symbols
+dotnet_naming_rule.parameters.style = parameter
 dotnet_naming_rule.parameters.severity = suggestion

docker/sparkd/dev/Dockerfile

@@ -16,7 +16,7 @@ RUN apt-get update && apt-get install -y \
     sqlite3 \
     tzdata
 
-# clang
+# clang 13, 15 (so we can build old branches)
 RUN apt-get update && apt-get install -y \
     clang-13 \
     clang-format-13 \
@@ -25,6 +25,19 @@ RUN apt-get update && apt-get install -y \
     clang-format-15 \
     clangd-15
 
+# clang 17 (so we can build new branches)
+RUN \
+    curl -fsSL https://apt.llvm.org/llvm-snapshot.gpg.key 2> /dev/null \
+    | gpg --dearmor - \
+    | tee /usr/share/keyrings/llvm-snapshot.gpg > /dev/null \
+    && echo "deb [signed-by=/usr/share/keyrings/llvm-snapshot.gpg] http://apt.llvm.org/jammy/ llvm-toolchain-jammy-17 main" \
+    | tee -a "/etc/apt/sources.list.d/llvm.list" \
+    && apt-get update \
+    && apt-get install -y \
+    clang-17 \
+    clang-format-17 \
+    clangd-17
+
 # g++
 RUN apt-get update && apt-get install -y \
     g++-11 \
@@ -57,10 +70,6 @@ RUN \
     && echo "deb [signed-by=/usr/share/keyrings/kitware-archive-keyring.gpg] https://apt.kitware.com/ubuntu/ $(lsb_release -cs) main" \
     | tee /etc/apt/sources.list.d/kitware.list > /dev/null \
     && apt-get update \
-    && rm /usr/share/keyrings/kitware-archive-keyring.gpg \
-    && apt-get install -y kitware-archive-keyring \
-    && echo "deb [signed-by=/usr/share/keyrings/kitware-archive-keyring.gpg] https://apt.kitware.com/ubuntu/ $(lsb_release -cs)-rc main" \
-    | tee -a /etc/apt/sources.list.d/kitware.list >/dev/null \
     && apt-get install -y cmake
 
 # node
@@ -99,6 +108,7 @@ RUN apt-get update && apt-get install -y \
 # dev utils
 RUN apt-get update && apt-get install -y \ 
     fish \
+    gdb \
     git-all \
     sudo \
     vim \
@@ -106,10 +116,10 @@ RUN apt-get update && apt-get install -y \
 
 # set up default executables (update-alternatives)
 RUN \
-    update-alternatives --install /usr/bin/clang++ clang++ /usr/bin/clang++-15 100; \
-    update-alternatives --install /usr/bin/clang clang /usr/bin/clang-15 100; \
-    update-alternatives --install /usr/bin/clang-format clang-format /usr/bin/clang-format-15 100; \
-    update-alternatives --install /usr/bin/clangd clangd /usr/bin/clangd-15 100; \
+    update-alternatives --install /usr/bin/clang++ clang++ /usr/bin/clang++-17 100; \
+    update-alternatives --install /usr/bin/clang clang /usr/bin/clang-17 100; \
+    update-alternatives --install /usr/bin/clang-format clang-format /usr/bin/clang-format-17 100; \
+    update-alternatives --install /usr/bin/clangd clangd /usr/bin/clangd-17 100; \
     update-alternatives --install /usr/bin/g++ g++ /usr/bin/g++-12 100; \
     update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-12 100
 
@@ -121,6 +131,8 @@ RUN \
     ln -s /usr/bin/ccache /usr/local/bin/clang++-13; \
     ln -s /usr/bin/ccache /usr/local/bin/clang-15; \
     ln -s /usr/bin/ccache /usr/local/bin/clang++-15; \
+    ln -s /usr/bin/ccache /usr/local/bin/clang-17; \
+    ln -s /usr/bin/ccache /usr/local/bin/clang++-17; \
     ln -s /usr/bin/ccache /usr/local/bin/gcc; \
     ln -s /usr/bin/ccache /usr/local/bin/g++; \
     ln -s /usr/bin/ccache /usr/local/bin/gcc-11; \

src/DataStreamSuite/Bts.DataStream.Websocket/Bts.DataStream.Websocket.csproj

@@ -1,7 +1,7 @@
 <Project Sdk="Microsoft.NET.Sdk">
 
   <PropertyGroup>
-    <TargetFramework>net7.0</TargetFramework>
+    <TargetFramework>net8.0</TargetFramework>
     <ImplicitUsings>enable</ImplicitUsings>
     <Nullable>enable</Nullable>
   </PropertyGroup>

src/DataStreamSuite/Bts.DataStream.Websocket/DataStreamDataSync.cs

@@ -0,0 +1,119 @@
+namespace Bts.DataStream.Websocket;
+
+using BCG;
+
+using bts;
+using bts.collections;
+using bts.websocket_core;
+
+using Bts.DataStreamExpressions;
+using Bts.Websocket;
+
+using mbts.utils;
+
+using Spark.Core.Disposables;
+
+using System.Text.Json;
+using System.Text.Json.Nodes;
+
+public class DataStreamDataSync(JsonSerializerOptions? serializerOpts = null) : JsonRpcDataSync(serializerOpts) {
+   readonly Dictionary<WebsocketClientID, List<ClientSubscription>> clients = [];
+
+   protected override void onConnect(WebsocketConnectNotification notification) {
+      var id = notification.Id;
+      lock (lockObj) {
+         clients.Add(id, []);
+      }
+      Log.Api.info($"DataStreamWebsocketDataSync client connected. clientId: {notification.Id}");
+      if (notification.Message is not null) {
+         onIncomingMessage(notification.Message);
+      }
+   }
+
+   protected override void onDisconnect(WebsocketDisconnectNotification notification) {
+      Log.Api.warn($"DataStreamWebsocketDataSync client disconnected. clientId: {notification.Id}");
+      unsubscribeAll(notification.Id);
+      clients.Remove(notification.Id);
+   }
+
+   protected override void onRequest(WebsocketClientID id, JsonRpcRequest<JsonObject?> req) {
+      switch (req.Method!.ToLower()) {
+         case "subscribe":
+            subscribe(id, req);
+            break;
+         case "unsubscribe":
+            unsubscribe(id, req);
+            break;
+         case "unsubscribe_all":
+            unsubscribeAll(id);
+            break;
+         default:
+            Log.Api.error($"Received request of unknown type: {req}");
+            break;
+      }
+   }
+
+   void subscribe(WebsocketClientID id, JsonRpcRequest<JsonObject?> req) {
+      var subParams = req.Params!.Deserialize<SubscriptionParams>()!;
+      var exp = DataStreamExpressionsInterpreter.Evaluate(subParams.Query!);
+      var stream = exp.getView(new Dictionary<string, StatementExpression>(0));
+      var sub = new ClientSubscription(id, subParams, Disposable.Create(Dispose));
+      lock (lockObj) {
+         clients[id].Add(sub);
+         sendResponse(id, req.Id!.Value, req.Params);
+         stream.onChange += OnChange;
+      }
+
+      void OnChange(object? newValue) {
+         var dp = ExceptionAssert.of_type<DataPacket>(newValue);
+         var rpcParams = new {
+            dp.addedRows,
+            dp.deletedRows
+         };
+         sendNotification(id, "subscription", rpcParams);
+      }
+
+      void Dispose() =>
+         stream.onChange -= OnChange;
+   }
+
+   void unsubscribe(WebsocketClientID id, JsonRpcRequest<JsonObject?> req) {
+      var subParams = req.Params!.Deserialize<SubscriptionParams>();
+      var subParamsStr = JsonSerializer.Serialize(subParams, serializerOpts);
+      // Minimize amount of code inside lock (just need it to find/remove the subscription)
+      List<ClientSubscription>? subs = null;
+      ClientSubscription? sub = null;
+      lock (lockObj) {
+         subs = clients.GetValueOrDefault(id);
+         subs?.TryRemoveFirst(s => s!.Params.Equals(subParams), out sub);
+      }
+      if (subs is null) {
+         // Didn't find client (log/throw).
+         var msg = $"Unable to unsubscribe: did not find client. Id: {id}.\nParams:\n{subParamsStr}";
+         Log.Api.error(msg);
+         throw new Exception(msg);
+      }
+      if (sub is null) {
+         // Found client, but didn't find subscription (log/throw).
+         var msg = $"Unable to unsubscribe: did not find the specified subscription. " +
+            $"Id: {id}. Request:\n{subParams}";
+         Log.Api.error(msg);
+         throw new Exception(msg);
+      }
+      // Found client and subscription (dispose/remove subscription).
+      Log.Api.info($"Unsubscribing client. Id: {id}.\nParams:\n{subParamsStr}");
+      sub.Dispose();
+   }
+
+   void unsubscribeAll(WebsocketClientID id) {
+      lock (lockObj) {
+         if (!clients.TryGetValue(id, out var subs)) {
+            throw new Exception(
+               $"Unable to unsubscribe: did not find client with id {id}");
+         }
+         subs.DisposeAll();
+         subs.Clear();
+      }
+   }
+
+}

src/DataStreamSuite/Bts.DataStream.Websocket/DataSync.cs

@@ -1,229 +1,228 @@
-namespace Bts.DataStream.Websocket;
-
-using BCG;
-
-using bts;
-using bts.api.core;
-using bts.collections;
-
-using Bts.DataStreamExpressions;
-using Bts.Websocket;
-
-using mbts.utils;
-
-using Spark.Core.Disposables;
-
-using System.Reactive.Subjects;
-using System.Text.Json;
-using System.Text.Json.Nodes;
-
-/// <summary>
-/// Represents an I/O model (<see cref="ISubject{TSource, TResult}"/>), that accepts websocket
-/// requests (from clients) as input, and emits data/notifications (to clients) as output.
-/// This class's chief role is managing data subscriptions to sparkd on the clients' behalf.
-/// </summary>
-public class DataSync(ServiceContext serviceCtx, JsonSerializerOptions? serializerOpts = null) :
-   ISubject<WebsocketConnectionUpdate, IEnumerable<WebsocketOutgoingMessage>> {
-   readonly ServiceContext serviceCtx = serviceCtx;
-   readonly JsonSerializerOptions serializerOpts = serializerOpts ?? JsonSerializerOptions.Default;
-   readonly Dictionary<WebsocketClientID, List<ClientSubscription>> clients = [];
-   readonly object lockObj = new();
-
-   event Action<IEnumerable<WebsocketOutgoingMessage>>? requestSend;
-
-   public void OnCompleted() =>
-      Log.Api.info("DataSync incoming request stream has completed.");
-
-   public void OnError(Exception error) =>
-      Log.Api.error($"DataSync error: {error}");
-
-   public void OnNext(WebsocketConnectionUpdate value) {
-      switch (value.Type) {
-         case WebsocketConnectionUpdateType.IncomingMessage:
-            onIncomingMessage(value.IncomingMessage);
-            break;
-         case WebsocketConnectionUpdateType.Connect:
-            onConnect(value.Connect);
-            break;
-         case WebsocketConnectionUpdateType.Disconnect:
-            onDisconnect(value.Disconnect);
-            break;
-         default:
-            Log.assert(false, $"Unexpected WebsocketConnectionUpdate type: {value.Type}");
-            break;
-      }
-   }
-
-   public IDisposable Subscribe(IObserver<IEnumerable<WebsocketOutgoingMessage>> observer) {
-      lock (lockObj) {
-         requestSend += observer.OnNext;
-         return Disposable.Create(dispose);
-      }
-      void dispose() {
-         lock (lockObj) {
-            requestSend -= observer.OnNext;
-         }
-      }
-   }
-
-   static JsonRpcId? extractRpcId(string messageBody) {
-      try {
-         return JsonNode.Parse(messageBody)
-            ?.AsObject()
-            ?.GetValueOrDefault("id")
-            ?.Deserialize<JsonRpcId>();
-      } catch {
-         return null;
-      }
-   }
-
-   void onConnect(WebsocketConnectNotification notification) {
-      var id = notification.Id;
-      lock (lockObj) {
-         clients.Add(id, new());
-      }
-      Log.Api.info($"DataStreamWebsocketDataSync client connected. clientId: {notification.Id}");
-      if (notification.Message is not null) {
-         onIncomingMessage(notification.Message);
-      }
-   }
-
-   void onDisconnect(WebsocketDisconnectNotification notification) {
-      Log.Api.warn($"DataStreamWebsocketDataSync client disconnected. clientId: {notification.Id}");
-      unsubscribeAll(notification.Id);
-      clients.Remove(notification.Id);
-   }
-
-   void onIncomingMessage(WebsocketIncomingMessage message) {
-      var id = message.sender_client_id;
-      if (!JsonRpcRequest.TryDeserialize(message, out var req, out var err)) {
-         onBadRequest(message, err);
-         return;
-      }
-      switch (req.Method!.ToLower()) {
-         case "subscribe":
-            subscribe(id, req);
-            break;
-         case "unsubscribe":
-            unsubscribe(id, req);
-            break;
-         case "unsubscribe_all":
-            unsubscribeAll(id);
-            break;
-         default:
-            Log.Api.error($"Received request of unknown type: {req}");
-            break;
-      }
-   }
-
-   void sendNotification<T>(WebsocketClientID id, string method, T rpcParams) {
-      var rpcNotification = JsonRpcRequest.CreateNotification(method, rpcParams);
-      var rpcNotificationStr = JsonSerializer.Serialize(rpcNotification, serializerOpts);
-      send(new WebsocketOutgoingMessage[] { new(id, rpcNotificationStr) });
-   }
-
-   void sendResponse<T>(WebsocketClientID id, JsonRpcId rpcId, T rpcParams) {
-      var rpcRes = JsonRpcResponse.Create(rpcId, rpcParams);
-      var rpcResStr = JsonSerializer.Serialize(rpcRes, serializerOpts);
-      send(new WebsocketOutgoingMessage[] { new(id, rpcResStr )});
-   }
-
-   void sendError<T>(WebsocketClientID id, JsonRpcId rpcId, JsonRpcError<T> error) {
-      var rpcRes = JsonRpcResponse.CreateError(rpcId, error);
-      var rpcResStr = JsonSerializer.Serialize(rpcRes, serializerOpts);
-      send(new WebsocketOutgoingMessage[] { new(id, rpcResStr) });
-   }
-
-   bool send(IEnumerable<WebsocketOutgoingMessage> messages) {
-      lock (lockObj) {
-         if (requestSend is null) {
-            Log.Api.warn(
-               $"Unable to send message(s) because '{nameof(requestSend)}' handler is null");
-            return false;
-         }
-         requestSend(messages);
-         return true;
-      }
-   }
-
-   void subscribe(WebsocketClientID id, JsonRpcRequest<JsonObject?> req) {
-      var subParams = req.Params!.Deserialize<SubscriptionParams>()!;
-      var exp = DataStreamExpressionsInterpreter.Evaluate(subParams.Query!);
-      var stream = exp.getView(new Dictionary<string, StatementExpression>(0));
-      var sub = new ClientSubscription(id, subParams, Disposable.Create(dispose));
-      lock (lockObj) {
-         clients[id].Add(sub);
-         sendResponse(id, req.Id!.Value, req.Params);
-         stream.onChange += onChange;
-      }
-
-      void onChange(object? newValue) {
-         var dp = ExceptionAssert.of_type<DataPacket>(newValue);
-         var rpcParams = new {
-            addedRows = dp.addedRows,
-            deletedRows = dp.deletedRows
-         };
-         sendNotification(id, "subscription", rpcParams);
-      }
-
-      void dispose() =>
-         stream.onChange -= onChange;
-   }
-
-   void unsubscribe(WebsocketClientID id, JsonRpcRequest<JsonObject?> req) {
-      var subParams = req.Params!.Deserialize<SubscriptionParams>();
-      var subParamsStr = JsonSerializer.Serialize(subParams, serializerOpts);
-      // Minimize amount of code inside lock (just need it to find/remove the subscription)
-      List<ClientSubscription>? subs = null;
-      ClientSubscription? sub = null;
-      lock (lockObj) {
-         subs = clients.GetValueOrDefault(id);
-         subs?.TryRemoveFirst(s => s!.Params.Equals(subParams), out sub);
-      }
-      if (subs is null) {
-         // Didn't find client (log/throw).
-         var msg = $"Unable to unsubscribe: did not find client. Id: {id}.\nParams:\n{subParamsStr}";
-         Log.Api.error(msg);
-         throw new Exception(msg);
-      }
-      if (sub is null) {
-         // Found client, but didn't find subscription (log/throw).
-         var msg = $"Unable to unsubscribe: did not find the specified subscription. " +
-            $"Id: {id}. Request:\n{subParams}";
-         Log.Api.error(msg);
-         throw new Exception(msg);
-      }
-      // Found client and subscription (dispose/remove subscription).
-      Log.Api.info($"Unsubscribing client. Id: {id}.\nParams:\n{subParamsStr}");
-      sub.Dispose();
-   }
-
-   void unsubscribeAll(WebsocketClientID id) {
-      lock (lockObj) {
-         if (!clients.TryGetValue(id, out var subs)) {
-            throw new Exception(
-               $"Unable to unsubscribe: did not find client with id {id}");
-         }
-         subs.DisposeAll();
-         subs.Clear();
-      }
-   }
-
-   void onBadRequest(WebsocketIncomingMessage req, string? errorMsg) {
-      var rpcId = extractRpcId(req.message_body);
-      var logMsg = $"Recieved invalid request:\n{req.message_body}";
-      if (errorMsg is not null) {
-         logMsg += $"\n\n**Exception**\n{errorMsg}";
-      }
-      Log.Api.error(logMsg);
-      if (rpcId is null) {
-         return;
-      }
-      // Send error back to client
-      var rpcErr = JsonRpcError.Create(
-         (long)JsonRpcErrorCode.InvalidRequest,
-         $"Received invalid request: {errorMsg ?? "unknown error"}",
-         req.message_body);
-      sendError(req.sender_client_id, rpcId.Value, rpcErr);
-   }
-}
+namespace Bts.DataStream.Websocket;
+
+using BCG;
+
+using bts;
+using bts.api.core;
+using bts.collections;
+
+using Bts.DataStreamExpressions;
+using Bts.Websocket;
+
+using mbts.utils;
+
+using Spark.Core.Disposables;
+
+using System.Reactive.Subjects;
+using System.Text.Json;
+using System.Text.Json.Nodes;
+
+/// <summary>
+/// Represents an I/O model (<see cref="ISubject{TSource, TResult}"/>), that accepts websocket
+/// requests (from clients) as input, and emits data/notifications (to clients) as output.
+/// This class's chief role is managing data subscriptions to sparkd on the clients' behalf.
+/// </summary>
+public class DataSync(JsonSerializerOptions? serializerOpts = null) :
+   ISubject<WebsocketConnectionUpdate, IEnumerable<WebsocketOutgoingMessage>> {
+   protected readonly object lockObj = new();
+   readonly JsonSerializerOptions serializerOpts = serializerOpts ?? JsonSerializerOptions.Default;
+   readonly Dictionary<WebsocketClientID, List<ClientSubscription>> clients = [];
+
+   event Action<IEnumerable<WebsocketOutgoingMessage>>? requestSend;
+
+   public void OnCompleted() =>
+      Log.Api.info("DataSync incoming request stream has completed.");
+
+   public void OnError(Exception error) =>
+      Log.Api.error($"DataSync error: {error}");
+
+   public void OnNext(WebsocketConnectionUpdate value) {
+      switch (value.Type) {
+         case WebsocketConnectionUpdateType.IncomingMessage:
+            onIncomingMessage(value.IncomingMessage);
+            break;
+         case WebsocketConnectionUpdateType.Connect:
+            onConnect(value.Connect);
+            break;
+         case WebsocketConnectionUpdateType.Disconnect:
+            onDisconnect(value.Disconnect);
+            break;
+         default:
+            Log.assert(false, $"Unexpected WebsocketConnectionUpdate type: {value.Type}");
+            break;
+      }
+   }
+
+   public IDisposable Subscribe(IObserver<IEnumerable<WebsocketOutgoingMessage>> observer) {
+      lock (lockObj) {
+         requestSend += observer.OnNext;
+         return Disposable.Create(Dispose);
+      }
+      void Dispose() {
+         lock (lockObj) {
+            requestSend -= observer.OnNext;
+         }
+      }
+   }
+
+   static JsonRpcId? extractRpcId(string messageBody) {
+      try {
+         return JsonNode.Parse(messageBody)
+            ?.AsObject()
+            ?.GetValueOrDefault("id")
+            ?.Deserialize<JsonRpcId>();
+      } catch {
+         return null;
+      }
+   }
+
+   void onConnect(WebsocketConnectNotification notification) {
+      var id = notification.Id;
+      lock (lockObj) {
+         clients.Add(id, []);
+      }
+      Log.Api.info($"DataStreamWebsocketDataSync client connected. clientId: {notification.Id}");
+      if (notification.Message is not null) {
+         onIncomingMessage(notification.Message);
+      }
+   }
+
+   void onDisconnect(WebsocketDisconnectNotification notification) {
+      Log.Api.warn($"DataStreamWebsocketDataSync client disconnected. clientId: {notification.Id}");
+      unsubscribeAll(notification.Id);
+      clients.Remove(notification.Id);
+   }
+
+   void onIncomingMessage(WebsocketIncomingMessage message) {
+      var id = message.sender_client_id;
+      if (!JsonRpcRequest.TryDeserialize(message, out var req, out var err)) {
+         onBadRequest(message, err);
+         return;
+      }
+      switch (req.Method!.ToLower()) {
+         case "subscribe":
+            subscribe(id, req);
+            break;
+         case "unsubscribe":
+            unsubscribe(id, req);
+            break;
+         case "unsubscribe_all":
+            unsubscribeAll(id);
+            break;
+         default:
+            Log.Api.error($"Received request of unknown type: {req}");
+            break;
+      }
+   }
+
+   void sendNotification<T>(WebsocketClientID id, string method, T rpcParams) {
+      var rpcNotification = JsonRpcRequest.CreateNotification(method, rpcParams);
+      var rpcNotificationStr = JsonSerializer.Serialize(rpcNotification, serializerOpts);
+      send([new(id, rpcNotificationStr)]);
+   }
+
+   void sendResponse<T>(WebsocketClientID id, JsonRpcId rpcId, T rpcParams) {
+      var rpcRes = JsonRpcResponse.Create(rpcId, rpcParams);
+      var rpcResStr = JsonSerializer.Serialize(rpcRes, serializerOpts);
+      send([new(id, rpcResStr )]);
+   }
+
+   void sendError<T>(WebsocketClientID id, JsonRpcId rpcId, JsonRpcError<T> error) {
+      var rpcRes = JsonRpcResponse.CreateError(rpcId, error);
+      var rpcResStr = JsonSerializer.Serialize(rpcRes, serializerOpts);
+      send([new(id, rpcResStr)]);
+   }
+
+   bool send(IEnumerable<WebsocketOutgoingMessage> messages) {
+      lock (lockObj) {
+         if (requestSend is null) {
+            Log.Api.warn(
+               $"Unable to send message(s) because '{nameof(requestSend)}' handler is null");
+            return false;
+         }
+         requestSend(messages);
+         return true;
+      }
+   }
+
+   void subscribe(WebsocketClientID id, JsonRpcRequest<JsonObject?> req) {
+      var subParams = req.Params!.Deserialize<SubscriptionParams>()!;
+      var exp = DataStreamExpressionsInterpreter.Evaluate(subParams.Query!);
+      var stream = exp.getView(new Dictionary<string, StatementExpression>(0));
+      var sub = new ClientSubscription(id, subParams, Disposable.Create(Dispose));
+      lock (lockObj) {
+         clients[id].Add(sub);
+         sendResponse(id, req.Id!.Value, req.Params);
+         stream.onChange += OnChange;
+      }
+
+      void OnChange(object? newValue) {
+         var dp = ExceptionAssert.of_type<DataPacket>(newValue);
+         var rpcParams = new {
+            dp.addedRows,
+            dp.deletedRows
+         };
+         sendNotification(id, "subscription", rpcParams);
+      }
+
+      void Dispose() =>
+         stream.onChange -= OnChange;
+   }
+
+   void unsubscribe(WebsocketClientID id, JsonRpcRequest<JsonObject?> req) {
+      var subParams = req.Params!.Deserialize<SubscriptionParams>();
+      var subParamsStr = JsonSerializer.Serialize(subParams, serializerOpts);
+      // Minimize amount of code inside lock (just need it to find/remove the subscription)
+      List<ClientSubscription>? subs = null;
+      ClientSubscription? sub = null;
+      lock (lockObj) {
+         subs = clients.GetValueOrDefault(id);
+         subs?.TryRemoveFirst(s => s!.Params.Equals(subParams), out sub);
+      }
+      if (subs is null) {
+         // Didn't find client (log/throw).
+         var msg = $"Unable to unsubscribe: did not find client. Id: {id}.\nParams:\n{subParamsStr}";
+         Log.Api.error(msg);
+         throw new Exception(msg);
+      }
+      if (sub is null) {
+         // Found client, but didn't find subscription (log/throw).
+         var msg = $"Unable to unsubscribe: did not find the specified subscription. " +
+            $"Id: {id}. Request:\n{subParams}";
+         Log.Api.error(msg);
+         throw new Exception(msg);
+      }
+      // Found client and subscription (dispose/remove subscription).
+      Log.Api.info($"Unsubscribing client. Id: {id}.\nParams:\n{subParamsStr}");
+      sub.Dispose();
+   }
+
+   void unsubscribeAll(WebsocketClientID id) {
+      lock (lockObj) {
+         if (!clients.TryGetValue(id, out var subs)) {
+            throw new Exception(
+               $"Unable to unsubscribe: did not find client with id {id}");
+         }
+         subs.DisposeAll();
+         subs.Clear();
+      }
+   }
+
+   void onBadRequest(WebsocketIncomingMessage req, string? errorMsg) {
+      var rpcId = extractRpcId(req.message_body);
+      var logMsg = $"Recieved invalid request:\n{req.message_body}";
+      if (errorMsg is not null) {
+         logMsg += $"\n\n**Exception**\n{errorMsg}";
+      }
+      Log.Api.error(logMsg);
+      if (rpcId is null) {
+         return;
+      }
+      // Send error back to client
+      var rpcErr = JsonRpcError.Create(
+         (long)JsonRpcErrorCode.InvalidRequest,
+         $"Received invalid request: {errorMsg ?? "unknown error"}",
+         req.message_body);
+      sendError(req.sender_client_id, rpcId.Value, rpcErr);
+   }
+}

src/DataStreamSuite/Bts.DataStream.Websocket/MultilineStringJsonConverter.cs

@@ -1,48 +1,48 @@
-namespace Bts.DataStream.Websocket;
-
-using Spark.Core;
-
-using System;
-using System.Text.Json;
-using System.Text.Json.Serialization;
-
-class MultilineStringJsonConverter : JsonConverter<string> {
-   public override string? Read(ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options) {
-      if (typeToConvert != typeof(string)) {
-         throw new ArgumentException($"Unable to parse {typeToConvert}", nameof(typeToConvert));
-      }
-      return reader.TokenType switch {
-         JsonTokenType.String => stringImpl(ref reader),
-         JsonTokenType.StartArray => arrayImpl(ref reader),
-         _ => throw new ArgumentException(
-            $"Unable to parse {typeof(SubscriptionParams)} from {reader.TokenType}"),
-      };
-
-      static string? stringImpl(ref Utf8JsonReader reader) =>
-         new(reader.GetString()!);
-
-      static string? arrayImpl(ref Utf8JsonReader reader) {
-         var lines = new List<string>(2);
-         while (reader.Read()) {
-            switch (reader.TokenType) {
-               case JsonTokenType.String:
-                  lines.Add(reader.GetString()!);
-                  break;
-               case JsonTokenType.EndArray:
-                  return string.Join("\n", lines);
-            }
-         }
-         throw new JsonException(
-            $"Failed to parse {typeof(SubscriptionParams)} (did not reach end of array)");
-      }
-   }
-
-   public override void Write(Utf8JsonWriter writer, string value, JsonSerializerOptions options) {
-      var lines = value.Split("\n");
-      if (lines.Length < 2) {
-         writer.WriteRawValue(value);
-         return;
-      }
-      writer.WriteRawValue($"[{", ".join(lines)}]");
-   }
-}
+namespace Bts.DataStream.Websocket;
+
+using Spark.Core;
+
+using System;
+using System.Text.Json;
+using System.Text.Json.Serialization;
+
+class MultilineStringJsonConverter : JsonConverter<string> {
+   public override string? Read(ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options) {
+      if (typeToConvert != typeof(string)) {
+         throw new ArgumentException($"Unable to parse {typeToConvert}", nameof(typeToConvert));
+      }
+      return reader.TokenType switch {
+         JsonTokenType.String => StringImpl(ref reader),
+         JsonTokenType.StartArray => ArrayImpl(ref reader),
+         _ => throw new ArgumentException(
+            $"Unable to parse {typeof(SubscriptionParams)} from {reader.TokenType}"),
+      };
+
+      static string? StringImpl(ref Utf8JsonReader reader) =>
+         new(reader.GetString()!);
+
+      static string? ArrayImpl(ref Utf8JsonReader reader) {
+         var lines = new List<string>(2);
+         while (reader.Read()) {
+            switch (reader.TokenType) {
+               case JsonTokenType.String:
+                  lines.Add(reader.GetString()!);
+                  break;
+               case JsonTokenType.EndArray:
+                  return string.Join("\n", lines);
+            }
+         }
+         throw new JsonException(
+            $"Failed to parse {typeof(SubscriptionParams)} (did not reach end of array)");
+      }
+   }
+
+   public override void Write(Utf8JsonWriter writer, string value, JsonSerializerOptions options) {
+      var lines = value.Split("\n");
+      if (lines.Length < 2) {
+         writer.WriteRawValue(value);
+         return;
+      }
+      writer.WriteRawValue($"[{", ".join(lines)}]");
+   }
+}

src/DataStreamSuite/Bts.DataStream.Websocket/Session.cs

@@ -1,27 +1,31 @@
-namespace Bts.DataStream.Websocket;
-
-using bts;
-using bts.api.core;
-
-using Bts.Websocket;
-
-using Google.Protobuf;
-
-using System.Text.Json;
-
-public static class Session {
-   static int sessionIdCounter;
-
-   public static async Task Start(
-      ServiceContext context, WebsocketServerConfig config, JsonSerializerOptions? opts) {
-      opts ??= JsonSerializerOptions.Default;
-      var sessionId = Interlocked.Increment(ref sessionIdCounter);
-      var gateway = new WebsocketGatewayObservable(config);
-      var sync = new DataSync(context, opts);
-      using var syncSub = gateway.Subscribe(sync);
-      using var gwSub = sync.Subscribe(gateway);
-      Log.Api.info($"Starting DataStream Websocket Session {sessionId}");
-      Log.Api.info($"DataStream Websocket Session {sessionId} config:\n{JsonFormatter.Default.Format(config)}");
-      await gateway.ConnectAsync();
-   }
-}
+namespace Bts.DataStream.Websocket;
+
+using bts;
+using bts.api.core;
+
+using Bts.Websocket;
+
+using Google.Protobuf;
+
+using Spark.Core.Disposables;
+
+
+using System.Text.Json;
+
+public static class Session {
+   static int sessionIdCounter;
+
+   public static async Task Start(
+      ServiceContext context, WebsocketServerConfig config, JsonSerializerOptions? opts) {
+      opts ??= JsonSerializerOptions.Default;
+      var sessionId = Interlocked.Increment(ref sessionIdCounter);
+      var gateway = new WebsocketGatewayObservable(config);
+      var sync = new DataSync(context, opts);
+      using (gateway.Subscribe(sync))
+      using (sync.Subscribe(gateway)) {
+         Log.Api.info($"Starting DataStream Websocket Session {sessionId}");
+         Log.Api.info($"DataStream Websocket Session {sessionId} config:\n{JsonFormatter.Default.Format(config)}");
+         await gateway.ConnectAsync();
+      }
+   }
+}

src/DataStreamSuite/Bts.DataStream.Websocket/SubscriptionParams.cs

@@ -1,110 +1,110 @@
-namespace Bts.DataStream.Websocket;
-
-using Spark.Core;
-
-using System;
-using System.Diagnostics.CodeAnalysis;
-using System.Text.Json;
-using System.Text.Json.Serialization;
-
-public class SubscriptionParams : IEquatable<SubscriptionParams> {
-   static class PropNames {
-      public const string QUERY = "query";
-   }
-
-   public SubscriptionParams(string? query) {
-      Query = query;
-   }
-
-   // Serializer supports plain string, or an array of lines for convenience
-   [MaybeNull]
-   [JsonPropertyName(PropNames.QUERY)]
-   [JsonConverter(typeof(MultilineStringJsonConverter))]
-   public string? Query { get; set; }
-
-   public bool Equals(SubscriptionParams? other) =>
-      other is not null &&
-      Query == other.Query;
-
-   class SubscriptionParamsConverter : JsonConverter<SubscriptionParams> {
-      public override bool CanConvert(Type typeToConvert) =>
-         typeToConvert == typeof(SubscriptionParams);
-
-      public override SubscriptionParams Read(
-         ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options) {
-         if (typeToConvert != typeof(SubscriptionParams)) {
-            throw new ArgumentException($"Unable to parse {typeToConvert}", nameof(typeToConvert));
-         }
-         return read(ref reader);
-      }
-
-      SubscriptionParams read(ref Utf8JsonReader reader) {
-         return reader.TokenType switch {
-            JsonTokenType.String => stringImpl(ref reader),
-            JsonTokenType.StartArray => arrayImpl(ref reader),
-            JsonTokenType.StartObject => objectImpl(ref reader),
-            _ => throw new ArgumentException(
-               $"Unable to parse {typeof(SubscriptionParams)} from {reader.TokenType}"),
-         };
-
-         static SubscriptionParams stringImpl(ref Utf8JsonReader reader) =>
-            new(reader.GetString()!);
-
-         static SubscriptionParams arrayImpl(ref Utf8JsonReader reader) {
-            var lines = new List<string>(2);
-            while (reader.Read()) {
-               switch (reader.TokenType) {
-                  case JsonTokenType.String:
-                     lines.Add(reader.GetString()!);
-                     break;
-                  case JsonTokenType.EndArray:
-                     return new("\n".join(lines));
-               }
-            }
-            throw new JsonException(
-               $"Failed to parse {typeof(SubscriptionParams)} (did not reach end of array)");
-         }
-
-         static SubscriptionParams objectImpl(ref Utf8JsonReader reader) {
-            var lines = new List<string>(2);
-            while (reader.Read()) {
-               switch (reader.TokenType) {
-                  case JsonTokenType.PropertyName:
-                     var propName = reader.GetString()!;
-                     if (propName == PropNames.QUERY) {
-                        switch (reader.TokenType) {
-                           case JsonTokenType.StartArray:
-                              return arrayImpl(ref reader);
-                           case JsonTokenType.String:
-                              return stringImpl(ref reader);
-                        }
-                     }
-                     break;
-                  case JsonTokenType.String:
-                     lines.Add(reader.GetString()!);
-                     break;
-                  case JsonTokenType.EndArray:
-                     return new("\n".join(lines));
-               }
-            }
-            throw new JsonException(
-               $"Failed to parse {typeof(SubscriptionParams)} (did not reach end of array)");
-         }
-      }
-
-      public override void Write(
-         Utf8JsonWriter writer, SubscriptionParams value, JsonSerializerOptions options) {
-         var q = value.Query;
-         var lines = q?.Split("\n");
-         if (lines is null || lines.Length < 2) {
-            writer.WriteString(PropNames.QUERY, value.Query);
-            return;
-         }
-         writer.WriteStartArray(PropNames.QUERY);
-         foreach (var l in lines) {
-            writer.WriteStringValue(l);
-         }
-         writer.WriteEndArray();
-      }
-   }
-}
+namespace Bts.DataStream.Websocket;
+
+using Spark.Core;
+
+using System;
+using System.Diagnostics.CodeAnalysis;
+using System.Text.Json;
+using System.Text.Json.Serialization;
+
+public class SubscriptionParams : IEquatable<SubscriptionParams> {
+   static class PropNames {
+      public const string QUERY = "query";
+   }
+
+   public SubscriptionParams(string? query) {
+      Query = query;
+   }
+
+   // Serializer supports plain string, or an array of lines for convenience
+   [MaybeNull]
+   [JsonPropertyName(PropNames.QUERY)]
+   [JsonConverter(typeof(MultilineStringJsonConverter))]
+   public string? Query { get; set; }
+
+   public bool Equals(SubscriptionParams? other) =>
+      other is not null &&
+      Query == other.Query;
+
+   class SubscriptionParamsConverter : JsonConverter<SubscriptionParams> {
+      public override bool CanConvert(Type typeToConvert) =>
+         typeToConvert == typeof(SubscriptionParams);
+
+      public override SubscriptionParams Read(
+         ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options) {
+         if (typeToConvert != typeof(SubscriptionParams)) {
+            throw new ArgumentException($"Unable to parse {typeToConvert}", nameof(typeToConvert));
+         }
+         return read(ref reader);
+      }
+
+      SubscriptionParams read(ref Utf8JsonReader reader) {
+         return reader.TokenType switch {
+            JsonTokenType.String => stringImpl(ref reader),
+            JsonTokenType.StartArray => arrayImpl(ref reader),
+            JsonTokenType.StartObject => objectImpl(ref reader),
+            _ => throw new ArgumentException(
+               $"Unable to parse {typeof(SubscriptionParams)} from {reader.TokenType}"),
+         };
+
+         static SubscriptionParams stringImpl(ref Utf8JsonReader reader) =>
+            new(reader.GetString()!);
+
+         static SubscriptionParams arrayImpl(ref Utf8JsonReader reader) {
+            var lines = new List<string>(2);
+            while (reader.Read()) {
+               switch (reader.TokenType) {
+                  case JsonTokenType.String:
+                     lines.Add(reader.GetString()!);
+                     break;
+                  case JsonTokenType.EndArray:
+                     return new("\n".join(lines));
+               }
+            }
+            throw new JsonException(
+               $"Failed to parse {typeof(SubscriptionParams)} (did not reach end of array)");
+         }
+
+         static SubscriptionParams objectImpl(ref Utf8JsonReader reader) {
+            var lines = new List<string>(2);
+            while (reader.Read()) {
+               switch (reader.TokenType) {
+                  case JsonTokenType.PropertyName:
+                     var propName = reader.GetString()!;
+                     if (propName == PropNames.QUERY) {
+                        switch (reader.TokenType) {
+                           case JsonTokenType.StartArray:
+                              return arrayImpl(ref reader);
+                           case JsonTokenType.String:
+                              return stringImpl(ref reader);
+                        }
+                     }
+                     break;
+                  case JsonTokenType.String:
+                     lines.Add(reader.GetString()!);
+                     break;
+                  case JsonTokenType.EndArray:
+                     return new("\n".join(lines));
+               }
+            }
+            throw new JsonException(
+               $"Failed to parse {typeof(SubscriptionParams)} (did not reach end of array)");
+         }
+      }
+
+      public override void Write(
+         Utf8JsonWriter writer, SubscriptionParams value, JsonSerializerOptions options) {
+         var q = value.Query;
+         var lines = q?.Split("\n");
+         if (lines is null || lines.Length < 2) {
+            writer.WriteString(PropNames.QUERY, value.Query);
+            return;
+         }
+         writer.WriteStartArray(PropNames.QUERY);
+         foreach (var l in lines) {
+            writer.WriteStringValue(l);
+         }
+         writer.WriteEndArray();
+      }
+   }
+}

src/DataStreamSuite/Bts.DataStream.Websocket/ViewStreamUpdate.cs

@@ -0,0 +1,41 @@
+#nullable enable
+using BCG;
+
+using Spark.Core.Json;
+
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text.Json;
+using System.Text.Json.Serialization;
+
+namespace Bts.DataStreamCore;
+
+class ViewStreamUpdate {
+   [JsonPropertyName("tableName")]
+   public string? TableName { get; set; }
+
+   [JsonPropertyName("columns")]
+   public List<ColumnInfo>? Columns { get; set; }
+
+   [JsonPropertyName("addedRows")]
+   public List<Dictionary<string, object?>>? AddedRows { get; set; }
+
+   [JsonPropertyName("deletedRows")]
+   public List<Dictionary<string, object?>>? DeletedRows { get; set; }
+
+   public DataPacket ToDataPacket(BlueTable state, bool applyChanges = true) {
+      var schema = new BlueTableSchema([.. Columns!], TableName);
+      var addedRowsTable = new BlueTable(schema);
+      var deletedRowsTable = new BlueTable(schema);
+      if (applyChanges) {
+         state.deleteRows(deletedRowsTable);
+         state.addRows(addedRowsTable);
+      }
+      return new(state, addedRowsTable, deletedRowsTable);
+   }
+
+   public static string ToJson() {
+      return "";
+   }
+}

src/DataStreamSuite/Bts.DataStream.WebsocketClient/Bts.DataStream.WebsocketClient.csproj

@@ -1,7 +1,7 @@
 <Project Sdk="Microsoft.NET.Sdk.Web">
 
   <PropertyGroup>
-    <TargetFramework>net7.0</TargetFramework>
+    <TargetFramework>net8.0</TargetFramework>
     <Nullable>enable</Nullable>
     <ImplicitUsings>enable</ImplicitUsings>
     <EnableDefaultContentItems>True</EnableDefaultContentItems>

src/DataStreamSuite/Bts.DataStream.WebsocketServer/Bts.DataStream.WebsocketServer.csproj

@@ -2,7 +2,7 @@
 
   <PropertyGroup>
     <OutputType>Exe</OutputType>
-    <TargetFramework>net7.0</TargetFramework>
+    <TargetFramework>net8.0</TargetFramework>
     <ImplicitUsings>enable</ImplicitUsings>
     <Nullable>enable</Nullable>
   </PropertyGroup>

src/DataStreamSuite/Bts.DataStreamWebApp.Client/package-lock.json

@@ -9,7 +9,8 @@
       "version": "0.0.0",
       "dependencies": {
         "react": "^18.2.0",
-        "react-dom": "^18.2.0"
+        "react-dom": "^18.2.0",
+        "react-use-websocket": "^4.5.0"
       },
       "devDependencies": {
         "@types/react": "^18.2.43",
@@ -3325,6 +3326,15 @@
         "node": ">=0.10.0"
       }
     },
+    "node_modules/react-use-websocket": {
+      "version": "4.5.0",
+      "resolved": "https://registry.npmjs.org/react-use-websocket/-/react-use-websocket-4.5.0.tgz",
+      "integrity": "sha512-oxYVLWM3Lv0InCfjW7hG/Hk0hkE0P1SiLd5/I3d5x0W4riAnDUkD4VEu7qNVAqxNjBF3nU7k0jLMOetLXpwfsA==",
+      "peerDependencies": {
+        "react": ">= 18.0.0",
+        "react-dom": ">= 18.0.0"
+      }
+    },
     "node_modules/reflect.getprototypeof": {
       "version": "1.0.4",
       "resolved": "https://registry.npmjs.org/reflect.getprototypeof/-/reflect.getprototypeof-1.0.4.tgz",

src/DataStreamSuite/Bts.DataStreamWebApp.Client/package.json

@@ -11,7 +11,8 @@
   },
   "dependencies": {
     "react": "^18.2.0",
-    "react-dom": "^18.2.0"
+    "react-dom": "^18.2.0",
+    "react-use-websocket": "^4.5.0"
   },
   "devDependencies": {
     "@types/react": "^18.2.43",

src/DataStreamSuite/Bts.DataStreamWebApp.Client/src/App.css

@@ -21,3 +21,24 @@ th, td {
     background-color: var(--cell-background-color);
     color: var(--cell-color);
 }
+
+textarea#query-input {
+  height: 5em;
+  width: 50em;
+}
+
+button#query-button {
+  height: 5em;
+  vertical-align: top;
+}
+
+table#query-output {
+  float: none;
+  clear: both;
+}
+
+p#footer {
+  float: none;
+  clear: both;
+  vertical-align: bottom;
+}

src/DataStreamSuite/Bts.DataStreamWebApp.Client/src/App.jsx

@@ -1,75 +1,28 @@
-import { useEffect, useState, Component } from 'react';
-import './App.css';
-
-function App() {
-    const [forecasts, setForecasts] = useState();
-
-    useEffect(() => {
-        populateWeatherData();
-    }, []);
-
-    const contents = forecasts === undefined
-        ? <p><em>Loading... Please refresh once the ASP.NET backend has started. See <a href="https://aka.ms/jspsintegrationreact">https://aka.ms/jspsintegrationreact</a> for more details.</em></p>
-        : <table className="table table-striped" aria-labelledby="tabelLabel">
-            <thead>
-                <tr>
-                    <th>Date</th>
-                    <th>Temp. (C)</th>
-                    <th>Temp. (F)</th>
-                    <th>Summary</th>
-                </tr>
-            </thead>
-            <tbody>
-                {forecasts.map(forecast =>
-                    <tr key={forecast.date}>
-                        <td>{forecast.date}</td>
-                        <td>{forecast.temperatureC}</td>
-                        <td>{forecast.temperatureF}</td>
-                        <td>{forecast.summary}</td>
-                    </tr>
-                )}
-            </tbody>
-        </table>;
-
-    return (
-        <div>
-            <h1 id="tabelLabel">Weather forecast</h1>
-            <p>This component demonstrates fetching data from the server.</p>
-            {contents}
-        </div>
-    );
-    
-    async function populateWeatherData() {
-        const response = await fetch('weatherforecast');
-        const data = await response.json();
-        setForecasts(data);
-    }
-}
-
-export default App;
+import { useEffect, useState, Component } from "react";
+import { useWebSocket } from "react-use-websocket";
+import "./App.css";
 
 export class Home extends Component {
   static displayName = Home.name;
   // static sparkdHost = '192.168.50.94';
-  static sparkdHost = 'localhost';
+  static sparkdHost = "localhost";
 
   // static websocket_url = 'wss://markets.api.uat.cmegroup.com/marketdatastream/v1';
   // static access_token = 'eyJhbGciOiJSUzI1NiIsImtpZCI6IjF0al9MTVFrT2pGVzJWQTZvdkZzOEZHUWViMCIsInBpLmF0bSI6IjEifQ.eyJzY29wZSI6W10sImNsaWVudF9pZCI6ImFwaV90aGlua2FscGhhMzIxIiwiYXVkIjoiTnIiLCJqdGkiOiIweERtb3NnU2NHcE5GUEFaIiwic3ViIjoiYXBpX3RoaW5rYWxwaGEzMjEiLCJleHAiOjE2OTgzNTY5OTJ9.uJJEhWpujGiyJ5_lypiIPB8xR0GkuXqydtCugp5xD0hyTuX6VkQdNoY_oaRXXGRCUPCvfsXpU-HfL7pRkcXCjMLTrVIqf3RjD4_gar_ExbIb180nbYMG87E4rOILi5bIin51KOcMNBdHW7pTKwsS221XGyIAKIj65WxD-K8QSVr1HN6s7KDfYdez5dstWWC6xGg_3IZkrJjx8Df1HNT73K9PwUaAl7gcK4_2RSr8L636QbF3_y4Tv26g5IEv90Mc1mnAFPvUGI-nDR4s5Vso0398DlYl_tw-zbQQmn-QuQ6rS_ZIcZKgyoHKwD-3lgV-J1_--ndHdh8qwJC7Hk8ZKg'
 
   render() {
     let socket = null;
-
+    const wsUrl = `ws://localhost:5268/ws`;
     function connectToSparkd() {
-      const wsUrl = `ws://${Home.sparkdHost}:8080`;
       // const wsUrl = Home.websocket_url;
       if (socket == null) {
-        socket = new WebSocket(wsUrl)
+        socket = new WebSocket(wsUrl);
       }
       if (socket && socket.readyState === WebSocket.OPEN) {
         return;
       }
 
-      const output = document.getElementById('query-output');
+      const output = document.getElementById("query-output");
       socket.onopen = (event) => {
         console.log(`Connected to ${wsUrl}`);
       };
@@ -99,31 +52,16 @@ export class Home extends Component {
         connectToSparkd();
       }
       if (socket == null || !socket) {
-        console.log('Failed to connect to websocket server');
+        console.log("Failed to connect to websocket server");
         return;
       }
       if (socket.readyState === WebSocket.OPEN) {
-        const queryInput = document.getElementById('query-input');
-        // const req = {
-        //   header: {
-        //     messageType: "subscribe",
-        //     sentTime: "",
-        //     version: "",
-        //     requestId: "1",
-        //     token: ""
-        //   },
-        //   payload: {
-        //     subscriptionMessageTypes: ["TRD"],
-        //     subscriptions: [{
-        //       exchangeMic: "XCME", productType: "FUT", productCode: "BTC"
-        //     }]
-        //   }
-        // };
+        const queryInput = document.getElementById("query-input");
         const rpcReq = {
-          jsonrpc: '2.0',
-          method: 'subscribe',
+          jsonrpc: "2.0",
+          method: "subscribe",
           params: {
-            query: queryInput.value
+            query: queryInput.value,
           },
           id: 1,
         };
@@ -136,19 +74,19 @@ export class Home extends Component {
         <h1>Data Stream Query Analyzer</h1>
         <p>Run queries against the BTS Cloud</p>
         <div>
-          <textarea id="query-input" placeholder='Enter Query'></textarea>
+          <textarea id="query-input" placeholder="Enter Query"></textarea>
+          <button id="query-button" onClick={executeQuery}>
+            Execute
+          </button>
         </div>
-        <div>
-          <button id="query-button" onClick={executeQuery}>Execute</button>
-        </div>
-        <table id="query-output">
-        </table>
-        <ul>
-          <li><strong>Client-side navigation</strong>. For example, click <em>Counter</em> then <em>Back</em> to return here.</li>
-          <li><strong>Development server integration</strong>. In development mode, the development server from <code>create-react-app</code> runs in the background automatically, so your client-side resources are dynamically built on demand and the page refreshes when you modify any file.</li>
-          <li><strong>Efficient production builds</strong>. In production mode, development-time features are disabled, and your <code>dotnet publish</code> configuration produces minified, efficiently bundled JavaScript files.</li>
-        </ul>
-        <p>The <code>ClientApp</code> subdirectory is a standard React application based on the <code>create-react-app</code> template. If you open a command prompt in that directory, you can run <code>npm</code> commands such as <code>npm test</code> or <code>npm install</code>.</p>
+        <table id="query-output"></table>
+        <p id="footer">
+          The <code>ClientApp</code> subdirectory is a standard React
+          application based on the <code>create-react-app</code> template. If
+          you open a command prompt in that directory, you can run{" "}
+          <code>npm</code> commands such as <code>npm test</code> or{" "}
+          <code>npm install</code>.
+        </p>
       </div>
     );
   }

src/DataStreamSuite/Bts.DataStreamWebApp.Client/src/main.jsx

@@ -1,10 +1,10 @@
 import React from 'react'
 import ReactDOM from 'react-dom/client'
-import App from './App.jsx'
+import {Home} from './App.jsx'
 import './index.css'
 
 ReactDOM.createRoot(document.getElementById('root')).render(
   <React.StrictMode>
-    <App />
+    <Home />
   </React.StrictMode>,
 )

src/DataStreamSuite/Bts.DataStreamWebApp.Server/Bts.DataStreamWebApp.Server.csproj

@@ -11,9 +11,11 @@
   </PropertyGroup>
 
   <ItemGroup>
+    <PackageReference Include="CommandLineParser" Version="2.9.1" />
     <PackageReference Include="Microsoft.AspNetCore.SpaProxy">
       <Version>8.*-*</Version>
     </PackageReference>
+    <PackageReference Include="Newtonsoft.Json" Version="13.0.3" />
     <PackageReference Include="Swashbuckle.AspNetCore" Version="6.4.0" />
   </ItemGroup>
 
@@ -23,6 +25,11 @@
     </ProjectReference>
     <ProjectReference Include="..\DataStream\DataStream.csproj" />
     <ProjectReference Include="..\SharedStreams\SharedStreams.csproj" />
+    <ProjectReference Include="..\..\client\spark_api\spark_api.csproj" />
+    <ProjectReference Include="..\..\client\spark_websocket\spark_websocket.csproj" />
   </ItemGroup>
 
+  <!-- <ItemGroup>
+    <Content Update="wwwroot\alt-index.html" CopyToPublishDirectory="Never" />
+  </ItemGroup> -->
 </Project>

src/DataStreamSuite/Bts.DataStreamWebApp.Server/Controllers/DataStreamSubscriptionController.cs

@@ -1,25 +1,117 @@
 namespace Bts.DataStreamWebApp.Server.Controllers;
+
+using BCG;
+
+using bts.api.core;
+
+using Bts.DataStreamExpressions;
+
 using Microsoft.AspNetCore.Mvc;
 
-[ApiController]
-[Route("[controller]")]
-public class DataStreamSubscriptionController : ControllerBase {
-   private static readonly string[] Summaries = new[]
-   {
-        "Freezing", "Bracing", "Chilly", "Cool", "Mild", "Warm", "Balmy", "Hot", "Sweltering", "Scorching"
-    };
+using Spark.Core.Disposables;
 
-   private readonly ILogger<DataStreamSubscriptionController> _logger;
+using System.Net.WebSockets;
+using System.Text;
+using System.Text.Json;
 
-   public DataStreamSubscriptionController(ILogger<DataStreamSubscriptionController> logger) {
-      _logger = logger;
+// [Route("[controller]")]
+[Route("/ws")]
+public class DataStreamSubscriptionController(ILogger<DataStreamSubscriptionController> logger) : ControllerBase {
+   readonly ILogger<DataStreamSubscriptionController> _logger = logger;
+   IDisposable? streamSubscription;
+
+   [Route("/ws")]
+   public async Task Get() {
+      if (!HttpContext.WebSockets.IsWebSocketRequest) {
+         _logger.LogError("Expected websocket request on path '{}'", HttpContext.Request.Path);
+         return;
+      }
+      Console.WriteLine("Establishing websocket connection...");
+      using var webSocket = await HttpContext.WebSockets.AcceptWebSocketAsync();
+      Console.WriteLine($"Websocket connection established.");
+      await runWebSocketServer(null, "bts", string.Empty, webSocket);
    }
 
-   [HttpGet(Name = "GetDataStreamSubscription")]
-   public IEnumerable<DataStreamSubscription> Get() {
-      return Enumerable.Range(1, 5).Select(index => new DataStreamSubscription {
-         Query = "fake queronious"
-      })
-      .ToArray();
+   public static async Task Echo(WebSocket webSocket) {
+      var buffer = new byte[1024 * 4];
+      var receiveResult = await webSocket.ReceiveAsync(
+          new ArraySegment<byte>(buffer), CancellationToken.None);
+
+      while (!receiveResult.CloseStatus.HasValue) {
+         await webSocket.SendAsync(
+             new ArraySegment<byte>(buffer, 0, receiveResult.Count),
+             receiveResult.MessageType,
+             receiveResult.EndOfMessage,
+             CancellationToken.None);
+
+         receiveResult = await webSocket.ReceiveAsync(
+             new ArraySegment<byte>(buffer), CancellationToken.None);
+      }
+
+      await webSocket.CloseAsync(
+          receiveResult.CloseStatus.Value,
+          receiveResult.CloseStatusDescription,
+          CancellationToken.None);
+   }
+
+   async Task<int> runWebSocketServer(string? sparkd, string user, string password, WebSocket webSocket) {
+      var context = ServiceContext.create(sparkd);
+      await context.login(user, password);
+      var buffer = new byte[1024 * 4];
+      var receiveResult = await webSocket.ReceiveAsync(
+          new ArraySegment<byte>(buffer), CancellationToken.None);
+
+      try {
+         while (!receiveResult.CloseStatus.HasValue) {
+            receiveResult = await webSocket.ReceiveAsync(
+                new ArraySegment<byte>(buffer), CancellationToken.None);
+            var msg = Encoding.UTF8.GetString(buffer, 0, receiveResult.Count);
+            // var queryArgs = new QueryArguments { Query = msg };
+            var rpcReq = JsonSerializer.Deserialize<Bts.Websocket.JsonRpcRequest<DataStreamSubscription>>(msg);
+            var queryArgs = rpcReq!.Params;
+            // if (queryArgs is null) {
+            //    Console.WriteLine("Invalid queryArgs.");
+            //    continue;
+            // }
+            var exp = DataStreamExpressionsInterpreter.Evaluate(queryArgs.Query!);
+            if (exp is null) {
+               Console.WriteLine($"Invalid query expression: {queryArgs.Query}.");
+               continue;
+            }
+            var vs = exp.getView(new Dictionary<string, StatementExpression>(0));
+            Disposable.dispose(ref streamSubscription);
+            vs.onChange += onStreamChanged;
+            streamSubscription = Disposable.Create(() => vs.onChange -= onStreamChanged);
+            // await webSocket.SendAsync(
+            //     new ArraySegment<byte>(buffer, 0, receiveResult.Count),
+            //     receiveResult.MessageType,
+            //     receiveResult.EndOfMessage,
+            //     CancellationToken.None);
+         }
+      } catch (Exception ex) {
+         Console.WriteLine($"Exception on WebsocketServer thread: {ex}");
+         return 1;
+      }
+
+      return 0;
+
+      async void onStreamChanged(object newValue) {
+         if (newValue is not DataPacket dp) {
+            Console.WriteLine(
+               $"OnStreamChanged delegate expected DataPacket (actually received {newValue?.GetType()}.");
+            return;
+         }
+         Console.WriteLine("Received stream update.");
+         var added_rows_str = dp.addedRows.get_printed_rows(val => val.Length);
+         var msg = $"Received new rows:\n{added_rows_str}";
+         // var payload = new ReadOnlyMemory<byte>([.. msg]);
+         var encoded = Encoding.UTF8.GetBytes(msg);
+         var payload = new ArraySegment<byte>(encoded, 0, encoded.Length);
+         await webSocket.SendAsync(
+             payload,
+             receiveResult.MessageType,
+             receiveResult.EndOfMessage,
+             CancellationToken.None);
+      }
    }
 }

src/DataStreamSuite/Bts.DataStreamWebApp.Server/Controllers/WeatherForecastController.cs

@@ -3,25 +3,16 @@ using Microsoft.AspNetCore.Mvc;
 
 [ApiController]
 [Route("[controller]")]
-public class WeatherForecastController : ControllerBase
-{
-   private static readonly string[] Summaries = new[]
-   {
-        "Freezing", "Bracing", "Chilly", "Cool", "Mild", "Warm", "Balmy", "Hot", "Sweltering", "Scorching"
-    };
+public class WeatherForecastController(ILogger<WeatherForecastController> logger) : ControllerBase {
+   static readonly string[] Summaries = [
+      "Freezing", "Bracing", "Chilly", "Cool", "Mild", "Warm", "Balmy", "Hot", "Sweltering", "Scorching"
+   ];
 
-   private readonly ILogger<WeatherForecastController> _logger;
-
-   public WeatherForecastController(ILogger<WeatherForecastController> logger)
-   {
-      _logger = logger;
-   }
+   readonly ILogger<WeatherForecastController> _logger = logger;
 
    [HttpGet(Name = "GetWeatherForecast")]
-   public IEnumerable<WeatherForecast> Get()
-   {
-      return Enumerable.Range(1, 5).Select(index => new WeatherForecast
-      {
+   public IEnumerable<WeatherForecast> Get() {
+      return Enumerable.Range(1, 5).Select(index => new WeatherForecast {
          Date = DateOnly.FromDateTime(DateTime.Now.AddDays(index)),
          TemperatureC = Random.Shared.Next(-20, 55),
          Summary = Summaries[Random.Shared.Next(Summaries.Length)]

src/DataStreamSuite/Bts.DataStreamWebApp.Server/DataStreamSubscription.cs

@@ -1,5 +1,8 @@
 namespace Bts.DataStreamWebApp.Server;
 
+using System.Text.Json.Serialization;
+
 public class DataStreamSubscription {
+   [JsonPropertyName("query")]
    public string? Query { get; set; }
 }

src/DataStreamSuite/Bts.DataStreamWebApp.Server/DataStreamUpdate.cs

@@ -0,0 +1,8 @@
+namespace Bts.DataStreamWebApp.Server;
+
+using BCG;
+
+class DataStreamUpdate {
+   public List<IBlueRow> addedRows { get; set; } = new(0);
+   public List<IBlueRow> deletedRows { get; set; } = new(0);
+}

src/DataStreamSuite/Bts.DataStreamWebApp.Server/DataStreamWebAppServerOptions.cs

@@ -0,0 +1,14 @@
+namespace Bts.DataStreamWebApp.Server;
+
+using CommandLine;
+
+class DataStreamWebAppServerOptions {
+   [Option("sparkd")]
+   public string? sparkd { get; set; }
+
+   [Option("user", Default = "bts")]
+   public string? user { get; set; }
+
+   [Option("password", Default = "")]
+   public string? password { get; set; }
+}

src/DataStreamSuite/Bts.DataStreamWebApp.Server/DataStreamWebsocketHandler.cs

@@ -0,0 +1,58 @@
+using Microsoft.AspNetCore.Http;
+using Microsoft.Extensions.Logging;
+
+using System;
+using System.Net.WebSockets;
+using System.Threading;
+using System.Threading.Tasks;
+
+namespace Bts.DataStreamWebApp.Server {
+   // public class DataStreamWebSocketHandler {
+   //    readonly WebSocketConnectionManager _connectionManager = new();
+   //    readonly ILogger<DataStreamWebSocketHandler> _logger;
+
+   //    public DataStreamWebSocketHandler(
+   //       WebSocketConnectionManager connectionManager,
+   //       ILogger<DataStreamWebSocketHandler> logger
+   //    ) {
+   //       _connectionManager = connectionManager;
+   //       _logger = logger;
+   //    }
+
+   //    public async Task HandleWebSocket(HttpContext context, WebSocket webSocket) {
+   //       var socketId = _connectionManager.AddSocket(webSocket);
+
+   //       _logger.LogInformation($"WebSocket connection established with ID {socketId}");
+
+   //       while (webSocket.State == WebSocketState.Open) {
+   //          var message = await ReceiveMessageAsync(webSocket);
+   //          if (message != null) {
+   //             _logger.LogInformation($"Received message from ID {socketId}: {message}");
+   //             await BroadcastMessageAsync(message);
+   //          }
+   //       }
+
+   //       _connectionManager.RemoveSocket(socketId);
+   //       _logger.LogInformation($"WebSocket connection closed with ID {socketId}");
+   //    }
+
+   //    async Task<string?> ReceiveMessageAsync(WebSocket webSocket) {
+   //       var buffer = new byte[1024];
+   //       var result = await webSocket.ReceiveAsync(new ArraySegment<byte>(buffer), CancellationToken.None);
+
+   //       if (result.CloseStatus.HasValue) {
+   //          return null;
+   //       }
+
+   //       return System.Text.Encoding.UTF8.GetString(buffer, 0, result.Count);
+   //    }
+
+   //    async Task BroadcastMessageAsync(string message) {
+   //       foreach (var socket in _connectionManager.GetAllSockets()) {
+   //          if (socket.Value.State == WebSocketState.Open) {
+   //             await socket.Value.SendAsync(System.Text.Encoding.UTF8.GetBytes(message), WebSocketMessageType.Text, true, CancellationToken.None);
+   //          }
+   //       }
+   //    }
+   // }
+}

src/DataStreamSuite/Bts.DataStreamWebApp.Server/Program.cs

@@ -1,30 +1,166 @@
-var builder = WebApplication.CreateBuilder(args);
+using BCG;
 
-// Add services to the container.
+using bts;
+using bts.api.core;
+using bts.utils;
 
-builder.Services.AddControllers();
-// Learn more about configuring Swagger/OpenAPI at https://aka.ms/aspnetcore/swashbuckle
-builder.Services.AddEndpointsApiExplorer();
-builder.Services.AddSwaggerGen();
+using Bts.DataStreamExpressions;
+using Bts.DataStreamWebApp.Server;
+using Bts.DataStreamWebApp.Server.Controllers;
 
-var app = builder.Build();
+using CommandLine;
 
-app.UseDefaultFiles();
-app.UseStaticFiles();
+using mbts.config;
 
-// Configure the HTTP request pipeline.
-if (app.Environment.IsDevelopment())
-{
-   app.UseSwagger();
-   app.UseSwaggerUI();
+using System.Net.WebSockets;
+using System.Reactive.Disposables;
+using System.Reactive.Linq;
+using System.Text;
+using System.Text.Json;
+using System.Text.Json.Serialization;
+
+static string fmtReq(HttpRequest req) =>
+   $"{req.Protocol}://{req.Host}/{req.Path}?{req.QueryString}";
+
+static string fmtDataPacket(DataPacket dp) {
+   var sb = new StringBuilder();
+   var rows = dp.addedRows.Cast<IBlueRow>().ToList();
+   var colWidths = dp.schema.columnInfo.Select(c => rows.Max(r => r[c.colNo]?.ToString()?.Length ?? 0)).ToArray();
+   foreach (IBlueRow r in dp.addedRows) {
+      for (int i = 0; i < colWidths.Length; i++) {
+         var str = r[i]?.ToString() ?? string.Empty;
+         if (i > 0) {
+            sb.Append(" | ");
+         }
+         sb.Append(str);
+         var width = colWidths[i];
+         if (str.Length < width) {
+            sb.Append(new string(' ', width - str.Length));
+         }
+      }
+      sb.AppendLine();
+   }
+   return sb.ToString();
+   // var upd = new DataStreamUpdate();
+   // upd.addedRows.AddRange(dp.addedRows.Cast<IBlueRow>());
+   // upd.deletedRows.AddRange(dp.deletedRows.Cast<IBlueRow>());
+   // return fmtDataStreamUpdate(upd);
 }
 
-app.UseHttpsRedirection();
+static string fmtDataStreamUpdate(DataStreamUpdate rows) {
+   return JsonSerializer.Serialize(rows);
+}
 
-app.UseAuthorization();
+var parser = new Parser(ps => {
+   ps.CaseInsensitiveEnumValues = true;
+   ps.HelpWriter = Console.Out;
+});
 
-app.MapControllers();
+await parser.ParseArguments<DataStreamWebAppServerOptions>(args).
+   MapResult(run, errors => Task.FromResult(1));
 
-app.MapFallbackToFile("/index.html");
+async Task<int> run(DataStreamWebAppServerOptions opts) {
+   BTSNativeMethods.provide_command_line(args, args.Length);
+   Configuration.initialize();
+   var context = ServiceContext.create(opts.sparkd);
+   await context.login(opts.user!, opts.password!);
+   var vsWorkLoop = Observable.Interval(TimeSpan.FromMilliseconds(200)).Subscribe(_ => {
+      ViewStreamDatabase.instance.doWork();
+      MasterWorker.getInstance(false).doWork();
+   });
+   var builder = WebApplication.CreateBuilder(args);
+   builder.Services.AddDirectoryBrowser();
+   var app = builder.Build();
+   var wsOpts = new WebSocketOptions {
+      KeepAliveInterval = TimeSpan.FromMinutes(2)
+   };
+   app.UseWebSockets(wsOpts);
+   app.UseDefaultFiles();
+   app.UseStaticFiles();
+   app.MapFallbackToFile("/index.html");
 
-app.Run();
+   app.Use(async (context, next) => {
+      var reqDesc = fmtReq(context.Request);
+      Console.WriteLine($"Webserver received: {reqDesc}");
+      if (context.Request.Path == "/ws") {
+         if (context.WebSockets.IsWebSocketRequest) {
+            Console.WriteLine("Accepting websocket connection...");
+            using var webSocket = await context.WebSockets.AcceptWebSocketAsync();
+            Console.WriteLine("Websocket connection accepted.");
+            await runWebSocketServer(
+               opts.sparkd, opts.user!, opts.password!, webSocket);
+         } else {
+            context.Response.StatusCode = StatusCodes.Status400BadRequest;
+         }
+      } else {
+         Console.WriteLine($"Webserver ignoring req: {reqDesc}");
+         await next(context);
+      }
+   });
+
+   try {
+      await app.RunAsync();
+   } catch (Exception ex) {
+      Console.WriteLine($"Exception on DataStreamWebsocketServer thread: {ex}");
+      return 1;
+   }
+   Spark.Core.Disposables.Disposable.dispose(ref vsWorkLoop);
+   return 0;
+}
+
+async Task<int> runWebSocketServer(string? sparkd, string user, string password, WebSocket webSocket) {
+   var buffer = new byte[1024 * 4];
+   var receiveResult = await webSocket.ReceiveAsync(
+       new ArraySegment<byte>(buffer), CancellationToken.None);
+
+   try {
+      while (!receiveResult.CloseStatus.HasValue) {
+         Console.WriteLine("Awaiting websocket message from client");
+         receiveResult = await webSocket.ReceiveAsync(
+             new ArraySegment<byte>(buffer), CancellationToken.None);
+         var msg = Encoding.UTF8.GetString(buffer, 0, receiveResult.Count);
+         var rpcReq = JsonSerializer.Deserialize<Bts.Websocket.JsonRpcRequest<DataStreamSubscription>>(msg);
+         var queryArgs = rpcReq!.Params!;
+         // var queryArgs = new DataStreamSubscription { Query = msg };
+         var exp = DataStreamExpressionsInterpreter.Evaluate(queryArgs.Query!);
+         if (exp is null) {
+            Console.WriteLine($"Invalid query expression: {queryArgs.Query}.");
+            continue;
+         }
+         var vs = exp.getView(new Dictionary<string, StatementExpression>(0));
+         Console.WriteLine("Subscribing to stream...");
+         vs.onChange += onStreamChanged;
+
+         await webSocket.SendAsync(
+             new ArraySegment<byte>(buffer, 0, receiveResult.Count),
+             receiveResult.MessageType,
+             receiveResult.EndOfMessage,
+             CancellationToken.None);
+      }
+   } catch (Exception ex) {
+      Console.WriteLine($"Exception on WebsocketServer thread: {ex}");
+      return 1;
+   }
+
+   return 0;
+
+   async void onStreamChanged(object newValue) {
+      if (newValue is not DataPacket dp) {
+         Console.WriteLine(
+            $"OnStreamChanged delegate expected DataPacket (actually received {newValue?.GetType()}.");
+         return;
+      }
+      Console.WriteLine("Received stream update.");
+      var sb = new StringBuilder();
+      var updateDesc = fmtDataPacket(dp);
+      var msg = $"Received new rows:\n{updateDesc}";
+      // var payload = new ReadOnlyMemory<byte>([.. msg]);
+      var encoded = Encoding.UTF8.GetBytes(msg);
+      var payload = new ArraySegment<byte>(encoded, 0, encoded.Length);
+      await webSocket.SendAsync(
+          payload,
+          receiveResult.MessageType,
+          receiveResult.EndOfMessage,
+          CancellationToken.None);
+   }
+}

src/DataStreamSuite/Bts.DataStreamWebApp.Server/Properties/launchSettings.json

@@ -1,5 +1,10 @@
 {
   "profiles": {
+    "default": {
+      "commandName": "Project",
+      "launchBrowser": true,
+      "applicationUrl": "http://localhost:5400"
+    },
     "http": {
       "commandName": "Project",
       "launchBrowser": true,
@@ -40,4 +45,4 @@
       "sslPort": 44383
     }
   }
-}
+}

src/DataStreamSuite/Bts.DataStreamWebApp.Server/WebsocketConnectionManager.cs

@@ -0,0 +1,37 @@
+using System;
+using System.Collections.Concurrent;
+using System.Net.WebSockets;
+
+namespace Bts.DataStreamWebApp.Server {
+   // public class WebSocketConnectionManager {
+   //    private readonly ConcurrentDictionary<Guid, WebSocket> _sockets = new ConcurrentDictionary<Guid, WebSocket>();
+
+   //    public Guid AddSocket(WebSocket socket) {
+   //       var socketId = Guid.NewGuid();
+   //       _sockets.TryAdd(socketId, socket);
+   //       return socketId;
+   //    }
+
+   //    public WebSocket? GetSocket(Guid socketId) {
+   //       _sockets.TryGetValue(socketId, out var socket);
+   //       return socket;
+   //    }
+
+   //    public ConcurrentDictionary<Guid, WebSocket>.ValueCollection GetAllSockets() {
+   //       return _sockets.Values;
+   //    }
+
+   //    public Guid? GetSocketId(WebSocket socket) {
+   //       foreach (var (key, value) in _sockets) {
+   //          if (value == socket) {
+   //             return key;
+   //          }
+   //       }
+   //       return null;
+   //    }
+
+   //    public void RemoveSocket(Guid socketId) {
+   //       _sockets.TryRemove(socketId, out _);
+   //    }
+   // }
+}

src/DataStreamSuite/Bts.DataStreamWebApp.Server/wwwroot/alt-index.html

@@ -0,0 +1,168 @@
+<!DOCTYPE html>
+<html>
+<head>
+    <meta charset="utf-8" />
+    <title></title>
+    <style>
+        table {
+            border: 0
+        }
+
+        .commslog-data {
+            font-family: Consolas, Courier New, Courier, monospace;
+        }
+
+        .commslog-server {
+            background-color: red;
+            color: white
+        }
+
+        .commslog-client {
+            background-color: green;
+            color: white
+        }
+    </style>
+</head>
+<body>
+    <h1>ALT WebSocket Sample Application</h1>
+    <p id="stateLabel">Ready to connect...</p>
+    <div>
+        <label for="connectionUrl">WebSocket Server URL:</label>
+        <input id="connectionUrl" />
+        <button id="connectButton" type="submit">Connect</button>
+    </div>
+    <p></p>
+    <div>
+        <label for="sendMessage">Message to send:</label>
+        <input id="sendMessage" disabled />
+        <button id="sendButton" type="submit" disabled>Send</button>
+        <button id="closeButton" disabled>Close Socket</button>
+    </div>
+
+    <h2>Communication Log</h2>
+    <table style="width: 800px">
+        <thead>
+            <tr>
+                <td style="width: 100px">From</td>
+                <td style="width: 100px">To</td>
+                <td>Data</td>
+            </tr>
+        </thead>
+        <tbody id="commsLog">
+        </tbody>
+    </table>
+
+    <script>
+        var connectionUrl = document.getElementById("connectionUrl");
+        var connectButton = document.getElementById("connectButton");
+        var stateLabel = document.getElementById("stateLabel");
+        var sendMessage = document.getElementById("sendMessage");
+        var sendButton = document.getElementById("sendButton");
+        var commsLog = document.getElementById("commsLog");
+        var closeButton = document.getElementById("closeButton");
+        var socket;
+
+        var scheme = document.location.protocol === "https:" ? "wss" : "ws";
+        var port = document.location.port ? (":" + document.location.port) : "";
+
+        connectionUrl.value = scheme + "://" + document.location.hostname + port + "/ws" ;
+
+        function updateState() {
+            function disable() {
+                sendMessage.disabled = true;
+                sendButton.disabled = true;
+                closeButton.disabled = true;
+            }
+            function enable() {
+                sendMessage.disabled = false;
+                sendButton.disabled = false;
+                closeButton.disabled = false;
+            }
+
+            connectionUrl.disabled = true;
+            connectButton.disabled = true;
+
+            if (!socket) {
+                disable();
+            } else {
+                switch (socket.readyState) {
+                    case WebSocket.CLOSED:
+                        stateLabel.innerHTML = "Closed";
+                        disable();
+                        connectionUrl.disabled = false;
+                        connectButton.disabled = false;
+                        break;
+                    case WebSocket.CLOSING:
+                        stateLabel.innerHTML = "Closing...";
+                        disable();
+                        break;
+                    case WebSocket.CONNECTING:
+                        stateLabel.innerHTML = "Connecting...";
+                        disable();
+                        break;
+                    case WebSocket.OPEN:
+                        stateLabel.innerHTML = "Open";
+                        enable();
+                        break;
+                    default:
+                        stateLabel.innerHTML = "Unknown WebSocket State: " + htmlEscape(socket.readyState);
+                        disable();
+                        break;
+                }
+            }
+        }
+
+        closeButton.onclick = function () {
+            if (!socket || socket.readyState !== WebSocket.OPEN) {
+                alert("socket not connected");
+            }
+            socket.close(1000, "Closing from client");
+        };
+
+        sendButton.onclick = function () {
+            if (!socket || socket.readyState !== WebSocket.OPEN) {
+                alert("socket not connected");
+            }
+            var data = sendMessage.value;
+            socket.send(data);
+            commsLog.innerHTML += '<tr>' +
+                '<td class="commslog-client">Client</td>' +
+                '<td class="commslog-server">Server</td>' +
+                '<td class="commslog-data">' + htmlEscape(data) + '</td></tr>';
+        };
+
+        connectButton.onclick = function() {
+            stateLabel.innerHTML = "Connecting";
+            socket = new WebSocket(connectionUrl.value);
+            socket.onopen = function (event) {
+                updateState();
+                commsLog.innerHTML += '<tr>' +
+                    '<td colspan="3" class="commslog-data">Connection opened</td>' +
+                '</tr>';
+            };
+            socket.onclose = function (event) {
+                updateState();
+                commsLog.innerHTML += '<tr>' +
+                    '<td colspan="3" class="commslog-data">Connection closed. Code: ' + htmlEscape(event.code) + '. Reason: ' + htmlEscape(event.reason) + '</td>' +
+                '</tr>';
+            };
+            socket.onerror = updateState;
+            socket.onmessage = function (event) {
+                commsLog.innerHTML += '<tr>' +
+                    '<td class="commslog-server">Server</td>' +
+                    '<td class="commslog-client">Client</td>' +
+                    '<td class="commslog-data">' + htmlEscape(event.data) + '</td></tr>';
+            };
+        };
+
+        function htmlEscape(str) {
+            return str.toString()
+                .replace(/&/g, '&amp;')
+                .replace(/"/g, '&quot;')
+                .replace(/'/g, '&#39;')
+                .replace(/</g, '&lt;')
+                .replace(/>/g, '&gt;');
+        }
+    </script>
+</body>
+</html>

src/DataStreamSuite/Bts.DataStreamWebApp.Server/wwwroot/index.html

@@ -0,0 +1,176 @@
+<!DOCTYPE html>
+<html>
+<head>
+    <meta charset="utf-8" />
+    <title></title>
+    <style>
+        table {
+            border: 0
+        }
+
+        .commslog-data {
+            font-family: Consolas, Courier New, Courier, monospace;
+        }
+
+        .commslog-server {
+            background-color: red;
+            color: white
+        }
+
+        .commslog-client {
+            background-color: green;
+            color: white
+        }
+    </style>
+</head>
+<body>
+    <h1>DataStreams via WebSocket</h1>
+    <p id="stateLabel">Ready to connect...</p>
+    <div>
+        <label for="connectionUrl">WebSocket Server URL:</label>
+        <input id="connectionUrl" />
+        <button id="connectButton" type="submit">Connect</button>
+    </div>
+    <p></p>
+    <div>
+        <label for="sendMessage">Query:</label>
+        <input id="sendMessage" disabled />
+        <button id="sendButton" type="submit" disabled>Send</button>
+        <button id="closeButton" disabled>Close Socket</button>
+    </div>
+
+    <h2>Communication Log</h2>
+    <table style="width: 800px">
+        <thead>
+            <tr>
+                <td style="width: 100px">From</td>
+                <td style="width: 100px">To</td>
+                <td>Data</td>
+            </tr>
+        </thead>
+        <tbody id="commsLog">
+        </tbody>
+    </table>
+
+    <script>
+        var connectionUrl = document.getElementById("connectionUrl");
+        var connectButton = document.getElementById("connectButton");
+        var stateLabel = document.getElementById("stateLabel");
+        var sendMessage = document.getElementById("sendMessage");
+        var sendButton = document.getElementById("sendButton");
+        var commsLog = document.getElementById("commsLog");
+        var closeButton = document.getElementById("closeButton");
+        var socket;
+
+        var scheme = document.location.protocol === "https:" ? "wss" : "ws";
+        var port = document.location.port ? (":" + document.location.port) : "";
+
+        connectionUrl.value = scheme + "://" + document.location.hostname + port + "/ws" ;
+
+        function updateState() {
+            function disable() {
+                sendMessage.disabled = true;
+                sendButton.disabled = true;
+                closeButton.disabled = true;
+            }
+            function enable() {
+                sendMessage.disabled = false;
+                sendButton.disabled = false;
+                closeButton.disabled = false;
+            }
+
+            connectionUrl.disabled = true;
+            connectButton.disabled = true;
+
+            if (!socket) {
+                disable();
+            } else {
+                switch (socket.readyState) {
+                    case WebSocket.CLOSED:
+                        stateLabel.innerHTML = "Closed";
+                        disable();
+                        connectionUrl.disabled = false;
+                        connectButton.disabled = false;
+                        break;
+                    case WebSocket.CLOSING:
+                        stateLabel.innerHTML = "Closing...";
+                        disable();
+                        break;
+                    case WebSocket.CONNECTING:
+                        stateLabel.innerHTML = "Connecting...";
+                        disable();
+                        break;
+                    case WebSocket.OPEN:
+                        stateLabel.innerHTML = "Open";
+                        enable();
+                        break;
+                    default:
+                        stateLabel.innerHTML = "Unknown WebSocket State: " + htmlEscape(socket.readyState);
+                        disable();
+                        break;
+                }
+            }
+        }
+
+        closeButton.onclick = function () {
+            if (!socket || socket.readyState !== WebSocket.OPEN) {
+                alert("socket not connected");
+            }
+            socket.close(1000, "Closing from client");
+        };
+
+        sendButton.onclick = function () {
+            if (!socket || socket.readyState !== WebSocket.OPEN) {
+                alert("socket not connected");
+            }
+            const rpcReq = {
+              jsonrpc: "2.0",
+              method: "subscribe",
+              params: {
+                query: sendMessage.value,
+              },
+              id: 1,
+            };
+            const rpcReqStr = JSON.stringify(rpcReq);
+            socket.send(rpcReqStr);
+            commsLog.innerHTML += '<tr>' +
+                '<td class="commslog-client">Client</td>' +
+                '<td class="commslog-server">Server</td>' +
+                '<td class="commslog-data">' + htmlEscape(rpcReqStr) + '</td></tr>';
+        };
+
+        connectButton.onclick = function() {
+            stateLabel.innerHTML = "Connecting";
+            socket = new WebSocket(connectionUrl.value);
+            socket.onopen = function (event) {
+                updateState();
+                commsLog.innerHTML += '<tr>' +
+                    '<td colspan="3" class="commslog-data">Connection opened</td>' +
+                '</tr>';
+            };
+            socket.onclose = function (event) {
+                updateState();
+                commsLog.innerHTML += '<tr>' +
+                    '<td colspan="3" class="commslog-data">Connection closed. Code: ' + htmlEscape(event.code) + '. Reason: ' + htmlEscape(event.reason) + '</td>' +
+                '</tr>';
+            };
+            socket.onerror = updateState;
+            socket.onmessage = function (event) {
+                commsLog.innerHTML += '<tr>' +
+                    '<td class="commslog-server">Server</td>' +
+                    '<td class="commslog-client">Client</td>' +
+                    '<td class="commslog-data">' + htmlEscape(event.data) + '</td></tr>';
+            };
+        };
+
+        function htmlEscape(str) {
+            return str.toString()
+                .replace(/&/g, '&amp;')
+                .replace(/"/g, '&quot;')
+                .replace(/'/g, '&#39;')
+                .replace(/</g, '&lt;')
+                .replace(/>/g, '&gt;');
+        }
+    </script>
+</body>
+</html>

src/DataStreamSuite/DataStream.Console/DataStream.Console.csproj

@@ -7,7 +7,7 @@
 
   <PropertyGroup>
     <OutputType>Exe</OutputType>
-    <TargetFramework>net7.0</TargetFramework>
+    <TargetFramework>net8.0</TargetFramework>
     <ImplicitUsings>enable</ImplicitUsings>
     <Nullable>enable</Nullable>
   </PropertyGroup>

src/DataStreamSuite/DataStreamCore/BlueData/ColumnInfo.cs

@@ -1,95 +1,103 @@
-using System;
-
-namespace BCG
-{
-   /// <summary>
-   /// Resprsents information about a column schema, ie its ordinal, the type, the name and constant value or expression. where applicable
-   /// </summary>
-   [Serializable]
-   public class ColumnInfo : IComparable
-   {
-      public static bool isAllowedType (Type t)
-      {
-         return 
-            t.Equals (typeof (System.Int32)) ||
-            t.Equals (typeof (System.Int64)) ||
-            t.Equals (typeof (System.Double)) ||
-            t.Equals (typeof (System.String)) ||
-            t.Equals (typeof (System.DateTime)) ? true :
-            false;
-      }
-
-      //the default column info does not cach values in a lookup table, therefore it is in group 0
-      public ColumnInfo(string colName, int colNo, Type colType)
-         : this(colName, colNo, colType, 0)
-      {
-
-
-      }
-
-      public ColumnInfo(string colName, int colNo, Type colType, short colGroup)
-      {
-         myColName = colName;
-         myColNo = colNo;
-         myColType = colType;
-         myLookupTableGroup = colGroup;
-         myColValue = null;
-      }
-
-
-      public ColumnInfo (string colName, int colNo, object colValue)
-         : this (colName, colNo, colValue.GetType())  {myColValue = colValue == null ? DBNull.Value : colValue;}
-
-      public ColumnInfo (string colName, int colNo, Type colType, object colValue)
-         : this (colName, colNo, colType) 
-      {
-
-         myColValue = colValue == null ? DBNull.Value : colValue;
-      
-      }
-
-
-      public int CompareTo (object obj)
-      {
-         int otherColNo = ((ColumnInfo)(obj)).colNo;
-         if (myColNo < otherColNo)
-            return -1;
-         else if (myColNo > otherColNo)
-            return 1;
-         return 0;
-      }
-
-      //public bool isCalculated{get{return myExpression != null;}}
-
-      public bool isConstant {get{return myColValue != null;}}
-
-      private int myColNo;
-      private string myColName;
-      private Type myColType;
-      //private string myExpression;
-      private object myColValue;
-
-      //no table will ever have more than 65,536 columns
-      //if it does, we are in serious trouble
-      private short myLookupTableGroup;
-
-      public int colNo{get{return myColNo;}}
-      public string colName {get{return myColName;}}
-      public Type colType {get{return myColType;}}
-
-
-      //this value corresponds to how the data in the column is serialized. 
-      //if the group number is 0, the values are not stored in a table
-      //if the vlaue is non-zero, all columns with the the same lookup table group are stored in the same lookup table together
-      //are stored together in a lookup table
-
-      
-      public short lookupTableGroup
-      {
-         get { return myLookupTableGroup; }
-         set { myLookupTableGroup = value; }
-      }
-      //public string expression {get{return myExpression;}}
-      public object colValue {get{return myColValue;}}
-   }
-}
+using System;
+using System.Text.Json.Serialization;
+
+namespace BCG
+{
+   /// <summary>
+   /// Resprsents information about a column schema, ie its ordinal, the type, the name and constant value or expression. where applicable
+   /// </summary>
+   [Serializable]
+   public class ColumnInfo : IComparable
+   {
+      public static bool isAllowedType (Type t)
+      {
+         return 
+            t.Equals (typeof (System.Int32)) ||
+            t.Equals (typeof (System.Int64)) ||
+            t.Equals (typeof (System.Double)) ||
+            t.Equals (typeof (System.String)) ||
+            t.Equals (typeof (System.DateTime)) ? true :
+            false;
+      }
+
+      //the default column info does not cach values in a lookup table, therefore it is in group 0
+      public ColumnInfo(string colName, int colNo, Type colType)
+         : this(colName, colNo, colType, 0)
+      {
+
+
+      }
+
+      public ColumnInfo(string colName, int colNo, Type colType, short colGroup)
+      {
+         myColName = colName;
+         myColNo = colNo;
+         myColType = colType;
+         myLookupTableGroup = colGroup;
+         myColValue = null;
+      }
+
+
+      public ColumnInfo (string colName, int colNo, object colValue)
+         : this (colName, colNo, colValue.GetType())  {myColValue = colValue == null ? DBNull.Value : colValue;}
+
+      public ColumnInfo (string colName, int colNo, Type colType, object colValue)
+         : this (colName, colNo, colType) 
+      {
+
+         myColValue = colValue == null ? DBNull.Value : colValue;
+      
+      }
+
+
+      public int CompareTo (object obj)
+      {
+         int otherColNo = ((ColumnInfo)(obj)).colNo;
+         if (myColNo < otherColNo)
+            return -1;
+         else if (myColNo > otherColNo)
+            return 1;
+         return 0;
+      }
+
+      //public bool isCalculated{get{return myExpression != null;}}
+
+      public bool isConstant {get{return myColValue != null;}}
+
+      private int myColNo;
+      private string myColName;
+      private Type myColType;
+      //private string myExpression;
+      private object myColValue;
+
+      //no table will ever have more than 65,536 columns
+      //if it does, we are in serious trouble
+      private short myLookupTableGroup;
+
+      [JsonPropertyName("colNo")]
+      public int colNo{get{return myColNo;}}
+
+      [JsonPropertyName("colName")]
+      public string colName {get{return myColName;}}
+
+      [JsonPropertyName("colType")]
+      public Type colType {get{return myColType;}}
+
+
+      //this value corresponds to how the data in the column is serialized. 
+      //if the group number is 0, the values are not stored in a table
+      //if the vlaue is non-zero, all columns with the the same lookup table group are stored in the same lookup table together
+      //are stored together in a lookup table
+
+      
+      [JsonIgnore]
+      public short lookupTableGroup
+      {
+         get { return myLookupTableGroup; }
+         set { myLookupTableGroup = value; }
+      }
+      //public string expression {get{return myExpression;}}
+      [JsonIgnore]
+      public object colValue {get{return myColValue;}}
+   }
+}

src/DataStreamSuite/DataStreamCore/BlueRowJsonConverter.cs

@@ -0,0 +1,61 @@
+#nullable enable
+using BCG;
+
+using System;
+using System.Text.Json;
+using System.Text.Json.Serialization;
+
+namespace Bts.DataStreamCore;
+
+class BlueRowJsonConverter : JsonConverter<IBlueRow> {
+   IBlueTableSchema? schema = null;
+
+   public override IBlueRow Read(
+      ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options) {
+      if (schema is null) {
+         throw new InvalidOperationException("Cannot read row from json because schema is null");
+      }
+      var mbr = new MutableBlueRow(schema);
+      int colNum = 0;
+      switch (reader.TokenType) {
+         case JsonTokenType.StartArray:
+            while (reader.Read() && reader.TokenType != JsonTokenType.EndArray) {
+               mbr.setValue(colNum++, readCellValue(ref reader));
+               colNum++;
+            }
+            break;
+         case JsonTokenType.StartObject:
+            while (reader.Read() && reader.TokenType != JsonTokenType.EndObject) {
+               if (reader.TokenType != JsonTokenType.PropertyName) {
+                  throw new InvalidOperationException(
+                     "Expected 'PropertyName' immediately after 'StartObject' token in row json");
+               }
+               var colName = reader.GetString();
+               if (!reader.Read()) {
+                  throw new InvalidOperationException(
+                     "Failed to read token immediately after 'PropertyName' token in row json");
+               }
+               mbr.setValue(colName, readCellValue(ref reader));
+            }
+            break;
+      }
+      if (mbr.Count != schema.columnInfo.Length) {
+         throw new InvalidOperationException(
+            $"Row json array length '{mbr.Count}' does not match the schema's " +
+            $"column length '{schema.columnInfo.Length}'.");
+      }
+      return mbr;
+
+      static object? readCellValue(ref Utf8JsonReader reader) =>
+         reader.TokenType switch {
+            JsonTokenType.String => reader.GetString(),
+            JsonTokenType.Number => reader.GetDouble(),
+            JsonTokenType.True or JsonTokenType.False => reader.GetBoolean(),
+            _ => throw new InvalidOperationException("Failed to read cell value"),
+         };
+   }
+
+   public override void Write(Utf8JsonWriter writer, IBlueRow value, JsonSerializerOptions options) {
+      throw new NotImplementedException();
+   }
+}

src/DataStreamSuite/DataStreamCore/DataPacketJsonConverter.cs

@@ -0,0 +1,154 @@
+#nullable enable
+using BCG;
+
+using Spark.Core.Json;
+
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text.Json;
+using System.Text.Json.Serialization;
+
+namespace Bts.DataStreamCore;
+
+class DataPacketJsonConverter : JsonConverter<DataPacket> {
+   /// <summary>
+   /// Whether or not to include a field for current rows (i.e., snapshot of current state). By
+   /// default, only incremental changes are included ('added' and 'deleted').
+   /// </summary>
+   public bool IncludeCurrent { get; set; }
+
+   /// <summary>
+   /// Whether to treat rows as objects (name/value pairs), or arrays (list of values). Arrays
+   /// result in smaller payloads, but aren't as easy to read as name/value pairs (for mere humans).
+   /// </summary>
+   public bool UseObjectRows { get; set; }
+
+   public override DataPacket? Read(ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options) {
+      if (!reader.ReadObject()) {
+         return null;
+      }
+      if (!reader.ReadPropertyName(out var prop)) {
+         return null;
+      }
+      if (!reader.Read()) {
+         return null;
+      }
+      if (prop == "added") {
+         var added = new List<MutableBlueRow>();
+         if (reader.TokenType != JsonTokenType.StartArray) {
+            return null;
+         }
+         while (true) {
+            if (!reader.Read()) {
+               break;
+            }
+            if (reader.TokenType != JsonTokenType.StartArray || !reader.Read()) {
+               return null;
+            }
+            switch (reader.TokenType) {
+               case JsonTokenType.String:
+                  // DAMNIT! Need the schema.
+                  break;
+            }
+         }
+      }
+   }
+
+   public override void Write(Utf8JsonWriter writer, DataPacket value, JsonSerializerOptions options) {
+      using var _ = writer.WriteObject();
+      using (writer.WriteArray("schema")) {
+         foreach (var col in value.schema.columnInfo) {
+            using (writer.WriteObject()) {
+               writer.WriteString("name", col.colName);
+               writer.WriteNumber("index", col.colNo);
+               // TODO: use language agnostic type names (e.g., 'string', 'integer', 'float', 'date' etc.)
+               writer.WriteString("type", col.colType.ToString());
+            }
+         }
+      }
+
+      var cellWriters = value.schema.columnInfo.Select(getCellWriter).ToList();
+      using (writer.WriteArray("added")) {
+         foreach (IBlueRow row in value.addedRows) {
+            using (writer.WriteArray()) {
+               for (int i = 0; i < cellWriters.Count; i++) {
+                  cellWriters[i](row[i]);
+               }
+            }
+         }
+      }
+      using (writer.WriteArray("deleted")) {
+         foreach (IBlueRow row in value.deletedRows) {
+            using (writer.WriteArray()) {
+               for (int i = 0; i < cellWriters.Count; i++) {
+                  cellWriters[i](row[i]);
+               }
+            }
+         }
+      }
+      if (IncludeCurrent) {
+         using (writer.WriteArray("current")) {
+            foreach (IBlueRow row in value.newRows) {
+               using (writer.WriteArray()) {
+                  for (int i = 0; i < cellWriters.Count; i++) {
+                     cellWriters[i](row[i]);
+                  }
+               }
+            }
+         }
+      }
+
+      Action<object?> getCellWriter(ColumnInfo colInfo) {
+         var writeVal = getCellValueWriter(colInfo);
+         if (!UseObjectRows) {
+            return writeVal;
+         }
+         return v => {
+            writer.WritePropertyName(colInfo.colName);
+            writeVal(v);
+         };
+      }
+
+      Action<object?> getCellValueWriter(ColumnInfo colInfo) {
+         var t = colInfo.colType;
+         if (t == typeof(int)) {
+            return nullCheckWrapper(v => writer.WriteNumberValue((int)v));
+         }
+         if (t == typeof(uint)) {
+            return nullCheckWrapper(v => writer.WriteNumberValue((uint)v));
+         }
+         if (t == typeof(long)) {
+            return nullCheckWrapper(v => writer.WriteNumberValue((long)v));
+         }
+         if (t == typeof(ulong)) {
+            return nullCheckWrapper(v => writer.WriteNumberValue((ulong)v));
+         }
+         if (t == typeof(short)) {
+            return nullCheckWrapper(v => writer.WriteNumberValue((short)v));
+         }
+         if (t == typeof(ushort)) {
+            return nullCheckWrapper(v => writer.WriteNumberValue((ushort)v));
+         }
+         if (t == typeof(double)) {
+            return nullCheckWrapper(v => writer.WriteNumberValue((double)v));
+         }
+         if (t == typeof(float)) {
+            return nullCheckWrapper(v => writer.WriteNumberValue((float)v));
+         }
+         if (t == typeof(bool)) {
+            return nullCheckWrapper(v => writer.WriteBooleanValue((bool)v));
+         }
+         return nullCheckWrapper(v => writer.WriteStringValue(v?.ToString()));
+
+         Action<object?> nullCheckWrapper(Action<object> writeNonNull) =>
+            v => {
+               if (v is null) {
+                  writer.WriteNullValue();
+               } else {
+                  writeNonNull(v);
+               }
+            };
+      }
+   }
+}

src/DataStreamSuite/ExpressionTester/ExpressionTester.csproj

@@ -6,7 +6,7 @@
 
   <PropertyGroup>
     <OutputType>Exe</OutputType>
-    <TargetFramework>net7.0</TargetFramework>
+    <TargetFramework>net8.0</TargetFramework>
     <ImplicitUsings>enable</ImplicitUsings>
     <Nullable>enable</Nullable>
   </PropertyGroup>

src/DataStreamSuite/ExpressionTests/ExpressionTests.csproj

@@ -1,7 +1,7 @@
 <Project Sdk="Microsoft.NET.Sdk">
 
   <PropertyGroup>
-    <TargetFramework>net7.0</TargetFramework>
+    <TargetFramework>net8.0</TargetFramework>
     <ImplicitUsings>enable</ImplicitUsings>
     <Nullable>enable</Nullable>
 

src/DataStreamSuite/TransportUtilities/TransportUtilities.csproj

@@ -1,7 +1,7 @@
 <Project Sdk="Microsoft.NET.Sdk">
 
   <PropertyGroup>
-    <TargetFramework>net7.0</TargetFramework>
+    <TargetFramework>net8.0</TargetFramework>
     <ImplicitUsings>enable</ImplicitUsings>
     <Nullable>enable</Nullable>
   </PropertyGroup>

src/client/Spark.Console/Spark.Console.csproj

@@ -4,7 +4,7 @@
     <OutputType>Exe</OutputType>
     <TargetFrameworks>net8.0</TargetFrameworks>
     <Nullable>enable</Nullable>
-    <RuntimeIdentifiers>win10-x64;ubuntu.16.04-x64</RuntimeIdentifiers>
+    <RuntimeIdentifiers>any</RuntimeIdentifiers>
     <Authors>Blue Trading Systems</Authors>
     <Company>Blue Trading Systems</Company>
     <Product>Spark</Product>

src/client/spark/Spark.csproj

@@ -11,7 +11,7 @@
     <AllowUnsafeBlocks>true</AllowUnsafeBlocks>
     <Configurations>Debug;Release</Configurations>
     <Platforms>x64</Platforms>
-    <RuntimeIdentifiers>win10-x64</RuntimeIdentifiers>
+    <RuntimeIdentifiers>win-x64</RuntimeIdentifiers>
     <PlatformTarget>x64</PlatformTarget>
     <LanguageTargets Condition="'$(VisualStudioVersion)'=='17.0'">
       C:/Program Files/Microsoft Visual Studio/2022/Community/Msbuild/Current/Bin/amd64/Microsoft.CSharp.targets

src/client/spark_core/Json/JsonExtensions.cs

@@ -0,0 +1,44 @@
+#nullable enable
+namespace Spark.Core.Json;
+
+using Spark.Core.Disposables;
+
+using System;
+using System.Diagnostics.CodeAnalysis;
+using System.Text.Json;
+
+public static class JsonExtensions {
+   public static bool ReadObject(this Utf8JsonReader reader) {
+      return reader.Read() && reader.TokenType == JsonTokenType.StartObject;
+   }
+
+   public static bool ReadPropertyName(
+      this Utf8JsonReader reader, [NotNullWhen(true)]out string? propertyName) {
+      if (reader.Read() && reader.TokenType == JsonTokenType.PropertyName) {
+         propertyName = reader.GetString()!;
+         return true;
+      }
+      propertyName = null;
+      return false;
+   }
+
+   public static IDisposable WriteArray(this Utf8JsonWriter writer) {
+      writer.WriteStartArray();
+      return Disposable.Create(writer.WriteEndArray);
+   }
+
+   public static IDisposable WriteArray(this Utf8JsonWriter writer, string propertyName) {
+      writer.WriteStartArray(propertyName);
+      return Disposable.Create(writer.WriteEndArray);
+   }
+
+   public static IDisposable WriteObject(this Utf8JsonWriter writer) {
+      writer.WriteStartObject();
+      return Disposable.Create(writer.WriteEndObject);
+   }
+
+   public static IDisposable WriteObject(this Utf8JsonWriter writer, string propertyName) {
+      writer.WritePropertyName(propertyName);
+      return Disposable.Create(writer.WriteEndObject);
+   }
+}

src/client/spark_websocket/Core/WebsocketConnectionGateway.cs

@@ -1,480 +1,478 @@
-using Bts.Websocket;
-
-using System.Net.WebSockets;
-using System.Text;
-using System.Collections.Concurrent;
-using Microsoft.AspNetCore.Builder;
-using Microsoft.AspNetCore.Hosting;
-using Microsoft.Extensions.DependencyInjection;
-using Microsoft.Extensions.Logging;
-
-namespace bts.websocket_core {
-   public enum MessageType {
-      Message,
-      Close
-   }
-   /// <summary>
-   /// gateway between sync and socket logic
-   /// 1. keeps all active clients in the store(updates store if sync or handler informs)
-   /// 2. listens for connections (upon connection passes to handler)
-   /// 3. gets messages from sync forwards to handler
-   /// 4. updates store and sync about any changes in connections (when told by handler)
-   /// </summary>
-   public class WebsocketConnectionGateway : IWebsocketConnectionGateway {
-      /// <summary>
-      /// Represents a connection to a client through a WebSocket. 
-      /// </summary>
-      public class Connection {
-         /// <summary>
-         /// The WebSocket instance associated with this connection.
-         /// </summary>
-         public WebSocket Ws { get; }
-         /// <summary>
-         /// A queue that holds outgoing messages to be sent to the client.
-         /// </summary>
-         public ConcurrentQueue<Tuple<MessageType, string>> OutgoingQueue { get; } = new();
-         /// <summary>
-         /// Initializes a new instance of the <see cref="Connection"/> class.
-         /// </summary>
-         /// <param name="websocket">The WebSocket associated with this connection.</param>
-         public Connection(WebSocket websocket) {
-            Ws = websocket;
-         }
-      }
-
-
-      /// <summary>
-      /// wrapper for concurrent Dictionary
-      /// Stores all the alive connection
-      /// Additions and removals done by gateway
-      /// websockethandler can access but not change anything
-      /// </summary>
-      class WebSocketConnectionStore {
-         private ConcurrentDictionary<WebsocketClientID, Connection> Connections { get; } = new();
-         /// <summary>
-         /// Adds a new connection to the connections dictionary using the given id and connection object
-         /// </summary>
-         /// <param name="id">The id to use as the key for the new connection</param>
-         /// <param name="conn">The connection object to add to the dictionary</param>
-         public void AddConnection(WebsocketClientID id, Connection conn) {
-            if (!Connections.TryAdd(id, conn)) {
-               var log_txt = $"WebsocketDataSync.createID client_id: {id.ToString()} Already exits";
-               bts.Log.Gen.warn(log_txt);
-               Connections[id] = conn;
-            }
-         }
-         /// <summary>
-         /// Removes a connection from the connections dictionary using the given id
-         /// </summary>
-         /// <param name="id">The id of the connection to remove</param>
-         public void RemoveConnection(WebsocketClientID id) {
-            if (Connections.TryRemove(id, out Connection? conn)) {
-               conn.Ws.Dispose();
-            }
-         }
-         /// <summary>
-         /// Retrieves the Connection object associated with the given id
-         /// </summary>
-         /// <param name="id">The id of the connection to retrieve</param>
-         /// <returns>The Connection object associated with the given id, or null if the id is not found</returns>
-         public Connection? GetConnection(WebsocketClientID id) => Connections.GetValueOrDefault(id);
-         /// <summary>
-         /// Retrieves the Connection object associated with the given id
-         /// </summary>
-         /// <param name="id">The id of the connection to retrieve</param>
-         /// <returns>The Connection object associated with the given id, or null if the id is not found</returns>
-         public WebSocket? GetSocket(WebsocketClientID id) => Connections.GetValueOrDefault(id)?.Ws;
-         /// <summary>
-         /// Get the outgoing message queue for a specific client connection identified by the given id.
-         /// </summary>
-         /// <param name="id">The unique identifier for the client connection.</param>
-         /// <returns>A concurrent queue of message tuples if the connection is found, null otherwise.</returns>
-         public ConcurrentQueue<Tuple<MessageType, string>>? GetSocketQueue(WebsocketClientID id) =>
-         Connections.GetValueOrDefault(id)?.OutgoingQueue;
-         /// <summary>
-         /// Check if the store contains a connection with the specified id.
-         /// </summary>
-         /// <param name="id">The unique identifier for the client connection.</param>
-         /// <returns>True if the connection is found, false otherwise.</returns>
-         public bool ContainsKey(WebsocketClientID id) => Connections.ContainsKey(id);
-      }
-
-      /// <summary>
-      /// is controlled by gateway: 
-      /// once a new connection is made the gateway passes the socket to this and this
-      /// handles everything: 
-      /// 1. keeps a queue of messages to send for each client (gateway can add to the queue),
-      /// 2. keeps listening for messages from clients (sends the gateway the messages),
-      /// 3. handles if client drop/wants to disconnect (informs gateway in either case),
-      /// 4. handles sending disconnect messages to client (gateway can add disconnect message to que)
-      /// can access the store but not change anything in it,
-      /// </summary>
-      class WebSocketHandler {
-
-         private static bool IsSocketStateOpen(WebSocket? ws) => ws != null && ws.State != WebSocketState.Closed && ws.State != WebSocketState.Aborted && ws.State != WebSocketState.None;
-         private static bool IsClosedProperly(WebSocket ws) => ws.CloseStatus == WebSocketCloseStatus.NormalClosure;
-         /// <summary>
-         /// GetConnectedWebSocket attempts to retrieve a connected WebSocket from the WebSocketConnectionStore using the provided client ID.
-         /// If the socket is not found or is not in an open state, it logs an error message and calls the HandleDisconnection method on the 
-         /// WebsocketConnectionGateway.
-         /// </summary>
-         /// <param name="id">The client ID to use to retrieve the WebSocket from the WebSocketConnectionStore.</param>
-         /// <param name="error_message">The error message to log if the WebSocket is not found or not open.</param>
-         /// <returns>The connected WebSocket or null if it was not found or not open.</returns>
-         private WebSocket? GetConnectedWebSocket(WebsocketClientID id, string error_message) {
-            LoggerContext logger = new(Log.Gen, error_message, "");
-            var socket = Connections.GetSocket(id);
-            if (IsSocketStateOpen(socket)) {
-               return socket;
-            }
-            if (socket != null) {
-               logger.warn(" failed as Socket closed");
-            } else {
-               logger.warn(" failed as could not find socket in dictionary");
-            }
-            gtwy.HandleDisconnection(id, error_message);
-            return null;
-         }
-         /// <summary>
-         /// Determines if a WebSocket connection needs to be disconnected
-         /// </summary>
-         /// <param name="id">The unique identifier of the WebSocket client</param>
-         /// <param name="conn">The Connection object of the WebSocket client</param>
-         /// <param name="error_message">The error message to be logged in case of improper connection closure</param>
-         /// <returns>True if the connection needs to be disconnected, False otherwise</returns>
-         public bool ShouldDisconnect(WebsocketClientID id, Connection conn, string error_message) {
-            LoggerContext logger = new(Log.Gen, error_message, "");
-            var socket = conn.Ws;
-            if (IsSocketStateOpen(socket)) {
-               return true;
-            }
-            if (!IsClosedProperly(socket)) {
-               logger.info(" closed improperly by other thread");
-               gtwy.NotifyDisconnection(id);
-            }
-            return false;
-         }
-         private WebsocketConnectionGateway gtwy;
-
-         private WebSocketConnectionStore? connections = null;
-         private WebSocketConnectionStore Connections {
-            get => Log.notnull<WebSocketConnectionStore>(connections);
-            set {
-               bts.Log.isnull(connections);
-               connections = value;
-            }
-         }
-         /// <summary>
-         /// Initializes a new instance of the <see cref="WebSocketHandler"/> class
-         /// </summary>
-         /// <param name="gateway">Websocket connection gateway</param>
-         /// <param name="conns">Connections dictionary</param>
-         public WebSocketHandler(WebsocketConnectionGateway gateway, WebSocketConnectionStore conns) {
-            gtwy = gateway;
-            Connections = conns;
-         }
-         /// <summary>
-         /// Processes a new WebSocket connection asynchronously by first handling the connection with the gateway, 
-         /// then running two tasks concurrently: one for processing incoming requests and one for processing messages in the queue.
-         /// </summary>
-         /// <param name="ws">The WebSocket connection to be processed</param>
-         /// <returns>A task that represents the asynchronous operation</returns>
-         public async Task ProcessWebSocketConnectionAsync(WebSocket ws) {
-            var id = await gtwy.HandleConnection(new Connection(ws), null);
-            var pushTask = Task.Run(() => ProcessWebSocketQueueAsync(id));
-            await ProcessWebSocketRequestAsync(id);
-            await pushTask;
-            // await Task.WhenAll(ProcessWebSocketQueueAsync(id), ProcessWebSocketRequestAsync(id));
-         }
-
-         /// <summary>
-         /// Adds a message to the queue of a specific websocket client.
-         /// </summary>
-         /// <param name="id">The unique identifier of the client</param>
-         /// <param name="mes">The message to be added to the queue</param>
-         /// <returns> True if succesfull otherwise fasle </returns>
-         public bool AddToQueue(WebsocketClientID id, Tuple<MessageType, string> mes) {
-            var que = Connections.GetSocketQueue(id);
-            if (que == null) {
-               return false;
-            }
-            que.Enqueue(mes);
-            return true;
-         }
-         /// <summary>
-         /// Processes the messages in the WebSocket queue for a specific client.
-         /// will keep running until socket is open
-         /// </summary>
-         /// <param name="id">The unique identifier of the client</param>
-         private async Task ProcessWebSocketQueueAsync(WebsocketClientID id) {
-            // Get the socket for the client
-            var socket = GetConnectedWebSocket(id, "WebSocketHandler.ProcessWebSocketQueueAsync could not process");
-            // Get the queue for the client
-            var que = Connections.GetSocketQueue(id);
-            // If either the socket or the queue is null, return
-            if (socket == null || que == null) {
-               return;
-            }
-            while (IsSocketStateOpen(socket)) {
-               // Try to dequeue a message from the queue
-               if (!que.TryDequeue(out var message)) {
-                  // If the queue is empty, wait for 100 milliseconds before trying again
-                  await Task.Delay(TimeSpan.FromMilliseconds(100));
-                  continue;
-               }
-               // If the message is a close request
-               if (message.Item1 == MessageType.Close) {
-                  // Send a disconnect request to the client
-                  await SendDisconnectRequestAsync(id, message.Item2);
-                  // Notify the gateway of the client's disconnection
-                  gtwy.NotifyDisconnection(id);
-                  break;
-               }
-               // If the message is a regular message
-               else if (message.Item1 == MessageType.Message) {
-                  // Send the message to the client
-                  await SendWebSocketRequestAsync(id, message.Item2);
-               }
-            }
-         }
-         /// <summary>
-         /// Sends a disconnect request to a websocket client, with an optional message.
-         /// </summary>
-         /// <param name="id">The unique identifier of the client to send the request to</param>
-         /// <param name="message">An optional message for the logger</param>
-         private async Task SendDisconnectRequestAsync(WebsocketClientID id, string message) {
-            LoggerContext logger = new(Log.Gen, $"WebSocketHandler.SendDisconnectRequestAsync tried closing client_id: {id} with message: {message}", "");
-            // Get the socket of the client
-            var sckt = Connections.GetSocket(id);
-            if (sckt == null) {
-               logger.info("  failed as could not find socket in dictionary");
-               return;
-            }
-            try {
-               // send a close message to the socket, using the close status 1000, which is a normal closure status
-               await sckt.CloseAsync((WebSocketCloseStatus)1000, "", CancellationToken.None);
-               logger.info(" done succesfully");
-            } catch (WebSocketException ex) {
-               logger.info($" failed as client didnt respond, Exception: {ex.Message}");
-            }
-         }
-         /// <summary>
-         /// recieves messages from a specific websocket client identified by its id. keeps running until socket remains open
-         /// </summary>
-         /// <param name="id">The unique identifier of the client</param>
-         private async Task ProcessWebSocketRequestAsync(WebsocketClientID id) {
-            // Get the socket for the given client ID
-            var socket = GetConnectedWebSocket(id, "WebSocketHandler.ProcessWebSocketRequestAsync could not process");
-            // If the socket is null, return
-            if (socket == null) {
-               return;
-            }
-            try {
-               // Create a buffer to receive the incoming message
-               ArraySegment<byte> buffer = new ArraySegment<byte>(new byte[256]);
-               WebSocketReceiveResult result;
-               string request = "";
-               while (IsSocketStateOpen(socket)) {
-                  // Receive the message
-                  result = await socket.ReceiveAsync(buffer, CancellationToken.None);
-                  // If the message is a close request, handle the disconnection and return
-                  if (result.MessageType == WebSocketMessageType.Close) {
-                     var log_txt = $"WebSocketHandler.ProcessWebSocketRequestAsync close request";
-                     gtwy.HandleDisconnection(id, log_txt);
-                     return;
-                  }
-                  if (buffer.Array != null)// added because of warning 
-                  {
-                     // Append the received message to the request string
-                     request += Encoding.UTF8.GetString(buffer.Array, 0, result.Count);
-                  } else {
-                     bts.Log.assert(false);//this should never happen
-                  }
-                  // If the message is the end of the message, handle the message and reset the request string
-                  if (result.EndOfMessage) {
-                     gtwy.HandleMessage(new WebsocketIncomingMessage(id, request));
-                     request = "";
-                  }
-               }
-
-            } catch (WebSocketException ex) {
-               var ex_str = ex.ToString();
-               var log_txt = $"WebsocketGateway.ProcessWebSocketRequestAsync connection closed Error: {ex_str}";
-               gtwy.HandleDisconnection(id, log_txt);
-            } catch (OperationCanceledException) {//not sure about this, read that this should just be ignored
-            }
-         }
-         /// <summary>
-         /// Sends a message to a specific websocket client identified by its id.
-         /// </summary>
-         /// <param name="id">The unique identifier of the client to receive the message</param>
-         /// <param name="message">The message to be sent</param>
-         public async Task SendWebSocketRequestAsync(WebsocketClientID id, string message) {
-            // Get the socket of the client
-            var socket = GetConnectedWebSocket(id, "WebSocketHandler.SendWebSocketRequestAsync could not send Request");
-            if (socket == null) {
-               return;
-            }
-            try {
-               // Send the message
-               await socket.SendAsync(Encoding.ASCII.GetBytes(message), WebSocketMessageType.Text, true, CancellationToken.None);
-            } catch (WebSocketException ex) {
-               // Handle the exception and disconnect the client
-               var ex_str = ex.ToString();
-               var log_txt = $"WebSocketHandler.SendWebSocketRequestAsync connection closed client_id: {id} Error: {ex_str}";
-               gtwy.HandleDisconnection(id, log_txt);
-            }
-         }
-
-
-      }
-
-      public event Action<WebsocketClientID>? Disconnected;
-      public event Action<WebsocketClientID, WebsocketIncomingMessage?>? Connected;
-      public event Action<WebsocketIncomingMessage>? MessageReceived;
-
-      private WebApplication? app = null;
-      private WebsocketServerConfig? _server_params = null;
-      private WebSocketConnectionStore Connections { get; } = new();
-      /// <summary>
-      /// creates a new connections dictionary and a new WebSocketHandler.
-      /// </summary>
-      private WebSocketHandler _WebSocketHandler { get; }
-      public WebsocketConnectionGateway() {
-         _WebSocketHandler = new WebSocketHandler(this, Connections);
-      }
-      /// <summary>
-      /// Creates a unique identifier for a websocket client and adds the client's connection to the list of active connections.
-      /// </summary>
-      /// <param name="conn">The connection of the client</param>
-      /// <returns>A unique identifier for the client</returns>
-      private WebsocketClientID createID(Connection conn) {
-         var id = new WebsocketClientID();
-         Connections.AddConnection(id, conn);
-         return id;
-      }
-      /// <summary>
-      /// forwards an incoming message from a websocket client to sync.
-      /// </summary>
-      /// <param name="message">The incoming message to be handled</param>
-      public void HandleMessage(WebsocketIncomingMessage message) =>
-         MessageReceived?.Invoke(message);
-
-      /// <summary>
-      /// Notifies of a client disconnection and removes the client's connection from the list of active connections.
-      /// </summary>
-      /// <param name="id">The unique identifier of the disconnected client</param>
-      public void NotifyDisconnection(WebsocketClientID id) {
-         Disconnected?.Invoke(id);
-         Connections.RemoveConnection(id);
-      }
-
-      /// <summary>
-      /// Handles a new websocket connection and returns a unique identifier for the client.
-      /// </summary>
-      /// <param name="conn">The new connection to be handled</param>
-      /// <param name="optional_message">An optional message associated with the connection</param>
-      /// <returns>A unique identifier for the client</returns>
-      public async Task<WebsocketClientID> HandleConnection(Connection conn, WebsocketIncomingMessage? optional_message) {
-         var id = createID(conn);
-         await Task.Run(() => Connected?.Invoke(id, optional_message));
-         return id;
-      }
-
-      /// <summary>
-      /// Handles disconnection of a websocket client.
-      /// </summary>
-      /// <param name="id">The unique identifier of the client to be disconnected</param>
-      /// <param name="optional_message">An optional message to add to the logger</param>
-      public void HandleDisconnection(WebsocketClientID id, string optional_message = "") {
-         LoggerContext logger = new(Log.Gen, $"WebsocketGateway.HandleDisconnection tried closing client_id: {id} with message: {optional_message}", "");
-
-         // Get the connection of the client
-         var conn = Connections.GetConnection(id);
-         if (conn == null) {
-            // If the client connection is not found, log the error and notify sync about the disconnection through the Sync object
-            logger.warn(" could not find socket in dictionary");
-            Disconnected?.Invoke(id);
-            return;
-         }
-         // Check if the connection needs to be disconnected
-         if (!_WebSocketHandler.ShouldDisconnect(id, conn, $"WebsocketGateway.HandleDisconnection tried closing client_id: {id}")) {
-            // If the connection does not need to be disconnected, remove it from the Connections dictionary
-            Connections.RemoveConnection(id);
-            return;
-         }
-         // Add the disconnection message to the WebSocketHandler's message queue
-         if (_WebSocketHandler.AddToQueue(id, Tuple.Create(MessageType.Close, "")))
-            logger.info(" added to que");
-      }
-      /// <summary>
-      /// Starts listening for incoming websocket connections. need to call init before this
-      /// </summary>
-      /// <param name="server_params">The parameters for the websocket server includind the URL and endpoint,</param>
-      public Task start_listening(WebsocketServerConfig server_params) {
-         // start listening, assert that init has been called already
-         bts.Log.isnull(_server_params);
-         bts.Log.isnull(app);
-
-
-         _server_params = server_params;
-         var builder = WebApplication.CreateBuilder();
-
-         var localServerParams = _server_params.Clone();
-         localServerParams.Scheme = "http";
-         var localUri = localServerParams.Uri();
-         builder.WebHost.UseUrls(localUri);
-
-         var publicServerParams = _server_params.Clone();
-         publicServerParams.Scheme = "https";
-         publicServerParams.Host = "192.168.50.94";
-         var publicUri = publicServerParams.Uri();
-         builder.WebHost.UseUrls(publicUri);
-
-         // var MyAllowSpecificOrigins = "_myAllowSpecificOrigins";
-         builder.Services.AddLogging(builder => {
-            builder.AddFilter("Microsoft", Microsoft.Extensions.Logging.LogLevel.Warning)
-               .AddFilter("System", Microsoft.Extensions.Logging.LogLevel.Warning)
-               .AddFilter("NToastNotify", Microsoft.Extensions.Logging.LogLevel.Warning)
-               .AddConsole();
-         });
-         // builder.Services.AddCors(options => {
-         //    options.AddPolicy(MyAllowSpecificOrigins,
-         //                          policy => policy
-         //                            .WithOrigins("http://localhost:3000")
-         //                            .AllowAnyHeader()
-         //                            .AllowAnyMethod());
-         // });
-         app = builder.Build();
-         app.UseWebSockets(_server_params.get_ws_options());
-         // app.Use(async (context, next) => {
-         //    context.Response.Headers.Add("Access-Control-Allow-Origin", "*");
-         //    context.Response.Headers.Add("Access-Control-Allow-Headers", "*");
-         //    await next();
-         // });
-         app.Map(_server_params.DefaultEndPoint, async context => {
-            if (context.WebSockets.IsWebSocketRequest) {
-               using (var ws = await context.WebSockets.AcceptWebSocketAsync()) {
-                  await _WebSocketHandler.ProcessWebSocketConnectionAsync(ws);
-               }
-            } else {
-               context.Response.StatusCode = (int)System.Net.HttpStatusCode.BadRequest;
-            }
-         });
-         return app.RunAsync();
-      }
-
-      /// <summary>
-      /// Sends a message to a list of websocket clients.
-      /// </summary>
-      /// <param name="messages">A collection of websocket messages to be sent to the clients</param>
-      public void send_to_clients(
-         IEnumerable<WebsocketOutgoingMessage> messages) {
-         foreach (var om in messages) {
-            // Add the message to the WebSocketHandler's message queue
-            if (!_WebSocketHandler.AddToQueue(om.receiver_client_id, Tuple.Create(MessageType.Message, om.message_body)))
-               break;
-         }
-      }
-   }
-}
+using Bts.Websocket;
+
+using System.Net.WebSockets;
+using System.Text;
+using System.Collections.Concurrent;
+using Microsoft.AspNetCore.Builder;
+using Microsoft.AspNetCore.Hosting;
+using Microsoft.Extensions.DependencyInjection;
+using Microsoft.Extensions.Logging;
+
+namespace bts.websocket_core {
+   public enum MessageType {
+      Message,
+      Close
+   }
+   /// <summary>
+   /// gateway between sync and socket logic
+   /// 1. keeps all active clients in the store(updates store if sync or handler informs)
+   /// 2. listens for connections (upon connection passes to handler)
+   /// 3. gets messages from sync forwards to handler
+   /// 4. updates store and sync about any changes in connections (when told by handler)
+   /// </summary>
+   public class WebsocketConnectionGateway : IWebsocketConnectionGateway {
+      /// <summary>
+      /// Represents a connection to a client through a WebSocket. 
+      /// </summary>
+      public class Connection {
+         /// <summary>
+         /// The WebSocket instance associated with this connection.
+         /// </summary>
+         public WebSocket Ws { get; }
+         /// <summary>
+         /// A queue that holds outgoing messages to be sent to the client.
+         /// </summary>
+         public ConcurrentQueue<Tuple<MessageType, string>> OutgoingQueue { get; } = new();
+         /// <summary>
+         /// Initializes a new instance of the <see cref="Connection"/> class.
+         /// </summary>
+         /// <param name="websocket">The WebSocket associated with this connection.</param>
+         public Connection(WebSocket websocket) {
+            Ws = websocket;
+         }
+      }
+
+
+      /// <summary>
+      /// wrapper for concurrent Dictionary
+      /// Stores all the alive connection
+      /// Additions and removals done by gateway
+      /// websockethandler can access but not change anything
+      /// </summary>
+      class WebSocketConnectionStore {
+         private ConcurrentDictionary<WebsocketClientID, Connection> Connections { get; } = new();
+         /// <summary>
+         /// Adds a new connection to the connections dictionary using the given id and connection object
+         /// </summary>
+         /// <param name="id">The id to use as the key for the new connection</param>
+         /// <param name="conn">The connection object to add to the dictionary</param>
+         public void AddConnection(WebsocketClientID id, Connection conn) {
+            if (!Connections.TryAdd(id, conn)) {
+               var log_txt = $"WebsocketDataSync.createID client_id: {id.ToString()} Already exits";
+               bts.Log.Gen.warn(log_txt);
+               Connections[id] = conn;
+            }
+         }
+         /// <summary>
+         /// Removes a connection from the connections dictionary using the given id
+         /// </summary>
+         /// <param name="id">The id of the connection to remove</param>
+         public void RemoveConnection(WebsocketClientID id) {
+            if (Connections.TryRemove(id, out Connection? conn)) {
+               conn.Ws.Dispose();
+            }
+         }
+         /// <summary>
+         /// Retrieves the Connection object associated with the given id
+         /// </summary>
+         /// <param name="id">The id of the connection to retrieve</param>
+         /// <returns>The Connection object associated with the given id, or null if the id is not found</returns>
+         public Connection? GetConnection(WebsocketClientID id) => Connections.GetValueOrDefault(id);
+         /// <summary>
+         /// Retrieves the Connection object associated with the given id
+         /// </summary>
+         /// <param name="id">The id of the connection to retrieve</param>
+         /// <returns>The Connection object associated with the given id, or null if the id is not found</returns>
+         public WebSocket? GetSocket(WebsocketClientID id) => Connections.GetValueOrDefault(id)?.Ws;
+         /// <summary>
+         /// Get the outgoing message queue for a specific client connection identified by the given id.
+         /// </summary>
+         /// <param name="id">The unique identifier for the client connection.</param>
+         /// <returns>A concurrent queue of message tuples if the connection is found, null otherwise.</returns>
+         public ConcurrentQueue<Tuple<MessageType, string>>? GetSocketQueue(WebsocketClientID id) =>
+         Connections.GetValueOrDefault(id)?.OutgoingQueue;
+         /// <summary>
+         /// Check if the store contains a connection with the specified id.
+         /// </summary>
+         /// <param name="id">The unique identifier for the client connection.</param>
+         /// <returns>True if the connection is found, false otherwise.</returns>
+         public bool ContainsKey(WebsocketClientID id) => Connections.ContainsKey(id);
+      }
+
+      /// <summary>
+      /// is controlled by gateway: 
+      /// once a new connection is made the gateway passes the socket to this and this
+      /// handles everything: 
+      /// 1. keeps a queue of messages to send for each client (gateway can add to the queue),
+      /// 2. keeps listening for messages from clients (sends the gateway the messages),
+      /// 3. handles if client drop/wants to disconnect (informs gateway in either case),
+      /// 4. handles sending disconnect messages to client (gateway can add disconnect message to que)
+      /// can access the store but not change anything in it,
+      /// </summary>
+      class WebSocketHandler {
+         /// <summary>
+         /// GetConnectedWebSocket attempts to retrieve a connected WebSocket from the WebSocketConnectionStore using the provided client ID.
+         /// If the socket is not found or is not in an open state, it logs an error message and calls the HandleDisconnection method on the 
+         /// WebsocketConnectionGateway.
+         /// </summary>
+         /// <param name="id">The client ID to use to retrieve the WebSocket from the WebSocketConnectionStore.</param>
+         /// <param name="error_message">The error message to log if the WebSocket is not found or not open.</param>
+         /// <returns>The connected WebSocket or null if it was not found or not open.</returns>
+         private WebSocket? GetConnectedWebSocket(WebsocketClientID id, string error_message) {
+            var socket = Connections.GetSocket(id);
+            if (socket?.IsSocketStateOpen() ?? false) {
+               return socket;
+            }
+            Log.Gen.warn(
+               $"GetConnectedWebsocket failed: {(socket is null ?
+                  "could not find socket in dictionary" :
+                  "socket is closed")}");
+            gtwy.HandleDisconnection(id, error_message);
+            return null;
+         }
+         /// <summary>
+         /// Determines if a WebSocket connection needs to be disconnected
+         /// </summary>
+         /// <param name="id">The unique identifier of the WebSocket client</param>
+         /// <param name="conn">The Connection object of the WebSocket client</param>
+         /// <param name="error_message">The error message to be logged in case of improper connection closure</param>
+         /// <returns>True if the connection needs to be disconnected, False otherwise</returns>
+         public bool ShouldDisconnect(WebsocketClientID id, Connection conn, string error_message) {
+            LoggerContext logger = new(Log.Gen, error_message, "");
+            var socket = conn.Ws;
+            if (socket is null) {
+               return false;
+            }
+            if (socket.IsSocketStateOpen()) {
+               return true;
+            }
+            if (!socket.IsClosedProperly()) {
+               logger.info(" closed improperly by other thread");
+               gtwy.NotifyDisconnection(id);
+            }
+            return false;
+         }
+         private WebsocketConnectionGateway gtwy;
+
+         private WebSocketConnectionStore? connections = null;
+         private WebSocketConnectionStore Connections {
+            get => Log.notnull<WebSocketConnectionStore>(connections);
+            set {
+               bts.Log.isnull(connections);
+               connections = value;
+            }
+         }
+         /// <summary>
+         /// Initializes a new instance of the <see cref="WebSocketHandler"/> class
+         /// </summary>
+         /// <param name="gateway">Websocket connection gateway</param>
+         /// <param name="conns">Connections dictionary</param>
+         public WebSocketHandler(WebsocketConnectionGateway gateway, WebSocketConnectionStore conns) {
+            gtwy = gateway;
+            Connections = conns;
+         }
+         /// <summary>
+         /// Processes a new WebSocket connection asynchronously by first handling the connection with the gateway, 
+         /// then running two tasks concurrently: one for processing incoming requests and one for processing messages in the queue.
+         /// </summary>
+         /// <param name="ws">The WebSocket connection to be processed</param>
+         /// <returns>A task that represents the asynchronous operation</returns>
+         public async Task ProcessWebSocketConnectionAsync(WebSocket ws) {
+            var id = await gtwy.HandleConnection(new Connection(ws), null);
+            var pushTask = Task.Run(() => ProcessWebSocketQueueAsync(id));
+            await ProcessWebSocketRequestAsync(id);
+            await pushTask;
+            // await Task.WhenAll(ProcessWebSocketQueueAsync(id), ProcessWebSocketRequestAsync(id));
+         }
+
+         /// <summary>
+         /// Adds a message to the queue of a specific websocket client.
+         /// </summary>
+         /// <param name="id">The unique identifier of the client</param>
+         /// <param name="mes">The message to be added to the queue</param>
+         /// <returns> True if succesfull otherwise fasle </returns>
+         public bool AddToQueue(WebsocketClientID id, Tuple<MessageType, string> mes) {
+            var que = Connections.GetSocketQueue(id);
+            if (que == null) {
+               return false;
+            }
+            que.Enqueue(mes);
+            return true;
+         }
+         /// <summary>
+         /// Processes the messages in the WebSocket queue for a specific client.
+         /// will keep running until socket is open
+         /// </summary>
+         /// <param name="id">The unique identifier of the client</param>
+         private async Task ProcessWebSocketQueueAsync(WebsocketClientID id) {
+            // Get the socket for the client
+            var socket = GetConnectedWebSocket(id, "WebSocketHandler.ProcessWebSocketQueueAsync could not process");
+            // Get the queue for the client
+            var que = Connections.GetSocketQueue(id);
+            // If either the socket or the queue is null, return
+            if (socket == null || que == null) {
+               return;
+            }
+            while (IsSocketStateOpen(socket)) {
+               // Try to dequeue a message from the queue
+               if (!que.TryDequeue(out var message)) {
+                  // If the queue is empty, wait for 100 milliseconds before trying again
+                  await Task.Delay(TimeSpan.FromMilliseconds(100));
+                  continue;
+               }
+               // If the message is a close request
+               if (message.Item1 == MessageType.Close) {
+                  // Send a disconnect request to the client
+                  await SendDisconnectRequestAsync(id, message.Item2);
+                  // Notify the gateway of the client's disconnection
+                  gtwy.NotifyDisconnection(id);
+                  break;
+               }
+               // If the message is a regular message
+               else if (message.Item1 == MessageType.Message) {
+                  // Send the message to the client
+                  await SendWebSocketRequestAsync(id, message.Item2);
+               }
+            }
+         }
+         /// <summary>
+         /// Sends a disconnect request to a websocket client, with an optional message.
+         /// </summary>
+         /// <param name="id">The unique identifier of the client to send the request to</param>
+         /// <param name="message">An optional message for the logger</param>
+         private async Task SendDisconnectRequestAsync(WebsocketClientID id, string message) {
+            LoggerContext logger = new(Log.Gen, $"WebSocketHandler.SendDisconnectRequestAsync tried closing client_id: {id} with message: {message}", "");
+            // Get the socket of the client
+            var sckt = Connections.GetSocket(id);
+            if (sckt == null) {
+               logger.info("  failed as could not find socket in dictionary");
+               return;
+            }
+            try {
+               // send a close message to the socket, using the close status 1000, which is a normal closure status
+               await sckt.CloseAsync((WebSocketCloseStatus)1000, "", CancellationToken.None);
+               logger.info(" done succesfully");
+            } catch (WebSocketException ex) {
+               logger.info($" failed as client didnt respond, Exception: {ex.Message}");
+            }
+         }
+         /// <summary>
+         /// recieves messages from a specific websocket client identified by its id. keeps running until socket remains open
+         /// </summary>
+         /// <param name="id">The unique identifier of the client</param>
+         private async Task ProcessWebSocketRequestAsync(WebsocketClientID id) {
+            // Get the socket for the given client ID
+            var socket = GetConnectedWebSocket(id, "WebSocketHandler.ProcessWebSocketRequestAsync could not process");
+            // If the socket is null, return
+            if (socket == null) {
+               return;
+            }
+            try {
+               // Create a buffer to receive the incoming message
+               ArraySegment<byte> buffer = new ArraySegment<byte>(new byte[256]);
+               WebSocketReceiveResult result;
+               string request = "";
+               while (IsSocketStateOpen(socket)) {
+                  // Receive the message
+                  result = await socket.ReceiveAsync(buffer, CancellationToken.None);
+                  // If the message is a close request, handle the disconnection and return
+                  if (result.MessageType == WebSocketMessageType.Close) {
+                     var log_txt = $"WebSocketHandler.ProcessWebSocketRequestAsync close request";
+                     gtwy.HandleDisconnection(id, log_txt);
+                     return;
+                  }
+                  if (buffer.Array != null)// added because of warning 
+                  {
+                     // Append the received message to the request string
+                     request += Encoding.UTF8.GetString(buffer.Array, 0, result.Count);
+                  } else {
+                     bts.Log.assert(false);//this should never happen
+                  }
+                  // If the message is the end of the message, handle the message and reset the request string
+                  if (result.EndOfMessage) {
+                     gtwy.HandleMessage(new WebsocketIncomingMessage(id, request));
+                     request = "";
+                  }
+               }
+
+            } catch (WebSocketException ex) {
+               var ex_str = ex.ToString();
+               var log_txt = $"WebsocketGateway.ProcessWebSocketRequestAsync connection closed Error: {ex_str}";
+               gtwy.HandleDisconnection(id, log_txt);
+            } catch (OperationCanceledException) {//not sure about this, read that this should just be ignored
+            }
+         }
+         /// <summary>
+         /// Sends a message to a specific websocket client identified by its id.
+         /// </summary>
+         /// <param name="id">The unique identifier of the client to receive the message</param>
+         /// <param name="message">The message to be sent</param>
+         public async Task SendWebSocketRequestAsync(WebsocketClientID id, string message) {
+            // Get the socket of the client
+            var socket = GetConnectedWebSocket(id, "WebSocketHandler.SendWebSocketRequestAsync could not send Request");
+            if (socket == null) {
+               return;
+            }
+            try {
+               // Send the message
+               await socket.SendAsync(Encoding.ASCII.GetBytes(message), WebSocketMessageType.Text, true, CancellationToken.None);
+            } catch (WebSocketException ex) {
+               // Handle the exception and disconnect the client
+               var ex_str = ex.ToString();
+               var log_txt = $"WebSocketHandler.SendWebSocketRequestAsync connection closed client_id: {id} Error: {ex_str}";
+               gtwy.HandleDisconnection(id, log_txt);
+            }
+         }
+
+
+      }
+
+      public event Action<WebsocketClientID>? Disconnected;
+      public event Action<WebsocketClientID, WebsocketIncomingMessage?>? Connected;
+      public event Action<WebsocketIncomingMessage>? MessageReceived;
+
+      private WebApplication? app = null;
+      private WebsocketServerConfig? _server_params = null;
+      private WebSocketConnectionStore Connections { get; } = new();
+      /// <summary>
+      /// creates a new connections dictionary and a new WebSocketHandler.
+      /// </summary>
+      private WebSocketHandler _WebSocketHandler { get; }
+      public WebsocketConnectionGateway() {
+         _WebSocketHandler = new WebSocketHandler(this, Connections);
+      }
+      /// <summary>
+      /// Creates a unique identifier for a websocket client and adds the client's connection to the list of active connections.
+      /// </summary>
+      /// <param name="conn">The connection of the client</param>
+      /// <returns>A unique identifier for the client</returns>
+      private WebsocketClientID createID(Connection conn) {
+         var id = new WebsocketClientID();
+         Connections.AddConnection(id, conn);
+         return id;
+      }
+      /// <summary>
+      /// forwards an incoming message from a websocket client to sync.
+      /// </summary>
+      /// <param name="message">The incoming message to be handled</param>
+      public void HandleMessage(WebsocketIncomingMessage message) =>
+         MessageReceived?.Invoke(message);
+
+      /// <summary>
+      /// Notifies of a client disconnection and removes the client's connection from the list of active connections.
+      /// </summary>
+      /// <param name="id">The unique identifier of the disconnected client</param>
+      public void NotifyDisconnection(WebsocketClientID id) {
+         Disconnected?.Invoke(id);
+         Connections.RemoveConnection(id);
+      }
+
+      /// <summary>
+      /// Handles a new websocket connection and returns a unique identifier for the client.
+      /// </summary>
+      /// <param name="conn">The new connection to be handled</param>
+      /// <param name="optional_message">An optional message associated with the connection</param>
+      /// <returns>A unique identifier for the client</returns>
+      public async Task<WebsocketClientID> HandleConnection(Connection conn, WebsocketIncomingMessage? optional_message) {
+         var id = createID(conn);
+         await Task.Run(() => Connected?.Invoke(id, optional_message));
+         return id;
+      }
+
+      /// <summary>
+      /// Handles disconnection of a websocket client.
+      /// </summary>
+      /// <param name="id">The unique identifier of the client to be disconnected</param>
+      /// <param name="optional_message">An optional message to add to the logger</param>
+      public void HandleDisconnection(WebsocketClientID id, string optional_message = "") {
+         LoggerContext logger = new(Log.Gen, $"WebsocketGateway.HandleDisconnection tried closing client_id: {id} with message: {optional_message}", "");
+
+         // Get the connection of the client
+         var conn = Connections.GetConnection(id);
+         if (conn == null) {
+            // If the client connection is not found, log the error and notify sync about the disconnection through the Sync object
+            logger.warn(" could not find socket in dictionary");
+            Disconnected?.Invoke(id);
+            return;
+         }
+         // Check if the connection needs to be disconnected
+         if (!_WebSocketHandler.ShouldDisconnect(id, conn, $"WebsocketGateway.HandleDisconnection tried closing client_id: {id}")) {
+            // If the connection does not need to be disconnected, remove it from the Connections dictionary
+            Connections.RemoveConnection(id);
+            return;
+         }
+         // Add the disconnection message to the WebSocketHandler's message queue
+         if (_WebSocketHandler.AddToQueue(id, Tuple.Create(MessageType.Close, "")))
+            logger.info(" added to que");
+      }
+      /// <summary>
+      /// Starts listening for incoming websocket connections. need to call init before this
+      /// </summary>
+      /// <param name="server_params">The parameters for the websocket server includind the URL and endpoint,</param>
+      public Task start_listening(WebsocketServerConfig server_params) {
+         // start listening, assert that init has been called already
+         bts.Log.isnull(_server_params);
+         bts.Log.isnull(app);
+
+
+         _server_params = server_params;
+         var builder = WebApplication.CreateBuilder();
+
+         var localServerParams = _server_params.Clone();
+         localServerParams.Scheme = "http";
+         var localUri = localServerParams.Uri();
+         builder.WebHost.UseUrls(localUri);
+
+         var publicServerParams = _server_params.Clone();
+         publicServerParams.Scheme = "https";
+         publicServerParams.Host = "192.168.50.94";
+         var publicUri = publicServerParams.Uri();
+         builder.WebHost.UseUrls(publicUri);
+
+         // var MyAllowSpecificOrigins = "_myAllowSpecificOrigins";
+         builder.Services.AddLogging(builder => {
+            builder.AddFilter("Microsoft", Microsoft.Extensions.Logging.LogLevel.Warning)
+               .AddFilter("System", Microsoft.Extensions.Logging.LogLevel.Warning)
+               .AddFilter("NToastNotify", Microsoft.Extensions.Logging.LogLevel.Warning)
+               .AddConsole();
+         });
+         // builder.Services.AddCors(options => {
+         //    options.AddPolicy(MyAllowSpecificOrigins,
+         //                          policy => policy
+         //                            .WithOrigins("http://localhost:3000")
+         //                            .AllowAnyHeader()
+         //                            .AllowAnyMethod());
+         // });
+         app = builder.Build();
+         app.UseWebSockets(_server_params.get_ws_options());
+         // app.Use(async (context, next) => {
+         //    context.Response.Headers.Add("Access-Control-Allow-Origin", "*");
+         //    context.Response.Headers.Add("Access-Control-Allow-Headers", "*");
+         //    await next();
+         // });
+         app.Map(_server_params.DefaultEndPoint, async context => {
+            if (context.WebSockets.IsWebSocketRequest) {
+               using (var ws = await context.WebSockets.AcceptWebSocketAsync()) {
+                  await _WebSocketHandler.ProcessWebSocketConnectionAsync(ws);
+               }
+            } else {
+               context.Response.StatusCode = (int)System.Net.HttpStatusCode.BadRequest;
+            }
+         });
+         return app.RunAsync();
+      }
+
+      /// <summary>
+      /// Sends a message to a list of websocket clients.
+      /// </summary>
+      /// <param name="messages">A collection of websocket messages to be sent to the clients</param>
+      public void send_to_clients(
+         IEnumerable<WebsocketOutgoingMessage> messages) {
+         foreach (var om in messages) {
+            // Add the message to the WebSocketHandler's message queue
+            if (!_WebSocketHandler.AddToQueue(om.receiver_client_id, Tuple.Create(MessageType.Message, om.message_body)))
+               break;
+         }
+      }
+   }
+}

src/client/spark_websocket/Core/WebsocketExtensions.cs

@@ -0,0 +1,14 @@
+namespace Bts.Websocket;
+
+using System.Net.WebSockets;
+
+public static class WebsocketExtensions {
+   public static bool IsSocketStateOpen(this WebSocket ws) =>
+      ws.State switch {
+         WebSocketState.None or WebSocketState.Closed or WebSocketState.Aborted => false,
+         _ => true
+      };
+
+   public static bool IsClosedProperly(this WebSocket ws) =>
+      ws.CloseStatus == WebSocketCloseStatus.NormalClosure;
+}

src/client/spark_websocket/Core/WebsocketGatewayObservable.cs

@@ -1,108 +1,103 @@
-namespace Bts.Websocket;
-
-using bts;
-using bts.websocket_core;
-
-using mbts;
-
-using System;
-using System.Reactive.Subjects;
-
-/// <summary>
-/// Represents an I/O model that provides websocket requests from clients as output, and accepts
-/// data/notifications to send back to the clients as input. This class doesn't interpret clients'
-/// requests; it doesn't subscribe to data on their behalf. It emits those requests as output so
-/// someone else can.
-/// </summary>
-public class WebsocketGatewayObservable : IConnectableObservable<WebsocketConnectionUpdate>, ISubject<IEnumerable<WebsocketOutgoingMessage>, WebsocketConnectionUpdate>, IDisposable {
-   readonly WebsocketServerConfig config;
-   readonly object lockObj = new();
-   IWebsocketConnectionGateway? gatewayBacking;
-
-   public WebsocketGatewayObservable(WebsocketServerConfig config)
-      : this(new WebsocketConnectionGateway(), config) { }
-
-   public WebsocketGatewayObservable(
-      IWebsocketConnectionGateway gateway, WebsocketServerConfig config) {
-      gatewayBacking = gateway;
-      this.config = config;
-   }
-
-   IWebsocketConnectionGateway gateway =>
-      /// The gateway is (only) set to null in <see cref="Dispose"/>
-      gatewayBacking ?? throw new ObjectDisposedException($"{GetType()}");
-
-   public IDisposable Connect() {
-      IWebsocketConnectionGateway? gw;
-      lock (lockObj) {
-         gw = gateway;
-         ConnectAsync().ignore();
-         return this;
-      }
-   }
-
-   public async Task ConnectAsync() {
-      var gw = gateway;
-      try {
-         await gw.start_listening(config);
-      } catch (Exception ex) {
-         Log.Api.error($"Error running {typeof(IWebsocketConnectionGateway)}: {ex}");
-         throw;
-      }
-   }
-
-   public void OnCompleted() =>
-      Log.Api.warn($"{GetType()} outgoing message stream completed.");
-
-   public void OnError(Exception error) =>
-      Log.Api.error($"{GetType()} detected error in outgoing message stream: {error}");
-
-   public void OnNext(IEnumerable<WebsocketOutgoingMessage> outMessages) =>
-      gateway.send_to_clients(outMessages);         
-
-   public IDisposable Subscribe(IObserver<WebsocketConnectionUpdate> observer) {
-      lock (lockObj) {
-         return new Subscription(observer, gateway);
-      }
-   }
-
-   public void Dispose() {
-      lock (lockObj) {
-         (gatewayBacking as IDisposable)?.Dispose();
-         gatewayBacking = null;
-      }
-   }
-
-   class Subscription : IDisposable {
-      readonly IObserver<WebsocketConnectionUpdate> observer;
-      readonly IWebsocketConnectionGateway gateway;
-
-      public Subscription(
-         IObserver<WebsocketConnectionUpdate> observer, IWebsocketConnectionGateway gateway) {
-         this.observer = observer;
-         this.gateway = gateway;
-         gateway.Connected += onConnected;
-         gateway.Disconnected += onDisconnected;
-         gateway.MessageReceived += onMessageReceived;
-      }
-
-      public void Dispose() {
-         gateway.Connected -= onConnected;
-         gateway.Disconnected -= onDisconnected;
-         gateway.MessageReceived -= onMessageReceived;
-      }
-
-      void onConnected(WebsocketClientID id, WebsocketIncomingMessage? message) {
-         observer.OnNext(WebsocketConnectionUpdate.Create(
-            id, new WebsocketConnectNotification(id, message)));
-      }
-
-      void onDisconnected(WebsocketClientID id) =>
-         observer.OnNext(WebsocketConnectionUpdate.Create(
-            id, new WebsocketDisconnectNotification(id)));
-
-      void onMessageReceived(WebsocketIncomingMessage message) =>
-         observer.OnNext(WebsocketConnectionUpdate.Create(
-            message.sender_client_id, message));
-   }
-}
+namespace Bts.Websocket;
+
+using bts;
+using bts.websocket_core;
+
+using mbts;
+
+using System;
+using System.Reactive.Subjects;
+
+/// <summary>
+/// Represents an I/O model that provides websocket requests from clients as output, and accepts
+/// data/notifications to send back to the clients as input. This class doesn't interpret clients'
+/// requests; it doesn't subscribe to data on their behalf. It emits those requests as output so
+/// someone else can.
+/// </summary>
+public class WebsocketGatewayObservable : IConnectableObservable<WebsocketConnectionUpdate>, ISubject<IEnumerable<WebsocketOutgoingMessage>, WebsocketConnectionUpdate>, IDisposable {
+   readonly WebsocketServerConfig config;
+   readonly object lockObj = new();
+   IWebsocketConnectionGateway? gatewayBacking;
+
+   public WebsocketGatewayObservable(WebsocketServerConfig config)
+      : this(new WebsocketConnectionGateway(), config) { }
+
+   public WebsocketGatewayObservable(
+      IWebsocketConnectionGateway gateway, WebsocketServerConfig config) {
+      gatewayBacking = gateway;
+      this.config = config;
+   }
+
+   IWebsocketConnectionGateway gateway =>
+      /// The gateway is (only) set to null in <see cref="Dispose"/>
+      gatewayBacking ?? throw new ObjectDisposedException($"{GetType()}");
+
+   public IDisposable Connect() {
+      ConnectAsync().Wait();
+      return this;
+   }
+
+   public async Task ConnectAsync() {
+      try {
+         await gateway.start_listening(config);
+      } catch (Exception ex) {
+         Log.Api.error($"Error running {typeof(IWebsocketConnectionGateway)}: {ex}");
+         throw;
+      }
+   }
+
+   public void OnCompleted() =>
+      Log.Api.warn($"{GetType()} outgoing message stream completed.");
+
+   public void OnError(Exception error) =>
+      Log.Api.error($"{GetType()} detected error in outgoing message stream: {error}");
+
+   public void OnNext(IEnumerable<WebsocketOutgoingMessage> outMessages) =>
+      gateway.send_to_clients(outMessages);         
+
+   public IDisposable Subscribe(IObserver<WebsocketConnectionUpdate> observer) {
+      lock (lockObj) {
+         return new Subscription(observer, gateway);
+      }
+   }
+
+   public void Dispose() {
+      lock (lockObj) {
+         (gatewayBacking as IDisposable)?.Dispose();
+         gatewayBacking = null;
+      }
+   }
+
+   class Subscription : IDisposable {
+      readonly IObserver<WebsocketConnectionUpdate> observer;
+      readonly IWebsocketConnectionGateway gateway;
+
+      public Subscription(
+         IObserver<WebsocketConnectionUpdate> observer, IWebsocketConnectionGateway gateway) {
+         this.observer = observer;
+         this.gateway = gateway;
+         gateway.Connected += onConnected;
+         gateway.Disconnected += onDisconnected;
+         gateway.MessageReceived += onMessageReceived;
+      }
+
+      public void Dispose() {
+         gateway.Connected -= onConnected;
+         gateway.Disconnected -= onDisconnected;
+         gateway.MessageReceived -= onMessageReceived;
+      }
+
+      void onConnected(WebsocketClientID id, WebsocketIncomingMessage? message) {
+         observer.OnNext(WebsocketConnectionUpdate.Create(
+            id, new WebsocketConnectNotification(id, message)));
+      }
+
+      void onDisconnected(WebsocketClientID id) =>
+         observer.OnNext(WebsocketConnectionUpdate.Create(
+            id, new WebsocketDisconnectNotification(id)));
+
+      void onMessageReceived(WebsocketIncomingMessage message) =>
+         observer.OnNext(WebsocketConnectionUpdate.Create(
+            message.sender_client_id, message));
+   }
+}

src/client/spark_websocket/IJsonRpcHandler.cs

@@ -0,0 +1,11 @@
+namespace Bts.Websocket {
+   using bts.websocket_core;
+
+   using System.Text.Json.Nodes;
+
+   public interface IJsonRpcHandler : IDisposable {
+      void OnRequest(WebsocketClientID id, JsonRpcRequest<JsonObject?> req);
+      void OnConnect(WebsocketConnectNotification notification);
+      void OnDisconnect(WebsocketDisconnectNotification notification);
+   }
+}

src/client/spark_websocket/JsonRpcDataSync.cs

@@ -0,0 +1,136 @@
+namespace Bts.DataStream.Websocket;
+
+using bts;
+using bts.websocket_core;
+
+using Bts.Websocket;
+
+using Spark.Core.Disposables;
+
+using System.Reactive.Subjects;
+using System.Text.Json;
+using System.Text.Json.Nodes;
+
+/// <summary>
+/// Represents an I/O model (<see cref="ISubject{TSource, TResult}"/>), that accepts websocket
+/// requests (from clients) as input, and emits data/notifications (to clients) as output.
+/// This class's chief role is managing data subscriptions to sparkd on the clients' behalf.
+/// </summary>
+public abstract class JsonRpcDataSync : ISubject<WebsocketConnectionUpdate, IEnumerable<WebsocketOutgoingMessage>> {
+   readonly JsonSerializerOptions serializerOpts;
+   protected readonly object lockObj = new();
+
+   protected JsonRpcDataSync(JsonSerializerOptions? serializerOpts = null) =>
+      this.serializerOpts = serializerOpts ?? JsonSerializerOptions.Default;
+
+   event Action<IEnumerable<WebsocketOutgoingMessage>>? requestSend;
+
+   public void OnCompleted() =>
+      Log.Api.info("DataSync incoming request stream has completed.");
+
+   public void OnError(Exception error) =>
+      Log.Api.error($"DataSync error: {error}");
+
+   public void OnNext(WebsocketConnectionUpdate value) {
+      switch (value.Type) {
+         case WebsocketConnectionUpdateType.IncomingMessage:
+            onIncomingMessage(value.IncomingMessage);
+            break;
+         case WebsocketConnectionUpdateType.Connect:
+            Log.Api.info($"DataStreamWebsocketDataSync client connected. clientId: {value.Connect.Id}");
+            onConnect(value.Connect);
+            if (value.Connect.Message is not null) {
+               onIncomingMessage(value.Connect.Message);
+            }
+            break;
+         case WebsocketConnectionUpdateType.Disconnect:
+            onDisconnect(value.Disconnect);
+            break;
+         default:
+            Log.assert(false, $"Unexpected WebsocketConnectionUpdate type: {value.Type}");
+            break;
+      }
+   }
+
+   public IDisposable Subscribe(IObserver<IEnumerable<WebsocketOutgoingMessage>> observer) {
+      lock (lockObj) {
+         requestSend += observer.OnNext;
+         return Disposable.Create(Dispose);
+      }
+      void Dispose() {
+         lock (lockObj) {
+            requestSend -= observer.OnNext;
+         }
+      }
+   }
+
+   protected abstract void onConnect(WebsocketConnectNotification notification);
+   protected abstract void onDisconnect(WebsocketDisconnectNotification notification);
+   protected abstract void onRequest(WebsocketClientID id, JsonRpcRequest<JsonObject?> req);
+
+   protected virtual void onInvalidRequest(WebsocketIncomingMessage request, string? errorMsg) {
+      var logMsg = $"Recieved invalid request:\n{request.message_body}";
+      if (errorMsg is not null) {
+         logMsg += $"\n\n**Exception**\n{errorMsg}";
+      }
+      Log.Api.error(logMsg);
+      var rpcId = extractRpcId(request.message_body);
+      if (rpcId is not null) {
+         // Send error back to client
+         var rpcErr = JsonRpcError.Create(
+            (long)JsonRpcErrorCode.InvalidRequest,
+            $"Received invalid request: {errorMsg ?? "unknown error"}",
+            request.message_body);
+         sendError(request.sender_client_id, rpcId.Value, rpcErr);
+      }
+   }
+
+   protected void sendNotification<T>(WebsocketClientID id, string method, T rpcParams) {
+      var rpcNotification = JsonRpcRequest.CreateNotification(method, rpcParams);
+      var rpcNotificationStr = JsonSerializer.Serialize(rpcNotification, serializerOpts);
+      send([new(id, rpcNotificationStr)]);
+   }
+
+   protected void sendResponse<T>(WebsocketClientID id, JsonRpcId rpcId, T rpcParams) {
+      var rpcRes = JsonRpcResponse.Create(rpcId, rpcParams);
+      var rpcResStr = JsonSerializer.Serialize(rpcRes, serializerOpts);
+      send([new(id, rpcResStr)]);
+   }
+
+   protected void sendError<T>(WebsocketClientID id, JsonRpcId rpcId, JsonRpcError<T> error) {
+      var rpcRes = JsonRpcResponse.CreateError(rpcId, error);
+      var rpcResStr = JsonSerializer.Serialize(rpcRes, serializerOpts);
+      send([new(id, rpcResStr)]);
+   }
+
+   protected bool send(IEnumerable<WebsocketOutgoingMessage> messages) {
+      lock (lockObj) {
+         if (requestSend is null) {
+            Log.Api.warn(
+               $"Unable to send message(s) because '{nameof(requestSend)}' handler is null");
+            return false;
+         }
+         requestSend(messages);
+         return true;
+      }
+   }
+
+   static JsonRpcId? extractRpcId(string messageBody) {
+      try {
+         return JsonNode.Parse(messageBody)
+            ?.AsObject()
+            ?.GetValueOrDefault("id")
+            ?.Deserialize<JsonRpcId>();
+      } catch {
+         return null;
+      }
+   }
+
+   void onIncomingMessage(WebsocketIncomingMessage message) {
+      if (JsonRpcRequest.TryDeserialize(message, out var req, out var err)) {
+         onRequest(message.sender_client_id, req);
+      } else {
+         onInvalidRequest(message, err);
+      }
+   }
+}

src/client/spark_websocket/JsonRpcRequest.cs

@@ -1,86 +1,86 @@
-namespace Bts.Websocket;
-
-using bts.websocket_core;
-
-using System.Diagnostics.CodeAnalysis;
-using System.Text.Json;
-using System.Text.Json.Nodes;
-using System.Text.Json.Serialization;
-
-public static class JsonRpcRequest {
-   public static JsonRpcRequest<T> Create<T>(JsonRpcId id, string method, T parameters) =>
-      new() {
-         Id = id,
-         Method = method,
-         Params = parameters
-      };
-
-   public static JsonRpcRequest<T> CreateNotification<T>(string method, T parameters) =>
-      new() {
-         Method = method,
-         Params = parameters
-      };
-
-   public static bool TryDeserialize(
-      WebsocketIncomingMessage message,
-      [NotNullWhen(true)]out JsonRpcRequest<JsonObject?>? result,
-      [NotNullWhen(false)]out string? error
-   ) {
-      try {
-         result = JsonSerializer.Deserialize<JsonRpcRequest<JsonObject?>>(message.message_body)!;
-         error = null;
-         return true;
-      } catch (Exception ex) {
-         error = ex.ToString();
-         result = null;
-         return false;
-      }
-   }
-}
-
-public class JsonRpcRequest<T> {
-   [JsonPropertyName("method")]
-   public string? Method { get; set; }
-
-   [JsonPropertyName("params")]
-   [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
-   [MaybeNull]
-   public T Params { get; set; }
-
-   [JsonPropertyName("jsonrpc")]
-   public string JsonRpc { get; set; } = "2.0";
-
-   [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
-   [JsonPropertyName("id")]
-   public JsonRpcId? Id { get; set; }
-
-   [JsonIgnore]
-   public bool IsNotification =>
-      !Id.HasValue;
-
-   public string Serialize() =>
-      Serialize(JsonSerializerOptions.Default);
-
-   public string Serialize(JsonSerializerOptions opts) =>
-      JsonSerializer.Serialize(this, opts);
-
-   public bool SerializeSafe(
-      [NotNullWhen(true)]out string? result,
-      [NotNullWhen(false)]out Exception? error) =>
-      SerializeSafe(JsonSerializerOptions.Default, out result, out error);
-
-   public bool SerializeSafe(
-      JsonSerializerOptions opts,
-      [NotNullWhen(true)]out string? result,
-      [NotNullWhen(false)]out Exception? error) {
-      try {
-         result = JsonSerializer.Serialize(this, opts);
-         error = null;
-         return true;
-      } catch (Exception ex) {
-         error = ex;
-         result = null;
-         return false;
-      }
-   }
-}
+namespace Bts.Websocket;
+
+using bts.websocket_core;
+
+using System.Diagnostics.CodeAnalysis;
+using System.Text.Json;
+using System.Text.Json.Nodes;
+using System.Text.Json.Serialization;
+
+public static class JsonRpcRequest {
+   public static JsonRpcRequest<T> Create<T>(JsonRpcId id, string method, T parameters) =>
+      new() {
+         Id = id,
+         Method = method,
+         Params = parameters
+      };
+
+   public static JsonRpcRequest<T> CreateNotification<T>(string method, T parameters) =>
+      new() {
+         Method = method,
+         Params = parameters
+      };
+
+   public static bool TryDeserialize(
+      WebsocketIncomingMessage message,
+      [NotNullWhen(true)]out JsonRpcRequest<JsonObject?>? result,
+      [NotNullWhen(false)]out string? error
+   ) {
+      try {
+         result = JsonSerializer.Deserialize<JsonRpcRequest<JsonObject?>>(message.message_body)!;
+         error = null;
+         return true;
+      } catch (Exception ex) {
+         error = ex.ToString();
+         result = null;
+         return false;
+      }
+   }
+}
+
+public class JsonRpcRequest<T> {
+   [JsonPropertyName("method")]
+   public string? Method { get; set; }
+
+   [JsonPropertyName("params")]
+   [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
+   [MaybeNull]
+   public T Params { get; set; }
+
+   [JsonPropertyName("jsonrpc")]
+   public string JsonRpc { get; set; } = "2.0";
+
+   [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
+   [JsonPropertyName("id")]
+   public JsonRpcId? Id { get; set; }
+
+   [JsonIgnore]
+   public bool IsNotification =>
+      !Id.HasValue;
+
+   public string Serialize() =>
+      Serialize(JsonSerializerOptions.Default);
+
+   public string Serialize(JsonSerializerOptions opts) =>
+      JsonSerializer.Serialize(this, opts);
+
+   public bool SerializeSafe(
+      [NotNullWhen(true)]out string? result,
+      [NotNullWhen(false)]out Exception? error) =>
+      SerializeSafe(JsonSerializerOptions.Default, out result, out error);
+
+   public bool SerializeSafe(
+      JsonSerializerOptions opts,
+      [NotNullWhen(true)]out string? result,
+      [NotNullWhen(false)]out Exception? error) {
+      try {
+         result = JsonSerializer.Serialize(this, opts);
+         error = null;
+         return true;
+      } catch (Exception ex) {
+         error = ex;
+         result = null;
+         return false;
+      }
+   }
+}