check pt: Before going to Chi (11/07/23, random scripts, etc.)

It should always be a number between 1-65535

.editorconfig

@@ -207,23 +207,17 @@ dotnet_naming_symbols.parameters.applicable_accessibilities = public,internal,pr
 dotnet_naming_symbols.private_symbols.applicable_kinds = property,method,field,event,delegate
 dotnet_naming_symbols.private_symbols.applicable_accessibilities = protected,private
 
-# local symbols
-dotnet_naming_symbols.local_symbols.applicable_kinds = local
-
-# parameter symbols
-dotnet_naming_symbols.parameter_symbols.applicable_kinds = parameter
-
 # Naming styles
 
-dotnet_naming_style.pascal_case.required_prefix =
-dotnet_naming_style.pascal_case.required_suffix =
-dotnet_naming_style.pascal_case.word_separator =
-dotnet_naming_style.pascal_case.capitalization = pascal_case
+dotnet_naming_style.snake_case.required_prefix =
+dotnet_naming_style.snake_case.required_suffix =
+dotnet_naming_style.snake_case.word_separator = _
+dotnet_naming_style.snake_case.capitalization = all_lower
 
-dotnet_naming_style.camel_case.required_prefix =
-dotnet_naming_style.camel_case.required_suffix =
-dotnet_naming_style.camel_case.word_separator =
-dotnet_naming_style.camel_case.capitalization = camel_case
+dotnet_naming_style.private_snake_case.required_prefix = _
+dotnet_naming_style.private_snake_case.required_suffix =
+dotnet_naming_style.private_snake_case.word_separator = _
+dotnet_naming_style.private_snake_case.capitalization = all_lower
 
 dotnet_naming_style.user_types.required_prefix =
 dotnet_naming_style.user_types.required_suffix =
@@ -235,10 +229,6 @@ dotnet_naming_style.interface.required_suffix =
 dotnet_naming_style.interface.word_separator =
 dotnet_naming_style.interface.capitalization = pascal_case
 
-dotnet_naming_style.local.capitalization = camel_case
-
-dotnet_naming_style.parameter.capitalization = camel_case
-
 # rules
 dotnet_naming_rule.user_types.symbols = user_types
 dotnet_naming_rule.user_types.style = user_types
@@ -248,18 +238,14 @@ dotnet_naming_rule.interfaces.symbols = interface
 dotnet_naming_rule.interfaces.style = interface_style
 dotnet_naming_rule.interfaces.severity = suggestion
 
-dotnet_naming_rule.public_members.symbols = public_symbols
-dotnet_naming_rule.public_members.style = pascal_case
-dotnet_naming_rule.public_members.severity = suggestion
+dotnet_naming_rule.public_members_snake_case.symbols = public_symbols
+dotnet_naming_rule.public_members_snake_case.style = snake_case
+dotnet_naming_rule.public_members_snake_case.severity = suggestion
 
-dotnet_naming_rule.private_members.symbols = private_symbols
-dotnet_naming_rule.private_members.style = camel_case
-dotnet_naming_rule.private_members.severity = suggestion
+dotnet_naming_rule.private_members_snake_case.symbols = private_symbols
+dotnet_naming_rule.private_members_snake_case.style = private_snake_case
+dotnet_naming_rule.private_members_snake_case.severity = suggestion
 
-dotnet_naming_rule.variables_are_camel_case.severity = suggestion
-dotnet_naming_rule.variables_are_camel_case.symbols = local_symbols
-dotnet_naming_rule.variables_are_camel_case.style = local
-
-dotnet_naming_rule.parameters.symbols = parameter_symbols
-dotnet_naming_rule.parameters.style = parameter
+dotnet_naming_rule.parameters.symbols = parameters
+dotnet_naming_rule.parameters.style = snake_case
 dotnet_naming_rule.parameters.severity = suggestion

.gitignore

@@ -47,8 +47,8 @@ trace_data/
 /src/.clang-format
 /src/server/sparkd/cmds/spartan/analysis/.ipynb_checkpoints
 /src/server/sparkd/cmds/spartan/analysis/data
-.tmp/
-.venv*/
+*.tmp/*
+/.venv*/*
 .cache/
 /src/Spark.Admin.WebApp/tsconfig.*.g.json
 /src/Spark.Admin.WebApp/src/bin/

CMakePresets.json

@@ -29,8 +29,8 @@
         "CMAKE_TOOLCHAIN_FILE": "${sourceDir}/vcpkg/scripts/buildsystems/vcpkg.cmake",
         "EXTERNALS_ROOT": "${sourceDir}/externals/windows",
         "CONFIG_DEFAULTS_PATH": "config/profiles/default/*.conf",
-        "DOTNET_BINARY_DIR": "${sourceDir}/build/$<CONFIG>/bin/net8.0",
-        "DOTNET_WINDOWS_BINARY_DIR": "${sourceDir}/build/$<CONFIG>/bin/net8.0-windows"
+        "DOTNET_BINARY_DIR": "${sourceDir}/build/$<CONFIG>/bin/net7.0",
+        "DOTNET_WINDOWS_BINARY_DIR": "${sourceDir}/build/$<CONFIG>/bin/net7.0-windows"
       }
     },
 
@@ -197,7 +197,7 @@
         "CMAKE_CXX_FLAGS_RELEASE": "-fPIC -Wall -pedantic -Wextra -Werror -Wno-missing-braces -Wno-ignored-qualifiers --std=c++20 -fdiagnostics-color -fno-omit-frame-pointer -DNDEBUG -O3",
         "CMAKE_INSTALL_PREFIX": "${sourceDir}/install/${presetName}",
         "CONFIG_DEFAULTS_PATH": "config/profiles/default/*.conf",
-        "DOTNET_BINARY_DIR": "${sourceDir}/build/$<CONFIG>/bin/net8.0"
+        "DOTNET_BINARY_DIR": "${sourceDir}/build/$<CONFIG>/bin/net7.0"
       }
     },
 

docker/sparkd/dev/Dockerfile

@@ -16,7 +16,7 @@ RUN apt-get update && apt-get install -y \
     sqlite3 \
     tzdata
 
-# clang 13, 15 (so we can build old branches)
+# clang
 RUN apt-get update && apt-get install -y \
     clang-13 \
     clang-format-13 \
@@ -25,19 +25,6 @@ RUN apt-get update && apt-get install -y \
     clang-format-15 \
     clangd-15
 
-# clang 17 (so we can build new branches)
-RUN \
-    curl -fsSL https://apt.llvm.org/llvm-snapshot.gpg.key 2> /dev/null \
-    | gpg --dearmor - \
-    | tee /usr/share/keyrings/llvm-snapshot.gpg > /dev/null \
-    && echo "deb [signed-by=/usr/share/keyrings/llvm-snapshot.gpg] http://apt.llvm.org/jammy/ llvm-toolchain-jammy-17 main" \
-    | tee -a "/etc/apt/sources.list.d/llvm.list" \
-    && apt-get update \
-    && apt-get install -y \
-    clang-17 \
-    clang-format-17 \
-    clangd-17
-
 # g++
 RUN apt-get update && apt-get install -y \
     g++-11 \
@@ -50,12 +37,8 @@ RUN apt-get update && apt-get install -y \
     && python3 -m pip install --upgrade pip
 
 # dotnet
-RUN \
-   curl -fsSLO "https://packages.microsoft.com/config/ubuntu/$(lsb_release -rs)/packages-microsoft-prod.deb" \
-   && dpkg -i packages-microsoft-prod.deb \
-   && rm packages-microsoft-prod.deb \
-   && apt-get update \
-   && apt-get install -y dotnet-sdk-8.0
+RUN apt-get update && apt-get install -y \
+    dotnet7
 
 # ssh
 RUN apt-get update && apt-get install -y \
@@ -70,6 +53,10 @@ RUN \
     && echo "deb [signed-by=/usr/share/keyrings/kitware-archive-keyring.gpg] https://apt.kitware.com/ubuntu/ $(lsb_release -cs) main" \
     | tee /etc/apt/sources.list.d/kitware.list > /dev/null \
     && apt-get update \
+    && rm /usr/share/keyrings/kitware-archive-keyring.gpg \
+    && apt-get install -y kitware-archive-keyring \
+    && echo "deb [signed-by=/usr/share/keyrings/kitware-archive-keyring.gpg] https://apt.kitware.com/ubuntu/ $(lsb_release -cs)-rc main" \
+    | tee -a /etc/apt/sources.list.d/kitware.list >/dev/null \
     && apt-get install -y cmake
 
 # node
@@ -108,7 +95,6 @@ RUN apt-get update && apt-get install -y \
 # dev utils
 RUN apt-get update && apt-get install -y \ 
     fish \
-    gdb \
     git-all \
     sudo \
     vim \
@@ -116,12 +102,12 @@ RUN apt-get update && apt-get install -y \
 
 # set up default executables (update-alternatives)
 RUN \
-    update-alternatives --install /usr/bin/clang++ clang++ /usr/bin/clang++-17 100; \
-    update-alternatives --install /usr/bin/clang clang /usr/bin/clang-17 100; \
-    update-alternatives --install /usr/bin/clang-format clang-format /usr/bin/clang-format-17 100; \
-    update-alternatives --install /usr/bin/clangd clangd /usr/bin/clangd-17 100; \
-    update-alternatives --install /usr/bin/g++ g++ /usr/bin/g++-12 100; \
-    update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-12 100
+    update-alternatives --install /usr/bin/clang++ clang++ /usr/bin/clang++-13 100; \
+    update-alternatives --install /usr/bin/clang clang /usr/bin/clang-13 100; \
+    update-alternatives --install /usr/bin/clang-format clang-format /usr/bin/clang-format-13 100; \
+    update-alternatives --install /usr/bin/clangd clangd /usr/bin/clangd-13 100; \
+    update-alternatives --install /usr/bin/g++ g++ /usr/bin/g++-11 100; \
+    update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-11 100
 
 # Add compiler symlinks for ccache 
 RUN \
@@ -131,8 +117,6 @@ RUN \
     ln -s /usr/bin/ccache /usr/local/bin/clang++-13; \
     ln -s /usr/bin/ccache /usr/local/bin/clang-15; \
     ln -s /usr/bin/ccache /usr/local/bin/clang++-15; \
-    ln -s /usr/bin/ccache /usr/local/bin/clang-17; \
-    ln -s /usr/bin/ccache /usr/local/bin/clang++-17; \
     ln -s /usr/bin/ccache /usr/local/bin/gcc; \
     ln -s /usr/bin/ccache /usr/local/bin/g++; \
     ln -s /usr/bin/ccache /usr/local/bin/gcc-11; \

src/DataStreamSuite/ArithmeticExpressions/ArithmeticExpressions.csproj

@@ -1,7 +1,7 @@
 <Project Sdk="Microsoft.NET.Sdk">
 
   <PropertyGroup>
-    <TargetFramework>net8.0</TargetFramework>
+    <TargetFramework>net7.0</TargetFramework>
     <ImplicitUsings>enable</ImplicitUsings>
     <Nullable>enable</Nullable>
   </PropertyGroup>

src/DataStreamSuite/BaseUtilities/BaseUtilities.csproj

@@ -1,7 +1,7 @@
 <Project Sdk="Microsoft.NET.Sdk">
 
   <PropertyGroup>
-    <TargetFramework>net8.0</TargetFramework>
+    <TargetFramework>net7.0</TargetFramework>
     <ImplicitUsings>enable</ImplicitUsings>
     <Nullable>enable</Nullable>
   </PropertyGroup>

src/DataStreamSuite/BlueRegistry/BlueRegistry.csproj

@@ -1,7 +1,7 @@
 <Project Sdk="Microsoft.NET.Sdk">
 
   <PropertyGroup>
-    <TargetFramework>net8.0-windows</TargetFramework>
+    <TargetFramework>net7.0-windows</TargetFramework>
     <ImplicitUsings>enable</ImplicitUsings>
     <Nullable>enable</Nullable>
     <UseWindowsForms>true</UseWindowsForms>

src/DataStreamSuite/BooleanExpressions/BooleanExpressions.csproj

@@ -1,7 +1,7 @@
 <Project Sdk="Microsoft.NET.Sdk">
 
   <PropertyGroup>
-    <TargetFramework>net8.0</TargetFramework>
+    <TargetFramework>net7.0</TargetFramework>
     <ImplicitUsings>enable</ImplicitUsings>
     <Nullable>enable</Nullable>
   </PropertyGroup>

src/DataStreamSuite/Bts.DataStream.Websocket/Bts.DataStream.Websocket.csproj

@@ -1,7 +1,7 @@
 <Project Sdk="Microsoft.NET.Sdk">
 
   <PropertyGroup>
-    <TargetFramework>net8.0</TargetFramework>
+    <TargetFramework>net7.0</TargetFramework>
     <ImplicitUsings>enable</ImplicitUsings>
     <Nullable>enable</Nullable>
   </PropertyGroup>

src/DataStreamSuite/Bts.DataStream.Websocket/DataStreamDataSync.cs

@@ -1,119 +0,0 @@
-namespace Bts.DataStream.Websocket;
-
-using BCG;
-
-using bts;
-using bts.collections;
-using bts.websocket_core;
-
-using Bts.DataStreamExpressions;
-using Bts.Websocket;
-
-using mbts.utils;
-
-using Spark.Core.Disposables;
-
-using System.Text.Json;
-using System.Text.Json.Nodes;
-
-public class DataStreamDataSync(JsonSerializerOptions? serializerOpts = null) : JsonRpcDataSync(serializerOpts) {
-   readonly Dictionary<WebsocketClientID, List<ClientSubscription>> clients = [];
-
-   protected override void onConnect(WebsocketConnectNotification notification) {
-      var id = notification.Id;
-      lock (lockObj) {
-         clients.Add(id, []);
-      }
-      Log.Api.info($"DataStreamWebsocketDataSync client connected. clientId: {notification.Id}");
-      if (notification.Message is not null) {
-         onIncomingMessage(notification.Message);
-      }
-   }
-
-   protected override void onDisconnect(WebsocketDisconnectNotification notification) {
-      Log.Api.warn($"DataStreamWebsocketDataSync client disconnected. clientId: {notification.Id}");
-      unsubscribeAll(notification.Id);
-      clients.Remove(notification.Id);
-   }
-
-   protected override void onRequest(WebsocketClientID id, JsonRpcRequest<JsonObject?> req) {
-      switch (req.Method!.ToLower()) {
-         case "subscribe":
-            subscribe(id, req);
-            break;
-         case "unsubscribe":
-            unsubscribe(id, req);
-            break;
-         case "unsubscribe_all":
-            unsubscribeAll(id);
-            break;
-         default:
-            Log.Api.error($"Received request of unknown type: {req}");
-            break;
-      }
-   }
-
-   void subscribe(WebsocketClientID id, JsonRpcRequest<JsonObject?> req) {
-      var subParams = req.Params!.Deserialize<SubscriptionParams>()!;
-      var exp = DataStreamExpressionsInterpreter.Evaluate(subParams.Query!);
-      var stream = exp.getView(new Dictionary<string, StatementExpression>(0));
-      var sub = new ClientSubscription(id, subParams, Disposable.Create(Dispose));
-      lock (lockObj) {
-         clients[id].Add(sub);
-         sendResponse(id, req.Id!.Value, req.Params);
-         stream.onChange += OnChange;
-      }
-
-      void OnChange(object? newValue) {
-         var dp = ExceptionAssert.of_type<DataPacket>(newValue);
-         var rpcParams = new {
-            dp.addedRows,
-            dp.deletedRows
-         };
-         sendNotification(id, "subscription", rpcParams);
-      }
-
-      void Dispose() =>
-         stream.onChange -= OnChange;
-   }
-
-   void unsubscribe(WebsocketClientID id, JsonRpcRequest<JsonObject?> req) {
-      var subParams = req.Params!.Deserialize<SubscriptionParams>();
-      var subParamsStr = JsonSerializer.Serialize(subParams, serializerOpts);
-      // Minimize amount of code inside lock (just need it to find/remove the subscription)
-      List<ClientSubscription>? subs = null;
-      ClientSubscription? sub = null;
-      lock (lockObj) {
-         subs = clients.GetValueOrDefault(id);
-         subs?.TryRemoveFirst(s => s!.Params.Equals(subParams), out sub);
-      }
-      if (subs is null) {
-         // Didn't find client (log/throw).
-         var msg = $"Unable to unsubscribe: did not find client. Id: {id}.\nParams:\n{subParamsStr}";
-         Log.Api.error(msg);
-         throw new Exception(msg);
-      }
-      if (sub is null) {
-         // Found client, but didn't find subscription (log/throw).
-         var msg = $"Unable to unsubscribe: did not find the specified subscription. " +
-            $"Id: {id}. Request:\n{subParams}";
-         Log.Api.error(msg);
-         throw new Exception(msg);
-      }
-      // Found client and subscription (dispose/remove subscription).
-      Log.Api.info($"Unsubscribing client. Id: {id}.\nParams:\n{subParamsStr}");
-      sub.Dispose();
-   }
-
-   void unsubscribeAll(WebsocketClientID id) {
-      lock (lockObj) {
-         if (!clients.TryGetValue(id, out var subs)) {
-            throw new Exception(
-               $"Unable to unsubscribe: did not find client with id {id}");
-         }
-         subs.DisposeAll();
-         subs.Clear();
-      }
-   }
-
-}

src/DataStreamSuite/Bts.DataStream.Websocket/DataSync.cs

@@ -1,228 +1,229 @@
-namespace Bts.DataStream.Websocket;
-
-using BCG;
-
-using bts;
-using bts.api.core;
-using bts.collections;
-
-using Bts.DataStreamExpressions;
-using Bts.Websocket;
-
-using mbts.utils;
-
-using Spark.Core.Disposables;
-
-using System.Reactive.Subjects;
-using System.Text.Json;
-using System.Text.Json.Nodes;
-
-/// <summary>
-/// Represents an I/O model (<see cref="ISubject{TSource, TResult}"/>), that accepts websocket
-/// requests (from clients) as input, and emits data/notifications (to clients) as output.
-/// This class's chief role is managing data subscriptions to sparkd on the clients' behalf.
-/// </summary>
-public class DataSync(JsonSerializerOptions? serializerOpts = null) :
-   ISubject<WebsocketConnectionUpdate, IEnumerable<WebsocketOutgoingMessage>> {
-   protected readonly object lockObj = new();
-   readonly JsonSerializerOptions serializerOpts = serializerOpts ?? JsonSerializerOptions.Default;
-   readonly Dictionary<WebsocketClientID, List<ClientSubscription>> clients = [];
-
-   event Action<IEnumerable<WebsocketOutgoingMessage>>? requestSend;
-
-   public void OnCompleted() =>
-      Log.Api.info("DataSync incoming request stream has completed.");
-
-   public void OnError(Exception error) =>
-      Log.Api.error($"DataSync error: {error}");
-
-   public void OnNext(WebsocketConnectionUpdate value) {
-      switch (value.Type) {
-         case WebsocketConnectionUpdateType.IncomingMessage:
-            onIncomingMessage(value.IncomingMessage);
-            break;
-         case WebsocketConnectionUpdateType.Connect:
-            onConnect(value.Connect);
-            break;
-         case WebsocketConnectionUpdateType.Disconnect:
-            onDisconnect(value.Disconnect);
-            break;
-         default:
-            Log.assert(false, $"Unexpected WebsocketConnectionUpdate type: {value.Type}");
-            break;
-      }
-   }
-
-   public IDisposable Subscribe(IObserver<IEnumerable<WebsocketOutgoingMessage>> observer) {
-      lock (lockObj) {
-         requestSend += observer.OnNext;
-         return Disposable.Create(Dispose);
-      }
-      void Dispose() {
-         lock (lockObj) {
-            requestSend -= observer.OnNext;
-         }
-      }
-   }
-
-   static JsonRpcId? extractRpcId(string messageBody) {
-      try {
-         return JsonNode.Parse(messageBody)
-            ?.AsObject()
-            ?.GetValueOrDefault("id")
-            ?.Deserialize<JsonRpcId>();
-      } catch {
-         return null;
-      }
-   }
-
-   void onConnect(WebsocketConnectNotification notification) {
-      var id = notification.Id;
-      lock (lockObj) {
-         clients.Add(id, []);
-      }
-      Log.Api.info($"DataStreamWebsocketDataSync client connected. clientId: {notification.Id}");
-      if (notification.Message is not null) {
-         onIncomingMessage(notification.Message);
-      }
-   }
-
-   void onDisconnect(WebsocketDisconnectNotification notification) {
-      Log.Api.warn($"DataStreamWebsocketDataSync client disconnected. clientId: {notification.Id}");
-      unsubscribeAll(notification.Id);
-      clients.Remove(notification.Id);
-   }
-
-   void onIncomingMessage(WebsocketIncomingMessage message) {
-      var id = message.sender_client_id;
-      if (!JsonRpcRequest.TryDeserialize(message, out var req, out var err)) {
-         onBadRequest(message, err);
-         return;
-      }
-      switch (req.Method!.ToLower()) {
-         case "subscribe":
-            subscribe(id, req);
-            break;
-         case "unsubscribe":
-            unsubscribe(id, req);
-            break;
-         case "unsubscribe_all":
-            unsubscribeAll(id);
-            break;
-         default:
-            Log.Api.error($"Received request of unknown type: {req}");
-            break;
-      }
-   }
-
-   void sendNotification<T>(WebsocketClientID id, string method, T rpcParams) {
-      var rpcNotification = JsonRpcRequest.CreateNotification(method, rpcParams);
-      var rpcNotificationStr = JsonSerializer.Serialize(rpcNotification, serializerOpts);
-      send([new(id, rpcNotificationStr)]);
-   }
-
-   void sendResponse<T>(WebsocketClientID id, JsonRpcId rpcId, T rpcParams) {
-      var rpcRes = JsonRpcResponse.Create(rpcId, rpcParams);
-      var rpcResStr = JsonSerializer.Serialize(rpcRes, serializerOpts);
-      send([new(id, rpcResStr )]);
-   }
-
-   void sendError<T>(WebsocketClientID id, JsonRpcId rpcId, JsonRpcError<T> error) {
-      var rpcRes = JsonRpcResponse.CreateError(rpcId, error);
-      var rpcResStr = JsonSerializer.Serialize(rpcRes, serializerOpts);
-      send([new(id, rpcResStr)]);
-   }
-
-   bool send(IEnumerable<WebsocketOutgoingMessage> messages) {
-      lock (lockObj) {
-         if (requestSend is null) {
-            Log.Api.warn(
-               $"Unable to send message(s) because '{nameof(requestSend)}' handler is null");
-            return false;
-         }
-         requestSend(messages);
-         return true;
-      }
-   }
-
-   void subscribe(WebsocketClientID id, JsonRpcRequest<JsonObject?> req) {
-      var subParams = req.Params!.Deserialize<SubscriptionParams>()!;
-      var exp = DataStreamExpressionsInterpreter.Evaluate(subParams.Query!);
-      var stream = exp.getView(new Dictionary<string, StatementExpression>(0));
-      var sub = new ClientSubscription(id, subParams, Disposable.Create(Dispose));
-      lock (lockObj) {
-         clients[id].Add(sub);
-         sendResponse(id, req.Id!.Value, req.Params);
-         stream.onChange += OnChange;
-      }
-
-      void OnChange(object? newValue) {
-         var dp = ExceptionAssert.of_type<DataPacket>(newValue);
-         var rpcParams = new {
-            dp.addedRows,
-            dp.deletedRows
-         };
-         sendNotification(id, "subscription", rpcParams);
-      }
-
-      void Dispose() =>
-         stream.onChange -= OnChange;
-   }
-
-   void unsubscribe(WebsocketClientID id, JsonRpcRequest<JsonObject?> req) {
-      var subParams = req.Params!.Deserialize<SubscriptionParams>();
-      var subParamsStr = JsonSerializer.Serialize(subParams, serializerOpts);
-      // Minimize amount of code inside lock (just need it to find/remove the subscription)
-      List<ClientSubscription>? subs = null;
-      ClientSubscription? sub = null;
-      lock (lockObj) {
-         subs = clients.GetValueOrDefault(id);
-         subs?.TryRemoveFirst(s => s!.Params.Equals(subParams), out sub);
-      }
-      if (subs is null) {
-         // Didn't find client (log/throw).
-         var msg = $"Unable to unsubscribe: did not find client. Id: {id}.\nParams:\n{subParamsStr}";
-         Log.Api.error(msg);
-         throw new Exception(msg);
-      }
-      if (sub is null) {
-         // Found client, but didn't find subscription (log/throw).
-         var msg = $"Unable to unsubscribe: did not find the specified subscription. " +
-            $"Id: {id}. Request:\n{subParams}";
-         Log.Api.error(msg);
-         throw new Exception(msg);
-      }
-      // Found client and subscription (dispose/remove subscription).
-      Log.Api.info($"Unsubscribing client. Id: {id}.\nParams:\n{subParamsStr}");
-      sub.Dispose();
-   }
-
-   void unsubscribeAll(WebsocketClientID id) {
-      lock (lockObj) {
-         if (!clients.TryGetValue(id, out var subs)) {
-            throw new Exception(
-               $"Unable to unsubscribe: did not find client with id {id}");
-         }
-         subs.DisposeAll();
-         subs.Clear();
-      }
-   }
-
-   void onBadRequest(WebsocketIncomingMessage req, string? errorMsg) {
-      var rpcId = extractRpcId(req.message_body);
-      var logMsg = $"Recieved invalid request:\n{req.message_body}";
-      if (errorMsg is not null) {
-         logMsg += $"\n\n**Exception**\n{errorMsg}";
-      }
-      Log.Api.error(logMsg);
-      if (rpcId is null) {
-         return;
-      }
-      // Send error back to client
-      var rpcErr = JsonRpcError.Create(
-         (long)JsonRpcErrorCode.InvalidRequest,
-         $"Received invalid request: {errorMsg ?? "unknown error"}",
-         req.message_body);
-      sendError(req.sender_client_id, rpcId.Value, rpcErr);
-   }
-}
+namespace Bts.DataStream.Websocket;
+
+using BCG;
+
+using bts;
+using bts.api.core;
+using bts.collections;
+
+using Bts.DataStreamExpressions;
+using Bts.Websocket;
+
+using mbts.utils;
+
+using Spark.Core.Disposables;
+
+using System.Reactive.Subjects;
+using System.Text.Json;
+using System.Text.Json.Nodes;
+
+/// <summary>
+/// Represents an I/O model (<see cref="ISubject{TSource, TResult}"/>), that accepts websocket
+/// requests (from clients) as input, and emits data/notifications (to clients) as output.
+/// This class's chief role is managing data subscriptions to sparkd on the clients' behalf.
+/// </summary>
+public class DataSync(ServiceContext serviceCtx, JsonSerializerOptions? serializerOpts = null) :
+   ISubject<WebsocketConnectionUpdate, IEnumerable<WebsocketOutgoingMessage>> {
+   readonly ServiceContext serviceCtx = serviceCtx;
+   readonly JsonSerializerOptions serializerOpts = serializerOpts ?? JsonSerializerOptions.Default;
+   readonly Dictionary<WebsocketClientID, List<ClientSubscription>> clients = [];
+   readonly object lockObj = new();
+
+   event Action<IEnumerable<WebsocketOutgoingMessage>>? requestSend;
+
+   public void OnCompleted() =>
+      Log.Api.info("DataSync incoming request stream has completed.");
+
+   public void OnError(Exception error) =>
+      Log.Api.error($"DataSync error: {error}");
+
+   public void OnNext(WebsocketConnectionUpdate value) {
+      switch (value.Type) {
+         case WebsocketConnectionUpdateType.IncomingMessage:
+            onIncomingMessage(value.IncomingMessage);
+            break;
+         case WebsocketConnectionUpdateType.Connect:
+            onConnect(value.Connect);
+            break;
+         case WebsocketConnectionUpdateType.Disconnect:
+            onDisconnect(value.Disconnect);
+            break;
+         default:
+            Log.assert(false, $"Unexpected WebsocketConnectionUpdate type: {value.Type}");
+            break;
+      }
+   }
+
+   public IDisposable Subscribe(IObserver<IEnumerable<WebsocketOutgoingMessage>> observer) {
+      lock (lockObj) {
+         requestSend += observer.OnNext;
+         return Disposable.Create(dispose);
+      }
+      void dispose() {
+         lock (lockObj) {
+            requestSend -= observer.OnNext;
+         }
+      }
+   }
+
+   static JsonRpcId? extractRpcId(string messageBody) {
+      try {
+         return JsonNode.Parse(messageBody)
+            ?.AsObject()
+            ?.GetValueOrDefault("id")
+            ?.Deserialize<JsonRpcId>();
+      } catch {
+         return null;
+      }
+   }
+
+   void onConnect(WebsocketConnectNotification notification) {
+      var id = notification.Id;
+      lock (lockObj) {
+         clients.Add(id, new());
+      }
+      Log.Api.info($"DataStreamWebsocketDataSync client connected. clientId: {notification.Id}");
+      if (notification.Message is not null) {
+         onIncomingMessage(notification.Message);
+      }
+   }
+
+   void onDisconnect(WebsocketDisconnectNotification notification) {
+      Log.Api.warn($"DataStreamWebsocketDataSync client disconnected. clientId: {notification.Id}");
+      unsubscribeAll(notification.Id);
+      clients.Remove(notification.Id);
+   }
+
+   void onIncomingMessage(WebsocketIncomingMessage message) {
+      var id = message.sender_client_id;
+      if (!JsonRpcRequest.TryDeserialize(message, out var req, out var err)) {
+         onBadRequest(message, err);
+         return;
+      }
+      switch (req.Method!.ToLower()) {
+         case "subscribe":
+            subscribe(id, req);
+            break;
+         case "unsubscribe":
+            unsubscribe(id, req);
+            break;
+         case "unsubscribe_all":
+            unsubscribeAll(id);
+            break;
+         default:
+            Log.Api.error($"Received request of unknown type: {req}");
+            break;
+      }
+   }
+
+   void sendNotification<T>(WebsocketClientID id, string method, T rpcParams) {
+      var rpcNotification = JsonRpcRequest.CreateNotification(method, rpcParams);
+      var rpcNotificationStr = JsonSerializer.Serialize(rpcNotification, serializerOpts);
+      send(new WebsocketOutgoingMessage[] { new(id, rpcNotificationStr) });
+   }
+
+   void sendResponse<T>(WebsocketClientID id, JsonRpcId rpcId, T rpcParams) {
+      var rpcRes = JsonRpcResponse.Create(rpcId, rpcParams);
+      var rpcResStr = JsonSerializer.Serialize(rpcRes, serializerOpts);
+      send(new WebsocketOutgoingMessage[] { new(id, rpcResStr )});
+   }
+
+   void sendError<T>(WebsocketClientID id, JsonRpcId rpcId, JsonRpcError<T> error) {
+      var rpcRes = JsonRpcResponse.CreateError(rpcId, error);
+      var rpcResStr = JsonSerializer.Serialize(rpcRes, serializerOpts);
+      send(new WebsocketOutgoingMessage[] { new(id, rpcResStr) });
+   }
+
+   bool send(IEnumerable<WebsocketOutgoingMessage> messages) {
+      lock (lockObj) {
+         if (requestSend is null) {
+            Log.Api.warn(
+               $"Unable to send message(s) because '{nameof(requestSend)}' handler is null");
+            return false;
+         }
+         requestSend(messages);
+         return true;
+      }
+   }
+
+   void subscribe(WebsocketClientID id, JsonRpcRequest<JsonObject?> req) {
+      var subParams = req.Params!.Deserialize<SubscriptionParams>()!;
+      var exp = DataStreamExpressionsInterpreter.Evaluate(subParams.Query!);
+      var stream = exp.getView(new Dictionary<string, StatementExpression>(0));
+      var sub = new ClientSubscription(id, subParams, Disposable.Create(dispose));
+      lock (lockObj) {
+         clients[id].Add(sub);
+         sendResponse(id, req.Id!.Value, req.Params);
+         stream.onChange += onChange;
+      }
+
+      void onChange(object? newValue) {
+         var dp = ExceptionAssert.of_type<DataPacket>(newValue);
+         var rpcParams = new {
+            addedRows = dp.addedRows,
+            deletedRows = dp.deletedRows
+         };
+         sendNotification(id, "subscription", rpcParams);
+      }
+
+      void dispose() =>
+         stream.onChange -= onChange;
+   }
+
+   void unsubscribe(WebsocketClientID id, JsonRpcRequest<JsonObject?> req) {
+      var subParams = req.Params!.Deserialize<SubscriptionParams>();
+      var subParamsStr = JsonSerializer.Serialize(subParams, serializerOpts);
+      // Minimize amount of code inside lock (just need it to find/remove the subscription)
+      List<ClientSubscription>? subs = null;
+      ClientSubscription? sub = null;
+      lock (lockObj) {
+         subs = clients.GetValueOrDefault(id);
+         subs?.TryRemoveFirst(s => s!.Params.Equals(subParams), out sub);
+      }
+      if (subs is null) {
+         // Didn't find client (log/throw).
+         var msg = $"Unable to unsubscribe: did not find client. Id: {id}.\nParams:\n{subParamsStr}";
+         Log.Api.error(msg);
+         throw new Exception(msg);
+      }
+      if (sub is null) {
+         // Found client, but didn't find subscription (log/throw).
+         var msg = $"Unable to unsubscribe: did not find the specified subscription. " +
+            $"Id: {id}. Request:\n{subParams}";
+         Log.Api.error(msg);
+         throw new Exception(msg);
+      }
+      // Found client and subscription (dispose/remove subscription).
+      Log.Api.info($"Unsubscribing client. Id: {id}.\nParams:\n{subParamsStr}");
+      sub.Dispose();
+   }
+
+   void unsubscribeAll(WebsocketClientID id) {
+      lock (lockObj) {
+         if (!clients.TryGetValue(id, out var subs)) {
+            throw new Exception(
+               $"Unable to unsubscribe: did not find client with id {id}");
+         }
+         subs.DisposeAll();
+         subs.Clear();
+      }
+   }
+
+   void onBadRequest(WebsocketIncomingMessage req, string? errorMsg) {
+      var rpcId = extractRpcId(req.message_body);
+      var logMsg = $"Recieved invalid request:\n{req.message_body}";
+      if (errorMsg is not null) {
+         logMsg += $"\n\n**Exception**\n{errorMsg}";
+      }
+      Log.Api.error(logMsg);
+      if (rpcId is null) {
+         return;
+      }
+      // Send error back to client
+      var rpcErr = JsonRpcError.Create(
+         (long)JsonRpcErrorCode.InvalidRequest,
+         $"Received invalid request: {errorMsg ?? "unknown error"}",
+         req.message_body);
+      sendError(req.sender_client_id, rpcId.Value, rpcErr);
+   }
+}

src/DataStreamSuite/Bts.DataStream.Websocket/MultilineStringJsonConverter.cs

@@ -1,48 +1,48 @@
-namespace Bts.DataStream.Websocket;
-
-using Spark.Core;
-
-using System;
-using System.Text.Json;
-using System.Text.Json.Serialization;
-
-class MultilineStringJsonConverter : JsonConverter<string> {
-   public override string? Read(ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options) {
-      if (typeToConvert != typeof(string)) {
-         throw new ArgumentException($"Unable to parse {typeToConvert}", nameof(typeToConvert));
-      }
-      return reader.TokenType switch {
-         JsonTokenType.String => StringImpl(ref reader),
-         JsonTokenType.StartArray => ArrayImpl(ref reader),
-         _ => throw new ArgumentException(
-            $"Unable to parse {typeof(SubscriptionParams)} from {reader.TokenType}"),
-      };
-
-      static string? StringImpl(ref Utf8JsonReader reader) =>
-         new(reader.GetString()!);
-
-      static string? ArrayImpl(ref Utf8JsonReader reader) {
-         var lines = new List<string>(2);
-         while (reader.Read()) {
-            switch (reader.TokenType) {
-               case JsonTokenType.String:
-                  lines.Add(reader.GetString()!);
-                  break;
-               case JsonTokenType.EndArray:
-                  return string.Join("\n", lines);
-            }
-         }
-         throw new JsonException(
-            $"Failed to parse {typeof(SubscriptionParams)} (did not reach end of array)");
-      }
-   }
-
-   public override void Write(Utf8JsonWriter writer, string value, JsonSerializerOptions options) {
-      var lines = value.Split("\n");
-      if (lines.Length < 2) {
-         writer.WriteRawValue(value);
-         return;
-      }
-      writer.WriteRawValue($"[{", ".join(lines)}]");
-   }
-}
+namespace Bts.DataStream.Websocket;
+
+using Spark.Core;
+
+using System;
+using System.Text.Json;
+using System.Text.Json.Serialization;
+
+class MultilineStringJsonConverter : JsonConverter<string> {
+   public override string? Read(ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options) {
+      if (typeToConvert != typeof(string)) {
+         throw new ArgumentException($"Unable to parse {typeToConvert}", nameof(typeToConvert));
+      }
+      return reader.TokenType switch {
+         JsonTokenType.String => stringImpl(ref reader),
+         JsonTokenType.StartArray => arrayImpl(ref reader),
+         _ => throw new ArgumentException(
+            $"Unable to parse {typeof(SubscriptionParams)} from {reader.TokenType}"),
+      };
+
+      static string? stringImpl(ref Utf8JsonReader reader) =>
+         new(reader.GetString()!);
+
+      static string? arrayImpl(ref Utf8JsonReader reader) {
+         var lines = new List<string>(2);
+         while (reader.Read()) {
+            switch (reader.TokenType) {
+               case JsonTokenType.String:
+                  lines.Add(reader.GetString()!);
+                  break;
+               case JsonTokenType.EndArray:
+                  return string.Join("\n", lines);
+            }
+         }
+         throw new JsonException(
+            $"Failed to parse {typeof(SubscriptionParams)} (did not reach end of array)");
+      }
+   }
+
+   public override void Write(Utf8JsonWriter writer, string value, JsonSerializerOptions options) {
+      var lines = value.Split("\n");
+      if (lines.Length < 2) {
+         writer.WriteRawValue(value);
+         return;
+      }
+      writer.WriteRawValue($"[{", ".join(lines)}]");
+   }
+}

src/DataStreamSuite/Bts.DataStream.Websocket/Session.cs

@@ -1,31 +1,27 @@
-namespace Bts.DataStream.Websocket;
-
-using bts;
-using bts.api.core;
-
-using Bts.Websocket;
-
-using Google.Protobuf;
-
-using Spark.Core.Disposables;
-
-
-using System.Text.Json;
-
-public static class Session {
-   static int sessionIdCounter;
-
-   public static async Task Start(
-      ServiceContext context, WebsocketServerConfig config, JsonSerializerOptions? opts) {
-      opts ??= JsonSerializerOptions.Default;
-      var sessionId = Interlocked.Increment(ref sessionIdCounter);
-      var gateway = new WebsocketGatewayObservable(config);
-      var sync = new DataSync(context, opts);
-      using (gateway.Subscribe(sync))
-      using (sync.Subscribe(gateway)) {
-         Log.Api.info($"Starting DataStream Websocket Session {sessionId}");
-         Log.Api.info($"DataStream Websocket Session {sessionId} config:\n{JsonFormatter.Default.Format(config)}");
-         await gateway.ConnectAsync();
-      }
-   }
-}
+namespace Bts.DataStream.Websocket;
+
+using bts;
+using bts.api.core;
+
+using Bts.Websocket;
+
+using Google.Protobuf;
+
+using System.Text.Json;
+
+public static class Session {
+   static int sessionIdCounter;
+
+   public static async Task Start(
+      ServiceContext context, WebsocketServerConfig config, JsonSerializerOptions? opts) {
+      opts ??= JsonSerializerOptions.Default;
+      var sessionId = Interlocked.Increment(ref sessionIdCounter);
+      var gateway = new WebsocketGatewayObservable(config);
+      var sync = new DataSync(context, opts);
+      using var syncSub = gateway.Subscribe(sync);
+      using var gwSub = sync.Subscribe(gateway);
+      Log.Api.info($"Starting DataStream Websocket Session {sessionId}");
+      Log.Api.info($"DataStream Websocket Session {sessionId} config:\n{JsonFormatter.Default.Format(config)}");
+      await gateway.ConnectAsync();
+   }
+}

src/DataStreamSuite/Bts.DataStream.Websocket/SubscriptionParams.cs

@@ -1,110 +1,110 @@
-namespace Bts.DataStream.Websocket;
-
-using Spark.Core;
-
-using System;
-using System.Diagnostics.CodeAnalysis;
-using System.Text.Json;
-using System.Text.Json.Serialization;
-
-public class SubscriptionParams : IEquatable<SubscriptionParams> {
-   static class PropNames {
-      public const string QUERY = "query";
-   }
-
-   public SubscriptionParams(string? query) {
-      Query = query;
-   }
-
-   // Serializer supports plain string, or an array of lines for convenience
-   [MaybeNull]
-   [JsonPropertyName(PropNames.QUERY)]
-   [JsonConverter(typeof(MultilineStringJsonConverter))]
-   public string? Query { get; set; }
-
-   public bool Equals(SubscriptionParams? other) =>
-      other is not null &&
-      Query == other.Query;
-
-   class SubscriptionParamsConverter : JsonConverter<SubscriptionParams> {
-      public override bool CanConvert(Type typeToConvert) =>
-         typeToConvert == typeof(SubscriptionParams);
-
-      public override SubscriptionParams Read(
-         ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options) {
-         if (typeToConvert != typeof(SubscriptionParams)) {
-            throw new ArgumentException($"Unable to parse {typeToConvert}", nameof(typeToConvert));
-         }
-         return read(ref reader);
-      }
-
-      SubscriptionParams read(ref Utf8JsonReader reader) {
-         return reader.TokenType switch {
-            JsonTokenType.String => stringImpl(ref reader),
-            JsonTokenType.StartArray => arrayImpl(ref reader),
-            JsonTokenType.StartObject => objectImpl(ref reader),
-            _ => throw new ArgumentException(
-               $"Unable to parse {typeof(SubscriptionParams)} from {reader.TokenType}"),
-         };
-
-         static SubscriptionParams stringImpl(ref Utf8JsonReader reader) =>
-            new(reader.GetString()!);
-
-         static SubscriptionParams arrayImpl(ref Utf8JsonReader reader) {
-            var lines = new List<string>(2);
-            while (reader.Read()) {
-               switch (reader.TokenType) {
-                  case JsonTokenType.String:
-                     lines.Add(reader.GetString()!);
-                     break;
-                  case JsonTokenType.EndArray:
-                     return new("\n".join(lines));
-               }
-            }
-            throw new JsonException(
-               $"Failed to parse {typeof(SubscriptionParams)} (did not reach end of array)");
-         }
-
-         static SubscriptionParams objectImpl(ref Utf8JsonReader reader) {
-            var lines = new List<string>(2);
-            while (reader.Read()) {
-               switch (reader.TokenType) {
-                  case JsonTokenType.PropertyName:
-                     var propName = reader.GetString()!;
-                     if (propName == PropNames.QUERY) {
-                        switch (reader.TokenType) {
-                           case JsonTokenType.StartArray:
-                              return arrayImpl(ref reader);
-                           case JsonTokenType.String:
-                              return stringImpl(ref reader);
-                        }
-                     }
-                     break;
-                  case JsonTokenType.String:
-                     lines.Add(reader.GetString()!);
-                     break;
-                  case JsonTokenType.EndArray:
-                     return new("\n".join(lines));
-               }
-            }
-            throw new JsonException(
-               $"Failed to parse {typeof(SubscriptionParams)} (did not reach end of array)");
-         }
-      }
-
-      public override void Write(
-         Utf8JsonWriter writer, SubscriptionParams value, JsonSerializerOptions options) {
-         var q = value.Query;
-         var lines = q?.Split("\n");
-         if (lines is null || lines.Length < 2) {
-            writer.WriteString(PropNames.QUERY, value.Query);
-            return;
-         }
-         writer.WriteStartArray(PropNames.QUERY);
-         foreach (var l in lines) {
-            writer.WriteStringValue(l);
-         }
-         writer.WriteEndArray();
-      }
-   }
-}
+namespace Bts.DataStream.Websocket;
+
+using Spark.Core;
+
+using System;
+using System.Diagnostics.CodeAnalysis;
+using System.Text.Json;
+using System.Text.Json.Serialization;
+
+public class SubscriptionParams : IEquatable<SubscriptionParams> {
+   static class PropNames {
+      public const string QUERY = "query";
+   }
+
+   public SubscriptionParams(string? query) {
+      Query = query;
+   }
+
+   // Serializer supports plain string, or an array of lines for convenience
+   [MaybeNull]
+   [JsonPropertyName(PropNames.QUERY)]
+   [JsonConverter(typeof(MultilineStringJsonConverter))]
+   public string? Query { get; set; }
+
+   public bool Equals(SubscriptionParams? other) =>
+      other is not null &&
+      Query == other.Query;
+
+   class SubscriptionParamsConverter : JsonConverter<SubscriptionParams> {
+      public override bool CanConvert(Type typeToConvert) =>
+         typeToConvert == typeof(SubscriptionParams);
+
+      public override SubscriptionParams Read(
+         ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options) {
+         if (typeToConvert != typeof(SubscriptionParams)) {
+            throw new ArgumentException($"Unable to parse {typeToConvert}", nameof(typeToConvert));
+         }
+         return read(ref reader);
+      }
+
+      SubscriptionParams read(ref Utf8JsonReader reader) {
+         return reader.TokenType switch {
+            JsonTokenType.String => stringImpl(ref reader),
+            JsonTokenType.StartArray => arrayImpl(ref reader),
+            JsonTokenType.StartObject => objectImpl(ref reader),
+            _ => throw new ArgumentException(
+               $"Unable to parse {typeof(SubscriptionParams)} from {reader.TokenType}"),
+         };
+
+         static SubscriptionParams stringImpl(ref Utf8JsonReader reader) =>
+            new(reader.GetString()!);
+
+         static SubscriptionParams arrayImpl(ref Utf8JsonReader reader) {
+            var lines = new List<string>(2);
+            while (reader.Read()) {
+               switch (reader.TokenType) {
+                  case JsonTokenType.String:
+                     lines.Add(reader.GetString()!);
+                     break;
+                  case JsonTokenType.EndArray:
+                     return new("\n".join(lines));
+               }
+            }
+            throw new JsonException(
+               $"Failed to parse {typeof(SubscriptionParams)} (did not reach end of array)");
+         }
+
+         static SubscriptionParams objectImpl(ref Utf8JsonReader reader) {
+            var lines = new List<string>(2);
+            while (reader.Read()) {
+               switch (reader.TokenType) {
+                  case JsonTokenType.PropertyName:
+                     var propName = reader.GetString()!;
+                     if (propName == PropNames.QUERY) {
+                        switch (reader.TokenType) {
+                           case JsonTokenType.StartArray:
+                              return arrayImpl(ref reader);
+                           case JsonTokenType.String:
+                              return stringImpl(ref reader);
+                        }
+                     }
+                     break;
+                  case JsonTokenType.String:
+                     lines.Add(reader.GetString()!);
+                     break;
+                  case JsonTokenType.EndArray:
+                     return new("\n".join(lines));
+               }
+            }
+            throw new JsonException(
+               $"Failed to parse {typeof(SubscriptionParams)} (did not reach end of array)");
+         }
+      }
+
+      public override void Write(
+         Utf8JsonWriter writer, SubscriptionParams value, JsonSerializerOptions options) {
+         var q = value.Query;
+         var lines = q?.Split("\n");
+         if (lines is null || lines.Length < 2) {
+            writer.WriteString(PropNames.QUERY, value.Query);
+            return;
+         }
+         writer.WriteStartArray(PropNames.QUERY);
+         foreach (var l in lines) {
+            writer.WriteStringValue(l);
+         }
+         writer.WriteEndArray();
+      }
+   }
+}

src/DataStreamSuite/Bts.DataStream.Websocket/ViewStreamUpdate.cs

@@ -1,41 +0,0 @@
-#nullable enable
-using BCG;
-
-using Spark.Core.Json;
-
-using System;
-using System.Collections.Generic;
-using System.Linq;
-using System.Text.Json;
-using System.Text.Json.Serialization;
-
-namespace Bts.DataStreamCore;
-
-class ViewStreamUpdate {
-   [JsonPropertyName("tableName")]
-   public string? TableName { get; set; }
-
-   [JsonPropertyName("columns")]
-   public List<ColumnInfo>? Columns { get; set; }
-
-   [JsonPropertyName("addedRows")]
-   public List<Dictionary<string, object?>>? AddedRows { get; set; }
-
-   [JsonPropertyName("deletedRows")]
-   public List<Dictionary<string, object?>>? DeletedRows { get; set; }
-
-   public DataPacket ToDataPacket(BlueTable state, bool applyChanges = true) {
-      var schema = new BlueTableSchema([.. Columns!], TableName);
-      var addedRowsTable = new BlueTable(schema);
-      var deletedRowsTable = new BlueTable(schema);
-      if (applyChanges) {
-         state.deleteRows(deletedRowsTable);
-         state.addRows(addedRowsTable);
-      }
-      return new(state, addedRowsTable, deletedRowsTable);
-   }
-
-   public static string ToJson() {
-      return "";
-   }
-}

src/DataStreamSuite/Bts.DataStream.WebsocketClient/Bts.DataStream.WebsocketClient.csproj

@@ -1,7 +1,7 @@
 <Project Sdk="Microsoft.NET.Sdk.Web">
 
   <PropertyGroup>
-    <TargetFramework>net8.0</TargetFramework>
+    <TargetFramework>net7.0</TargetFramework>
     <Nullable>enable</Nullable>
     <ImplicitUsings>enable</ImplicitUsings>
     <EnableDefaultContentItems>True</EnableDefaultContentItems>

src/DataStreamSuite/Bts.DataStream.WebsocketServer/Bts.DataStream.WebsocketServer.csproj

@@ -2,7 +2,7 @@
 
   <PropertyGroup>
     <OutputType>Exe</OutputType>
-    <TargetFramework>net8.0</TargetFramework>
+    <TargetFramework>net7.0</TargetFramework>
     <ImplicitUsings>enable</ImplicitUsings>
     <Nullable>enable</Nullable>
   </PropertyGroup>

src/DataStreamSuite/Bts.DataStreamWebApp.Client/.eslintrc.cjs

@@ -1,20 +0,0 @@
-module.exports = {
-  root: true,
-  env: { browser: true, es2020: true },
-  extends: [
-    'eslint:recommended',
-    'plugin:react/recommended',
-    'plugin:react/jsx-runtime',
-    'plugin:react-hooks/recommended',
-  ],
-  ignorePatterns: ['dist', '.eslintrc.cjs'],
-  parserOptions: { ecmaVersion: 'latest', sourceType: 'module' },
-  settings: { react: { version: '18.2' } },
-  plugins: ['react-refresh'],
-  rules: {
-    'react-refresh/only-export-components': [
-      'warn',
-      { allowConstantExport: true },
-    ],
-  },
-}

src/DataStreamSuite/Bts.DataStreamWebApp.Client/.gitignore

@@ -1,24 +0,0 @@
-# Logs
-logs
-*.log
-npm-debug.log*
-yarn-debug.log*
-yarn-error.log*
-pnpm-debug.log*
-lerna-debug.log*
-
-node_modules
-dist
-dist-ssr
-*.local
-
-# Editor directories and files
-.vscode/*
-!.vscode/extensions.json
-.idea
-.DS_Store
-*.suo
-*.ntvs*
-*.njsproj
-*.sln
-*.sw?

src/DataStreamSuite/Bts.DataStreamWebApp.Client/Bts.DataStreamWebApp.Client.esproj

@@ -1,11 +0,0 @@
-<Project Sdk="Microsoft.VisualStudio.JavaScript.Sdk/0.5.128-alpha">
-  <PropertyGroup>
-    <StartupCommand>npm run dev</StartupCommand>
-    <JavaScriptTestRoot>src\</JavaScriptTestRoot>
-    <JavaScriptTestFramework>Jest</JavaScriptTestFramework>
-    <!-- Allows the build (or compile) script located on package.json to run on Build -->
-    <ShouldRunBuildScript>false</ShouldRunBuildScript>
-    <!-- Folder where production build objects will be placed -->
-    <PublishAssetsDirectory>$(MSBuildProjectDirectory)\dist</PublishAssetsDirectory>
-  </PropertyGroup>
-</Project>

src/DataStreamSuite/Bts.DataStreamWebApp.Client/README.md

@@ -1,8 +0,0 @@
-# React + Vite
-
-This template provides a minimal setup to get React working in Vite with HMR and some ESLint rules.
-
-Currently, two official plugins are available:
-
-- [@vitejs/plugin-react](https://github.com/vitejs/vite-plugin-react/blob/main/packages/plugin-react/README.md) uses [Babel](https://babeljs.io/) for Fast Refresh
-- [@vitejs/plugin-react-swc](https://github.com/vitejs/vite-plugin-react-swc) uses [SWC](https://swc.rs/) for Fast Refresh

src/DataStreamSuite/Bts.DataStreamWebApp.Client/index.html

@@ -1,13 +0,0 @@
-<!doctype html>
-<html lang="en">
-  <head>
-    <meta charset="UTF-8" />
-    <link rel="icon" type="image/svg+xml" href="/vite.svg" />
-    <meta name="viewport" content="width=device-width, initial-scale=1.0" />
-    <title>Vite + React</title>
-  </head>
-  <body>
-    <div id="root"></div>
-    <script type="module" src="/src/main.jsx"></script>
-  </body>
-</html>

src/DataStreamSuite/Bts.DataStreamWebApp.Client/nuget.config

@@ -1,10 +0,0 @@
-<?xml version="1.0" encoding="utf-8"?>
-<configuration>
-  <packageSources>
-    <clear />
-    <add key="nuget.org" value="https://api.nuget.org/v3/index.json" />
-  </packageSources>
-  <disabledPackageSources>
-    <clear />
-  </disabledPackageSources>
-</configuration>

src/DataStreamSuite/Bts.DataStreamWebApp.Client/package.json

@@ -1,27 +0,0 @@
-{
-  "name": "bts.datastreamwebapp.client",
-  "private": true,
-  "version": "0.0.0",
-  "type": "module",
-  "scripts": {
-    "dev": "vite",
-    "build": "vite build",
-    "lint": "eslint . --ext js,jsx --report-unused-disable-directives --max-warnings 0",
-    "preview": "vite preview"
-  },
-  "dependencies": {
-    "react": "^18.2.0",
-    "react-dom": "^18.2.0",
-    "react-use-websocket": "^4.5.0"
-  },
-  "devDependencies": {
-    "@types/react": "^18.2.43",
-    "@types/react-dom": "^18.2.17",
-    "@vitejs/plugin-react": "^4.2.1",
-    "eslint": "^8.55.0",
-    "eslint-plugin-react": "^7.33.2",
-    "eslint-plugin-react-hooks": "^4.6.0",
-    "eslint-plugin-react-refresh": "^0.4.5",
-    "vite": "^5.0.8"
-  }
-}

src/DataStreamSuite/Bts.DataStreamWebApp.Client/public/vite.svg

@@ -1 +0,0 @@
-<svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" class="iconify iconify--logos" width="31.88" height="32" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 257"><defs><linearGradient id="IconifyId1813088fe1fbc01fb466" x1="-.828%" x2="57.636%" y1="7.652%" y2="78.411%"><stop offset="0%" stop-color="#41D1FF"></stop><stop offset="100%" stop-color="#BD34FE"></stop></linearGradient><linearGradient id="IconifyId1813088fe1fbc01fb467" x1="43.376%" x2="50.316%" y1="2.242%" y2="89.03%"><stop offset="0%" stop-color="#FFEA83"></stop><stop offset="8.333%" stop-color="#FFDD35"></stop><stop offset="100%" stop-color="#FFA800"></stop></linearGradient></defs><path fill="url(#IconifyId1813088fe1fbc01fb466)" d="M255.153 37.938L134.897 252.976c-2.483 4.44-8.862 4.466-11.382.048L.875 37.958c-2.746-4.814 1.371-10.646 6.827-9.67l120.385 21.517a6.537 6.537 0 0 0 2.322-.004l117.867-21.483c5.438-.991 9.574 4.796 6.877 9.62Z"></path><path fill="url(#IconifyId1813088fe1fbc01fb467)" d="M185.432.063L96.44 17.501a3.268 3.268 0 0 0-2.634 3.014l-5.474 92.456a3.268 3.268 0 0 0 3.997 3.378l24.777-5.718c2.318-.535 4.413 1.507 3.936 3.838l-7.361 36.047c-.495 2.426 1.782 4.5 4.151 3.78l15.304-4.649c2.372-.72 4.652 1.36 4.15 3.788l-11.698 56.621c-.732 3.542 3.979 5.473 5.943 2.437l1.313-2.028l72.516-144.72c1.215-2.423-.88-5.186-3.54-4.672l-25.505 4.922c-2.396.462-4.435-1.77-3.759-4.114l16.646-57.705c.677-2.35-1.37-4.583-3.769-4.113Z"></path></svg>

src/DataStreamSuite/Bts.DataStreamWebApp.Client/src/App.css

@@ -1,44 +0,0 @@
-#root {
-    max-width: 1280px;
-    margin: 0 auto;
-    padding: 2rem;
-    text-align: center;
-    --cell-background-color: #293136;
-    --cell-color: #cbe3ef;
-}
-
-tr:nth-child(even) {
-    background: #F2F2F2;
-}
-
-tr:nth-child(odd) {
-    background: #FFF;
-}
-
-th, td {
-    padding-left: 1rem;
-    padding-right: 1rem;
-    background-color: var(--cell-background-color);
-    color: var(--cell-color);
-}
-
-textarea#query-input {
-  height: 5em;
-  width: 50em;
-}
-
-button#query-button {
-  height: 5em;
-  vertical-align: top;
-}
-
-table#query-output {
-  float: none;
-  clear: both;
-}
-
-p#footer {
-  float: none;
-  clear: both;
-  vertical-align: bottom;
-}

src/DataStreamSuite/Bts.DataStreamWebApp.Client/src/App.jsx

@@ -1,93 +0,0 @@
-import { useEffect, useState, Component } from "react";
-import { useWebSocket } from "react-use-websocket";
-import "./App.css";
-
-export class Home extends Component {
-  static displayName = Home.name;
-  // static sparkdHost = '192.168.50.94';
-  static sparkdHost = "localhost";
-
-  // static websocket_url = 'wss://markets.api.uat.cmegroup.com/marketdatastream/v1';
-  // static access_token = 'eyJhbGciOiJSUzI1NiIsImtpZCI6IjF0al9MTVFrT2pGVzJWQTZvdkZzOEZHUWViMCIsInBpLmF0bSI6IjEifQ.eyJzY29wZSI6W10sImNsaWVudF9pZCI6ImFwaV90aGlua2FscGhhMzIxIiwiYXVkIjoiTnIiLCJqdGkiOiIweERtb3NnU2NHcE5GUEFaIiwic3ViIjoiYXBpX3RoaW5rYWxwaGEzMjEiLCJleHAiOjE2OTgzNTY5OTJ9.uJJEhWpujGiyJ5_lypiIPB8xR0GkuXqydtCugp5xD0hyTuX6VkQdNoY_oaRXXGRCUPCvfsXpU-HfL7pRkcXCjMLTrVIqf3RjD4_gar_ExbIb180nbYMG87E4rOILi5bIin51KOcMNBdHW7pTKwsS221XGyIAKIj65WxD-K8QSVr1HN6s7KDfYdez5dstWWC6xGg_3IZkrJjx8Df1HNT73K9PwUaAl7gcK4_2RSr8L636QbF3_y4Tv26g5IEv90Mc1mnAFPvUGI-nDR4s5Vso0398DlYl_tw-zbQQmn-QuQ6rS_ZIcZKgyoHKwD-3lgV-J1_--ndHdh8qwJC7Hk8ZKg'
-
-  render() {
-    let socket = null;
-    const wsUrl = `ws://localhost:5268/ws`;
-    function connectToSparkd() {
-      // const wsUrl = Home.websocket_url;
-      if (socket == null) {
-        socket = new WebSocket(wsUrl);
-      }
-      if (socket && socket.readyState === WebSocket.OPEN) {
-        return;
-      }
-
-      const output = document.getElementById("query-output");
-      socket.onopen = (event) => {
-        console.log(`Connected to ${wsUrl}`);
-      };
-
-      socket.onmessage = (event) => {
-        console.log(`Received message: ${event.data}`);
-        output.innerHTML = event.data;
-      };
-
-      socket.onerror = (error) => {
-        console.log(`Error on websocket ${wsUrl}: ${error}`);
-      };
-
-      socket.onclose = (event) => {
-        if (event.wasClean) {
-          const msg = `Connection to ${wsUrl} closed cleanly, code=${event.code}, reason=${event.reason}`;
-          console.log(msg);
-        } else {
-          const msg = `Connection to ${wsUrl} abruptly closed`;
-          console.log(msg);
-        }
-      };
-    }
-
-    function executeQuery() {
-      if (socket == null || !socket) {
-        connectToSparkd();
-      }
-      if (socket == null || !socket) {
-        console.log("Failed to connect to websocket server");
-        return;
-      }
-      if (socket.readyState === WebSocket.OPEN) {
-        const queryInput = document.getElementById("query-input");
-        const rpcReq = {
-          jsonrpc: "2.0",
-          method: "subscribe",
-          params: {
-            query: queryInput.value,
-          },
-          id: 1,
-        };
-        socket.send(JSON.stringify(rpcReq));
-      }
-    }
-
-    return (
-      <div>
-        <h1>Data Stream Query Analyzer</h1>
-        <p>Run queries against the BTS Cloud</p>
-        <div>
-          <textarea id="query-input" placeholder="Enter Query"></textarea>
-          <button id="query-button" onClick={executeQuery}>
-            Execute
-          </button>
-        </div>
-        <table id="query-output"></table>
-        <p id="footer">
-          The <code>ClientApp</code> subdirectory is a standard React
-          application based on the <code>create-react-app</code> template. If
-          you open a command prompt in that directory, you can run{" "}
-          <code>npm</code> commands such as <code>npm test</code> or{" "}
-          <code>npm install</code>.
-        </p>
-      </div>
-    );
-  }
-}

src/DataStreamSuite/Bts.DataStreamWebApp.Client/src/assets/react.svg

@@ -1 +0,0 @@
-<svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" class="iconify iconify--logos" width="35.93" height="32" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 228"><path fill="#00D8FF" d="M210.483 73.824a171.49 171.49 0 0 0-8.24-2.597c.465-1.9.893-3.777 1.273-5.621c6.238-30.281 2.16-54.676-11.769-62.708c-13.355-7.7-35.196.329-57.254 19.526a171.23 171.23 0 0 0-6.375 5.848a155.866 155.866 0 0 0-4.241-3.917C100.759 3.829 77.587-4.822 63.673 3.233C50.33 10.957 46.379 33.89 51.995 62.588a170.974 170.974 0 0 0 1.892 8.48c-3.28.932-6.445 1.924-9.474 2.98C17.309 83.498 0 98.307 0 113.668c0 15.865 18.582 31.778 46.812 41.427a145.52 145.52 0 0 0 6.921 2.165a167.467 167.467 0 0 0-2.01 9.138c-5.354 28.2-1.173 50.591 12.134 58.266c13.744 7.926 36.812-.22 59.273-19.855a145.567 145.567 0 0 0 5.342-4.923a168.064 168.064 0 0 0 6.92 6.314c21.758 18.722 43.246 26.282 56.54 18.586c13.731-7.949 18.194-32.003 12.4-61.268a145.016 145.016 0 0 0-1.535-6.842c1.62-.48 3.21-.974 4.76-1.488c29.348-9.723 48.443-25.443 48.443-41.52c0-15.417-17.868-30.326-45.517-39.844Zm-6.365 70.984c-1.4.463-2.836.91-4.3 1.345c-3.24-10.257-7.612-21.163-12.963-32.432c5.106-11 9.31-21.767 12.459-31.957c2.619.758 5.16 1.557 7.61 2.4c23.69 8.156 38.14 20.213 38.14 29.504c0 9.896-15.606 22.743-40.946 31.14Zm-10.514 20.834c2.562 12.94 2.927 24.64 1.23 33.787c-1.524 8.219-4.59 13.698-8.382 15.893c-8.067 4.67-25.32-1.4-43.927-17.412a156.726 156.726 0 0 1-6.437-5.87c7.214-7.889 14.423-17.06 21.459-27.246c12.376-1.098 24.068-2.894 34.671-5.345a134.17 134.17 0 0 1 1.386 6.193ZM87.276 214.515c-7.882 2.783-14.16 2.863-17.955.675c-8.075-4.657-11.432-22.636-6.853-46.752a156.923 156.923 0 0 1 1.869-8.499c10.486 2.32 22.093 3.988 34.498 4.994c7.084 9.967 14.501 19.128 21.976 27.15a134.668 134.668 0 0 1-4.877 4.492c-9.933 8.682-19.886 14.842-28.658 17.94ZM50.35 144.747c-12.483-4.267-22.792-9.812-29.858-15.863c-6.35-5.437-9.555-10.836-9.555-15.216c0-9.322 13.897-21.212 37.076-29.293c2.813-.98 5.757-1.905 8.812-2.773c3.204 10.42 7.406 21.315 12.477 32.332c-5.137 11.18-9.399 22.249-12.634 32.792a134.718 134.718 0 0 1-6.318-1.979Zm12.378-84.26c-4.811-24.587-1.616-43.134 6.425-47.789c8.564-4.958 27.502 2.111 47.463 19.835a144.318 144.318 0 0 1 3.841 3.545c-7.438 7.987-14.787 17.08-21.808 26.988c-12.04 1.116-23.565 2.908-34.161 5.309a160.342 160.342 0 0 1-1.76-7.887Zm110.427 27.268a347.8 347.8 0 0 0-7.785-12.803c8.168 1.033 15.994 2.404 23.343 4.08c-2.206 7.072-4.956 14.465-8.193 22.045a381.151 381.151 0 0 0-7.365-13.322Zm-45.032-43.861c5.044 5.465 10.096 11.566 15.065 18.186a322.04 322.04 0 0 0-30.257-.006c4.974-6.559 10.069-12.652 15.192-18.18ZM82.802 87.83a323.167 323.167 0 0 0-7.227 13.238c-3.184-7.553-5.909-14.98-8.134-22.152c7.304-1.634 15.093-2.97 23.209-3.984a321.524 321.524 0 0 0-7.848 12.897Zm8.081 65.352c-8.385-.936-16.291-2.203-23.593-3.793c2.26-7.3 5.045-14.885 8.298-22.6a321.187 321.187 0 0 0 7.257 13.246c2.594 4.48 5.28 8.868 8.038 13.147Zm37.542 31.03c-5.184-5.592-10.354-11.779-15.403-18.433c4.902.192 9.899.29 14.978.29c5.218 0 10.376-.117 15.453-.343c-4.985 6.774-10.018 12.97-15.028 18.486Zm52.198-57.817c3.422 7.8 6.306 15.345 8.596 22.52c-7.422 1.694-15.436 3.058-23.88 4.071a382.417 382.417 0 0 0 7.859-13.026a347.403 347.403 0 0 0 7.425-13.565Zm-16.898 8.101a358.557 358.557 0 0 1-12.281 19.815a329.4 329.4 0 0 1-23.444.823c-7.967 0-15.716-.248-23.178-.732a310.202 310.202 0 0 1-12.513-19.846h.001a307.41 307.41 0 0 1-10.923-20.627a310.278 310.278 0 0 1 10.89-20.637l-.001.001a307.318 307.318 0 0 1 12.413-19.761c7.613-.576 15.42-.876 23.31-.876H128c7.926 0 15.743.303 23.354.883a329.357 329.357 0 0 1 12.335 19.695a358.489 358.489 0 0 1 11.036 20.54a329.472 329.472 0 0 1-11 20.722Zm22.56-122.124c8.572 4.944 11.906 24.881 6.52 51.026c-.344 1.668-.73 3.367-1.15 5.09c-10.622-2.452-22.155-4.275-34.23-5.408c-7.034-10.017-14.323-19.124-21.64-27.008a160.789 160.789 0 0 1 5.888-5.4c18.9-16.447 36.564-22.941 44.612-18.3ZM128 90.808c12.625 0 22.86 10.235 22.86 22.86s-10.235 22.86-22.86 22.86s-22.86-10.235-22.86-22.86s10.235-22.86 22.86-22.86Z"></path></svg>

src/DataStreamSuite/Bts.DataStreamWebApp.Client/src/index.css

@@ -1,68 +0,0 @@
-:root {
-  font-family: Inter, system-ui, Avenir, Helvetica, Arial, sans-serif;
-  line-height: 1.5;
-  font-weight: 400;
-
-  color-scheme: light dark;
-  color: rgba(255, 255, 255, 0.87);
-  background-color: #242424;
-
-  font-synthesis: none;
-  text-rendering: optimizeLegibility;
-  -webkit-font-smoothing: antialiased;
-  -moz-osx-font-smoothing: grayscale;
-}
-
-a {
-  font-weight: 500;
-  color: #646cff;
-  text-decoration: inherit;
-}
-a:hover {
-  color: #535bf2;
-}
-
-body {
-  margin: 0;
-  display: flex;
-  place-items: center;
-  min-width: 320px;
-  min-height: 100vh;
-}
-
-h1 {
-  font-size: 3.2em;
-  line-height: 1.1;
-}
-
-button {
-  border-radius: 8px;
-  border: 1px solid transparent;
-  padding: 0.6em 1.2em;
-  font-size: 1em;
-  font-weight: 500;
-  font-family: inherit;
-  background-color: #1a1a1a;
-  cursor: pointer;
-  transition: border-color 0.25s;
-}
-button:hover {
-  border-color: #646cff;
-}
-button:focus,
-button:focus-visible {
-  outline: 4px auto -webkit-focus-ring-color;
-}
-
-@media (prefers-color-scheme: light) {
-  :root {
-    color: #213547;
-    background-color: #ffffff;
-  }
-  a:hover {
-    color: #747bff;
-  }
-  button {
-    background-color: #f9f9f9;
-  }
-}

src/DataStreamSuite/Bts.DataStreamWebApp.Client/src/main.jsx

@@ -1,10 +0,0 @@
-import React from 'react'
-import ReactDOM from 'react-dom/client'
-import {Home} from './App.jsx'
-import './index.css'
-
-ReactDOM.createRoot(document.getElementById('root')).render(
-  <React.StrictMode>
-    <Home />
-  </React.StrictMode>,
-)

src/DataStreamSuite/Bts.DataStreamWebApp.Client/vite.config.js

@@ -1,60 +0,0 @@
-import { fileURLToPath, URL } from 'node:url';
-
-import { defineConfig } from 'vite';
-import plugin from '@vitejs/plugin-react';
-import fs from 'fs';
-import path from 'path';
-import child_process from 'child_process';
-
-const baseFolder =
-    process.env.APPDATA !== undefined && process.env.APPDATA !== ''
-        ? `${process.env.APPDATA}/ASP.NET/https`
-        : `${process.env.HOME}/.aspnet/https`;
-
-const certificateArg = process.argv.map(arg => arg.match(/--name=(?<value>.+)/i)).filter(Boolean)[0];
-const certificateName = certificateArg ? certificateArg.groups.value : "bts.datastreamwebapp.client";
-
-if (!certificateName) {
-    console.error('Invalid certificate name. Run this script in the context of an npm/yarn script or pass --name=<<app>> explicitly.')
-    process.exit(-1);
-}
-
-const certFilePath = path.join(baseFolder, `${certificateName}.pem`);
-const keyFilePath = path.join(baseFolder, `${certificateName}.key`);
-
-if (!fs.existsSync(certFilePath) || !fs.existsSync(keyFilePath)) {
-    if (0 !== child_process.spawnSync('dotnet', [
-        'dev-certs',
-        'https',
-        '--export-path',
-        certFilePath,
-        '--format',
-        'Pem',
-        '--no-password',
-    ], { stdio: 'inherit', }).status) {
-        throw new Error("Could not create certificate.");
-    }
-}
-
-// https://vitejs.dev/config/
-export default defineConfig({
-    plugins: [plugin()],
-    resolve: {
-        alias: {
-            '@': fileURLToPath(new URL('./src', import.meta.url))
-        }
-    },
-    server: {
-        proxy: {
-            '^/weatherforecast': {
-                target: 'https://localhost:7128/',
-                secure: false
-            }
-        },
-        port: 5173,
-        https: {
-            key: fs.readFileSync(keyFilePath),
-            cert: fs.readFileSync(certFilePath),
-        }
-    }
-})

src/DataStreamSuite/Bts.DataStreamWebApp.Server/Bts.DataStreamWebApp.Server.csproj

@@ -1,35 +0,0 @@
-<Project Sdk="Microsoft.NET.Sdk.Web">
-
-  <PropertyGroup>
-    <TargetFramework>net8.0</TargetFramework>
-    <Nullable>enable</Nullable>
-    <ImplicitUsings>enable</ImplicitUsings>
-    <InvariantGlobalization>true</InvariantGlobalization>
-    <SpaRoot>..\Bts.DataStreamWebApp.Client</SpaRoot>
-    <SpaProxyLaunchCommand>npm run dev</SpaProxyLaunchCommand>
-    <SpaProxyServerUrl>https://localhost:5173</SpaProxyServerUrl>
-  </PropertyGroup>
-
-  <ItemGroup>
-    <PackageReference Include="CommandLineParser" Version="2.9.1" />
-    <PackageReference Include="Microsoft.AspNetCore.SpaProxy">
-      <Version>8.*-*</Version>
-    </PackageReference>
-    <PackageReference Include="Newtonsoft.Json" Version="13.0.3" />
-    <PackageReference Include="Swashbuckle.AspNetCore" Version="6.4.0" />
-  </ItemGroup>
-
-  <ItemGroup>
-    <ProjectReference Include="..\Bts.DataStreamWebApp.Client\Bts.DataStreamWebApp.Client.esproj">
-      <ReferenceOutputAssembly>false</ReferenceOutputAssembly>
-    </ProjectReference>
-    <ProjectReference Include="..\DataStream\DataStream.csproj" />
-    <ProjectReference Include="..\SharedStreams\SharedStreams.csproj" />
-    <ProjectReference Include="..\..\client\spark_api\spark_api.csproj" />
-    <ProjectReference Include="..\..\client\spark_websocket\spark_websocket.csproj" />
-  </ItemGroup>
-
-  <!-- <ItemGroup>
-    <Content Update="wwwroot\alt-index.html" CopyToPublishDirectory="Never" />
-  </ItemGroup> -->
-</Project>

src/DataStreamSuite/Bts.DataStreamWebApp.Server/Bts.DataStreamWebApp.Server.http

@@ -1,6 +0,0 @@
-@Bts.DataStreamWebApp.Server_HostAddress = http://localhost:5268
-
-GET {{Bts.DataStreamWebApp.Server_HostAddress}}/weatherforecast/
-Accept: application/json
-
-###

src/DataStreamSuite/Bts.DataStreamWebApp.Server/Controllers/DataStreamSubscriptionController.cs

@@ -1,117 +0,0 @@
-namespace Bts.DataStreamWebApp.Server.Controllers;
-
-using BCG;
-
-using bts.api.core;
-
-using Bts.DataStreamExpressions;
-
-using Microsoft.AspNetCore.Mvc;
-
-using Spark.Core.Disposables;
-
-using System.Net.WebSockets;
-using System.Text;
-using System.Text.Json;
-
-// [Route("[controller]")]
-[Route("/ws")]
-public class DataStreamSubscriptionController(ILogger<DataStreamSubscriptionController> logger) : ControllerBase {
-   readonly ILogger<DataStreamSubscriptionController> _logger = logger;
-   IDisposable? streamSubscription;
-
-   [Route("/ws")]
-   public async Task Get() {
-      if (!HttpContext.WebSockets.IsWebSocketRequest) {
-         _logger.LogError("Expected websocket request on path '{}'", HttpContext.Request.Path);
-         return;
-      }
-      Console.WriteLine("Establishing websocket connection...");
-      using var webSocket = await HttpContext.WebSockets.AcceptWebSocketAsync();
-      Console.WriteLine($"Websocket connection established.");
-      await runWebSocketServer(null, "bts", string.Empty, webSocket);
-   }
-
-   public static async Task Echo(WebSocket webSocket) {
-      var buffer = new byte[1024 * 4];
-      var receiveResult = await webSocket.ReceiveAsync(
-          new ArraySegment<byte>(buffer), CancellationToken.None);
-
-      while (!receiveResult.CloseStatus.HasValue) {
-         await webSocket.SendAsync(
-             new ArraySegment<byte>(buffer, 0, receiveResult.Count),
-             receiveResult.MessageType,
-             receiveResult.EndOfMessage,
-             CancellationToken.None);
-
-         receiveResult = await webSocket.ReceiveAsync(
-             new ArraySegment<byte>(buffer), CancellationToken.None);
-      }
-
-      await webSocket.CloseAsync(
-          receiveResult.CloseStatus.Value,
-          receiveResult.CloseStatusDescription,
-          CancellationToken.None);
-   }
-
-   async Task<int> runWebSocketServer(string? sparkd, string user, string password, WebSocket webSocket) {
-      var context = ServiceContext.create(sparkd);
-      await context.login(user, password);
-      var buffer = new byte[1024 * 4];
-      var receiveResult = await webSocket.ReceiveAsync(
-          new ArraySegment<byte>(buffer), CancellationToken.None);
-
-      try {
-         while (!receiveResult.CloseStatus.HasValue) {
-            receiveResult = await webSocket.ReceiveAsync(
-                new ArraySegment<byte>(buffer), CancellationToken.None);
-            var msg = Encoding.UTF8.GetString(buffer, 0, receiveResult.Count);
-            // var queryArgs = new QueryArguments { Query = msg };
-            var rpcReq = JsonSerializer.Deserialize<Bts.Websocket.JsonRpcRequest<DataStreamSubscription>>(msg);
-            var queryArgs = rpcReq!.Params;
-            // if (queryArgs is null) {
-            //    Console.WriteLine("Invalid queryArgs.");
-            //    continue;
-            // }
-            var exp = DataStreamExpressionsInterpreter.Evaluate(queryArgs.Query!);
-            if (exp is null) {
-               Console.WriteLine($"Invalid query expression: {queryArgs.Query}.");
-               continue;
-            }
-            var vs = exp.getView(new Dictionary<string, StatementExpression>(0));
-            Disposable.dispose(ref streamSubscription);
-            vs.onChange += onStreamChanged;
-            streamSubscription = Disposable.Create(() => vs.onChange -= onStreamChanged);
-            // await webSocket.SendAsync(
-            //     new ArraySegment<byte>(buffer, 0, receiveResult.Count),
-            //     receiveResult.MessageType,
-            //     receiveResult.EndOfMessage,
-            //     CancellationToken.None);
-         }
-      } catch (Exception ex) {
-         Console.WriteLine($"Exception on WebsocketServer thread: {ex}");
-         return 1;
-      }
-
-      return 0;
-
-      async void onStreamChanged(object newValue) {
-         if (newValue is not DataPacket dp) {
-            Console.WriteLine(
-               $"OnStreamChanged delegate expected DataPacket (actually received {newValue?.GetType()}.");
-            return;
-         }
-         Console.WriteLine("Received stream update.");
-         var added_rows_str = dp.addedRows.get_printed_rows(val => val.Length);
-         var msg = $"Received new rows:\n{added_rows_str}";
-         // var payload = new ReadOnlyMemory<byte>([.. msg]);
-         var encoded = Encoding.UTF8.GetBytes(msg);
-         var payload = new ArraySegment<byte>(encoded, 0, encoded.Length);
-         await webSocket.SendAsync(
-             payload,
-             receiveResult.MessageType,
-             receiveResult.EndOfMessage,
-             CancellationToken.None);
-      }
-   }
-}

src/DataStreamSuite/Bts.DataStreamWebApp.Server/Controllers/WeatherForecastController.cs

@@ -1,22 +0,0 @@
-namespace Bts.DataStreamWebApp.Server.Controllers;
-using Microsoft.AspNetCore.Mvc;
-
-[ApiController]
-[Route("[controller]")]
-public class WeatherForecastController(ILogger<WeatherForecastController> logger) : ControllerBase {
-   static readonly string[] Summaries = [
-      "Freezing", "Bracing", "Chilly", "Cool", "Mild", "Warm", "Balmy", "Hot", "Sweltering", "Scorching"
-   ];
-
-   readonly ILogger<WeatherForecastController> _logger = logger;
-
-   [HttpGet(Name = "GetWeatherForecast")]
-   public IEnumerable<WeatherForecast> Get() {
-      return Enumerable.Range(1, 5).Select(index => new WeatherForecast {
-         Date = DateOnly.FromDateTime(DateTime.Now.AddDays(index)),
-         TemperatureC = Random.Shared.Next(-20, 55),
-         Summary = Summaries[Random.Shared.Next(Summaries.Length)]
-      })
-      .ToArray();
-   }
-}

src/DataStreamSuite/Bts.DataStreamWebApp.Server/DataStreamSubscription.cs

@@ -1,8 +0,0 @@
-namespace Bts.DataStreamWebApp.Server;
-
-using System.Text.Json.Serialization;
-
-public class DataStreamSubscription {
-   [JsonPropertyName("query")]
-   public string? Query { get; set; }
-}

src/DataStreamSuite/Bts.DataStreamWebApp.Server/DataStreamUpdate.cs

@@ -1,8 +0,0 @@
-namespace Bts.DataStreamWebApp.Server;
-
-using BCG;
-
-class DataStreamUpdate {
-   public List<IBlueRow> addedRows { get; set; } = new(0);
-   public List<IBlueRow> deletedRows { get; set; } = new(0);
-}

src/DataStreamSuite/Bts.DataStreamWebApp.Server/DataStreamWebAppServerOptions.cs

@@ -1,14 +0,0 @@
-namespace Bts.DataStreamWebApp.Server;
-
-using CommandLine;
-
-class DataStreamWebAppServerOptions {
-   [Option("sparkd")]
-   public string? sparkd { get; set; }
-
-   [Option("user", Default = "bts")]
-   public string? user { get; set; }
-
-   [Option("password", Default = "")]
-   public string? password { get; set; }
-}

src/DataStreamSuite/Bts.DataStreamWebApp.Server/DataStreamWebsocketHandler.cs

@@ -1,58 +0,0 @@
-using Microsoft.AspNetCore.Http;
-using Microsoft.Extensions.Logging;
-
-using System;
-using System.Net.WebSockets;
-using System.Threading;
-using System.Threading.Tasks;
-
-namespace Bts.DataStreamWebApp.Server {
-   // public class DataStreamWebSocketHandler {
-   //    readonly WebSocketConnectionManager _connectionManager = new();
-   //    readonly ILogger<DataStreamWebSocketHandler> _logger;
-
-   //    public DataStreamWebSocketHandler(
-   //       WebSocketConnectionManager connectionManager,
-   //       ILogger<DataStreamWebSocketHandler> logger
-   //    ) {
-   //       _connectionManager = connectionManager;
-   //       _logger = logger;
-   //    }
-
-   //    public async Task HandleWebSocket(HttpContext context, WebSocket webSocket) {
-   //       var socketId = _connectionManager.AddSocket(webSocket);
-
-   //       _logger.LogInformation($"WebSocket connection established with ID {socketId}");
-
-   //       while (webSocket.State == WebSocketState.Open) {
-   //          var message = await ReceiveMessageAsync(webSocket);
-   //          if (message != null) {
-   //             _logger.LogInformation($"Received message from ID {socketId}: {message}");
-   //             await BroadcastMessageAsync(message);
-   //          }
-   //       }
-
-   //       _connectionManager.RemoveSocket(socketId);
-   //       _logger.LogInformation($"WebSocket connection closed with ID {socketId}");
-   //    }
-
-   //    async Task<string?> ReceiveMessageAsync(WebSocket webSocket) {
-   //       var buffer = new byte[1024];
-   //       var result = await webSocket.ReceiveAsync(new ArraySegment<byte>(buffer), CancellationToken.None);
-
-   //       if (result.CloseStatus.HasValue) {
-   //          return null;
-   //       }
-
-   //       return System.Text.Encoding.UTF8.GetString(buffer, 0, result.Count);
-   //    }
-
-   //    async Task BroadcastMessageAsync(string message) {
-   //       foreach (var socket in _connectionManager.GetAllSockets()) {
-   //          if (socket.Value.State == WebSocketState.Open) {
-   //             await socket.Value.SendAsync(System.Text.Encoding.UTF8.GetBytes(message), WebSocketMessageType.Text, true, CancellationToken.None);
-   //          }
-   //       }
-   //    }
-   // }
-}

src/DataStreamSuite/Bts.DataStreamWebApp.Server/Program.cs

@@ -1,166 +0,0 @@
-using BCG;
-
-using bts;
-using bts.api.core;
-using bts.utils;
-
-using Bts.DataStreamExpressions;
-using Bts.DataStreamWebApp.Server;
-using Bts.DataStreamWebApp.Server.Controllers;
-
-using CommandLine;
-
-using mbts.config;
-
-using System.Net.WebSockets;
-using System.Reactive.Disposables;
-using System.Reactive.Linq;
-using System.Text;
-using System.Text.Json;
-using System.Text.Json.Serialization;
-
-static string fmtReq(HttpRequest req) =>
-   $"{req.Protocol}://{req.Host}/{req.Path}?{req.QueryString}";
-
-static string fmtDataPacket(DataPacket dp) {
-   var sb = new StringBuilder();
-   var rows = dp.addedRows.Cast<IBlueRow>().ToList();
-   var colWidths = dp.schema.columnInfo.Select(c => rows.Max(r => r[c.colNo]?.ToString()?.Length ?? 0)).ToArray();
-   foreach (IBlueRow r in dp.addedRows) {
-      for (int i = 0; i < colWidths.Length; i++) {
-         var str = r[i]?.ToString() ?? string.Empty;
-         if (i > 0) {
-            sb.Append(" | ");
-         }
-         sb.Append(str);
-         var width = colWidths[i];
-         if (str.Length < width) {
-            sb.Append(new string(' ', width - str.Length));
-         }
-      }
-      sb.AppendLine();
-   }
-   return sb.ToString();
-   // var upd = new DataStreamUpdate();
-   // upd.addedRows.AddRange(dp.addedRows.Cast<IBlueRow>());
-   // upd.deletedRows.AddRange(dp.deletedRows.Cast<IBlueRow>());
-   // return fmtDataStreamUpdate(upd);
-}
-
-static string fmtDataStreamUpdate(DataStreamUpdate rows) {
-   return JsonSerializer.Serialize(rows);
-}
-
-var parser = new Parser(ps => {
-   ps.CaseInsensitiveEnumValues = true;
-   ps.HelpWriter = Console.Out;
-});
-
-await parser.ParseArguments<DataStreamWebAppServerOptions>(args).
-   MapResult(run, errors => Task.FromResult(1));
-
-async Task<int> run(DataStreamWebAppServerOptions opts) {
-   BTSNativeMethods.provide_command_line(args, args.Length);
-   Configuration.initialize();
-   var context = ServiceContext.create(opts.sparkd);
-   await context.login(opts.user!, opts.password!);
-   var vsWorkLoop = Observable.Interval(TimeSpan.FromMilliseconds(200)).Subscribe(_ => {
-      ViewStreamDatabase.instance.doWork();
-      MasterWorker.getInstance(false).doWork();
-   });
-   var builder = WebApplication.CreateBuilder(args);
-   builder.Services.AddDirectoryBrowser();
-   var app = builder.Build();
-   var wsOpts = new WebSocketOptions {
-      KeepAliveInterval = TimeSpan.FromMinutes(2)
-   };
-   app.UseWebSockets(wsOpts);
-   app.UseDefaultFiles();
-   app.UseStaticFiles();
-   app.MapFallbackToFile("/index.html");
-
-   app.Use(async (context, next) => {
-      var reqDesc = fmtReq(context.Request);
-      Console.WriteLine($"Webserver received: {reqDesc}");
-      if (context.Request.Path == "/ws") {
-         if (context.WebSockets.IsWebSocketRequest) {
-            Console.WriteLine("Accepting websocket connection...");
-            using var webSocket = await context.WebSockets.AcceptWebSocketAsync();
-            Console.WriteLine("Websocket connection accepted.");
-            await runWebSocketServer(
-               opts.sparkd, opts.user!, opts.password!, webSocket);
-         } else {
-            context.Response.StatusCode = StatusCodes.Status400BadRequest;
-         }
-      } else {
-         Console.WriteLine($"Webserver ignoring req: {reqDesc}");
-         await next(context);
-      }
-   });
-
-   try {
-      await app.RunAsync();
-   } catch (Exception ex) {
-      Console.WriteLine($"Exception on DataStreamWebsocketServer thread: {ex}");
-      return 1;
-   }
-   Spark.Core.Disposables.Disposable.dispose(ref vsWorkLoop);
-   return 0;
-}
-
-async Task<int> runWebSocketServer(string? sparkd, string user, string password, WebSocket webSocket) {
-   var buffer = new byte[1024 * 4];
-   var receiveResult = await webSocket.ReceiveAsync(
-       new ArraySegment<byte>(buffer), CancellationToken.None);
-
-   try {
-      while (!receiveResult.CloseStatus.HasValue) {
-         Console.WriteLine("Awaiting websocket message from client");
-         receiveResult = await webSocket.ReceiveAsync(
-             new ArraySegment<byte>(buffer), CancellationToken.None);
-         var msg = Encoding.UTF8.GetString(buffer, 0, receiveResult.Count);
-         var rpcReq = JsonSerializer.Deserialize<Bts.Websocket.JsonRpcRequest<DataStreamSubscription>>(msg);
-         var queryArgs = rpcReq!.Params!;
-         // var queryArgs = new DataStreamSubscription { Query = msg };
-         var exp = DataStreamExpressionsInterpreter.Evaluate(queryArgs.Query!);
-         if (exp is null) {
-            Console.WriteLine($"Invalid query expression: {queryArgs.Query}.");
-            continue;
-         }
-         var vs = exp.getView(new Dictionary<string, StatementExpression>(0));
-         Console.WriteLine("Subscribing to stream...");
-         vs.onChange += onStreamChanged;
-
-         await webSocket.SendAsync(
-             new ArraySegment<byte>(buffer, 0, receiveResult.Count),
-             receiveResult.MessageType,
-             receiveResult.EndOfMessage,
-             CancellationToken.None);
-      }
-   } catch (Exception ex) {
-      Console.WriteLine($"Exception on WebsocketServer thread: {ex}");
-      return 1;
-   }
-
-   return 0;
-
-   async void onStreamChanged(object newValue) {
-      if (newValue is not DataPacket dp) {
-         Console.WriteLine(
-            $"OnStreamChanged delegate expected DataPacket (actually received {newValue?.GetType()}.");
-         return;
-      }
-      Console.WriteLine("Received stream update.");
-      var sb = new StringBuilder();
-      var updateDesc = fmtDataPacket(dp);
-      var msg = $"Received new rows:\n{updateDesc}";
-      // var payload = new ReadOnlyMemory<byte>([.. msg]);
-      var encoded = Encoding.UTF8.GetBytes(msg);
-      var payload = new ArraySegment<byte>(encoded, 0, encoded.Length);
-      await webSocket.SendAsync(
-          payload,
-          receiveResult.MessageType,
-          receiveResult.EndOfMessage,
-          CancellationToken.None);
-   }
-}

src/DataStreamSuite/Bts.DataStreamWebApp.Server/Properties/launchSettings.json

@@ -1,48 +0,0 @@
-{
-  "profiles": {
-    "default": {
-      "commandName": "Project",
-      "launchBrowser": true,
-      "applicationUrl": "http://localhost:5400"
-    },
-    "http": {
-      "commandName": "Project",
-      "launchBrowser": true,
-      "launchUrl": "swagger",
-      "environmentVariables": {
-        "ASPNETCORE_ENVIRONMENT": "Development",
-        "ASPNETCORE_HOSTINGSTARTUPASSEMBLIES": "Microsoft.AspNetCore.SpaProxy"
-      },
-      "dotnetRunMessages": true,
-      "applicationUrl": "http://localhost:5268"
-    },
-    "https": {
-      "commandName": "Project",
-      "launchUrl": "swagger",
-      "environmentVariables": {
-        "ASPNETCORE_ENVIRONMENT": "Development",
-        "ASPNETCORE_HOSTINGSTARTUPASSEMBLIES": "Microsoft.AspNetCore.SpaProxy"
-      },
-      "dotnetRunMessages": true,
-      "applicationUrl": "https://localhost:7128;http://localhost:5268"
-    },
-    "IIS Express": {
-      "commandName": "IISExpress",
-      "launchBrowser": true,
-      "launchUrl": "swagger",
-      "environmentVariables": {
-        "ASPNETCORE_ENVIRONMENT": "Development",
-        "ASPNETCORE_HOSTINGSTARTUPASSEMBLIES": "Microsoft.AspNetCore.SpaProxy"
-      }
-    }
-  },
-  "$schema": "http://json.schemastore.org/launchsettings.json",
-  "iisSettings": {
-    "windowsAuthentication": false,
-    "anonymousAuthentication": true,
-    "iisExpress": {
-      "applicationUrl": "http://localhost:2083",
-      "sslPort": 44383
-    }
-  }
-}

src/DataStreamSuite/Bts.DataStreamWebApp.Server/WeatherForecast.cs

@@ -1,12 +0,0 @@
-namespace Bts.DataStreamWebApp.Server;
-
-public class WeatherForecast
-{
-   public DateOnly Date { get; set; }
-
-   public int TemperatureC { get; set; }
-
-   public int TemperatureF => 32 + (int)(TemperatureC / 0.5556);
-
-   public string? Summary { get; set; }
-}

src/DataStreamSuite/Bts.DataStreamWebApp.Server/WebsocketConnectionManager.cs

@@ -1,37 +0,0 @@
-using System;
-using System.Collections.Concurrent;
-using System.Net.WebSockets;
-
-namespace Bts.DataStreamWebApp.Server {
-   // public class WebSocketConnectionManager {
-   //    private readonly ConcurrentDictionary<Guid, WebSocket> _sockets = new ConcurrentDictionary<Guid, WebSocket>();
-
-   //    public Guid AddSocket(WebSocket socket) {
-   //       var socketId = Guid.NewGuid();
-   //       _sockets.TryAdd(socketId, socket);
-   //       return socketId;
-   //    }
-
-   //    public WebSocket? GetSocket(Guid socketId) {
-   //       _sockets.TryGetValue(socketId, out var socket);
-   //       return socket;
-   //    }
-
-   //    public ConcurrentDictionary<Guid, WebSocket>.ValueCollection GetAllSockets() {
-   //       return _sockets.Values;
-   //    }
-
-   //    public Guid? GetSocketId(WebSocket socket) {
-   //       foreach (var (key, value) in _sockets) {
-   //          if (value == socket) {
-   //             return key;
-   //          }
-   //       }
-   //       return null;
-   //    }
-
-   //    public void RemoveSocket(Guid socketId) {
-   //       _sockets.TryRemove(socketId, out _);
-   //    }
-   // }
-}

src/DataStreamSuite/Bts.DataStreamWebApp.Server/appsettings.Development.json

@@ -1,8 +0,0 @@
-{
-  "Logging": {
-    "LogLevel": {
-      "Default": "Information",
-      "Microsoft.AspNetCore": "Warning"
-    }
-  }
-}

src/DataStreamSuite/Bts.DataStreamWebApp.Server/appsettings.json

@@ -1,9 +0,0 @@
-{
-  "Logging": {
-    "LogLevel": {
-      "Default": "Information",
-      "Microsoft.AspNetCore": "Warning"
-    }
-  },
-  "AllowedHosts": "*"
-}

src/DataStreamSuite/Bts.DataStreamWebApp.Server/wwwroot/alt-index.html

@@ -1,168 +0,0 @@
-<!DOCTYPE html>
-<html>
-<head>
-    <meta charset="utf-8" />
-    <title></title>
-    <style>
-        table {
-            border: 0
-        }
-
-        .commslog-data {
-            font-family: Consolas, Courier New, Courier, monospace;
-        }
-
-        .commslog-server {
-            background-color: red;
-            color: white
-        }
-
-        .commslog-client {
-            background-color: green;
-            color: white
-        }
-    </style>
-</head>
-<body>
-    <h1>ALT WebSocket Sample Application</h1>
-    <p id="stateLabel">Ready to connect...</p>
-    <div>
-        <label for="connectionUrl">WebSocket Server URL:</label>
-        <input id="connectionUrl" />
-        <button id="connectButton" type="submit">Connect</button>
-    </div>
-    <p></p>
-    <div>
-        <label for="sendMessage">Message to send:</label>
-        <input id="sendMessage" disabled />
-        <button id="sendButton" type="submit" disabled>Send</button>
-        <button id="closeButton" disabled>Close Socket</button>
-    </div>
-
-    <h2>Communication Log</h2>
-    <table style="width: 800px">
-        <thead>
-            <tr>
-                <td style="width: 100px">From</td>
-                <td style="width: 100px">To</td>
-                <td>Data</td>
-            </tr>
-        </thead>
-        <tbody id="commsLog">
-        </tbody>
-    </table>
-
-    <script>
-        var connectionUrl = document.getElementById("connectionUrl");
-        var connectButton = document.getElementById("connectButton");
-        var stateLabel = document.getElementById("stateLabel");
-        var sendMessage = document.getElementById("sendMessage");
-        var sendButton = document.getElementById("sendButton");
-        var commsLog = document.getElementById("commsLog");
-        var closeButton = document.getElementById("closeButton");
-        var socket;
-
-        var scheme = document.location.protocol === "https:" ? "wss" : "ws";
-        var port = document.location.port ? (":" + document.location.port) : "";
-
-        connectionUrl.value = scheme + "://" + document.location.hostname + port + "/ws" ;
-
-        function updateState() {
-            function disable() {
-                sendMessage.disabled = true;
-                sendButton.disabled = true;
-                closeButton.disabled = true;
-            }
-            function enable() {
-                sendMessage.disabled = false;
-                sendButton.disabled = false;
-                closeButton.disabled = false;
-            }
-
-            connectionUrl.disabled = true;
-            connectButton.disabled = true;
-
-            if (!socket) {
-                disable();
-            } else {
-                switch (socket.readyState) {
-                    case WebSocket.CLOSED:
-                        stateLabel.innerHTML = "Closed";
-                        disable();
-                        connectionUrl.disabled = false;
-                        connectButton.disabled = false;
-                        break;
-                    case WebSocket.CLOSING:
-                        stateLabel.innerHTML = "Closing...";
-                        disable();
-                        break;
-                    case WebSocket.CONNECTING:
-                        stateLabel.innerHTML = "Connecting...";
-                        disable();
-                        break;
-                    case WebSocket.OPEN:
-                        stateLabel.innerHTML = "Open";
-                        enable();
-                        break;
-                    default:
-                        stateLabel.innerHTML = "Unknown WebSocket State: " + htmlEscape(socket.readyState);
-                        disable();
-                        break;
-                }
-            }
-        }
-
-        closeButton.onclick = function () {
-            if (!socket || socket.readyState !== WebSocket.OPEN) {
-                alert("socket not connected");
-            }
-            socket.close(1000, "Closing from client");
-        };
-
-        sendButton.onclick = function () {
-            if (!socket || socket.readyState !== WebSocket.OPEN) {
-                alert("socket not connected");
-            }
-            var data = sendMessage.value;
-            socket.send(data);
-            commsLog.innerHTML += '<tr>' +
-                '<td class="commslog-client">Client</td>' +
-                '<td class="commslog-server">Server</td>' +
-                '<td class="commslog-data">' + htmlEscape(data) + '</td></tr>';
-        };
-
-        connectButton.onclick = function() {
-            stateLabel.innerHTML = "Connecting";
-            socket = new WebSocket(connectionUrl.value);
-            socket.onopen = function (event) {
-                updateState();
-                commsLog.innerHTML += '<tr>' +
-                    '<td colspan="3" class="commslog-data">Connection opened</td>' +
-                '</tr>';
-            };
-            socket.onclose = function (event) {
-                updateState();
-                commsLog.innerHTML += '<tr>' +
-                    '<td colspan="3" class="commslog-data">Connection closed. Code: ' + htmlEscape(event.code) + '. Reason: ' + htmlEscape(event.reason) + '</td>' +
-                '</tr>';
-            };
-            socket.onerror = updateState;
-            socket.onmessage = function (event) {
-                commsLog.innerHTML += '<tr>' +
-                    '<td class="commslog-server">Server</td>' +
-                    '<td class="commslog-client">Client</td>' +
-                    '<td class="commslog-data">' + htmlEscape(event.data) + '</td></tr>';
-            };
-        };
-
-        function htmlEscape(str) {
-            return str.toString()
-                .replace(/&/g, '&amp;')
-                .replace(/"/g, '&quot;')
-                .replace(/'/g, '&#39;')
-                .replace(/</g, '&lt;')
-                .replace(/>/g, '&gt;');
-        }
-    </script>
-</body>
-</html>

src/DataStreamSuite/Bts.DataStreamWebApp.Server/wwwroot/index.html

@@ -1,176 +0,0 @@
-<!DOCTYPE html>
-<html>
-<head>
-    <meta charset="utf-8" />
-    <title></title>
-    <style>
-        table {
-            border: 0
-        }
-
-        .commslog-data {
-            font-family: Consolas, Courier New, Courier, monospace;
-        }
-
-        .commslog-server {
-            background-color: red;
-            color: white
-        }
-
-        .commslog-client {
-            background-color: green;
-            color: white
-        }
-    </style>
-</head>
-<body>
-    <h1>DataStreams via WebSocket</h1>
-    <p id="stateLabel">Ready to connect...</p>
-    <div>
-        <label for="connectionUrl">WebSocket Server URL:</label>
-        <input id="connectionUrl" />
-        <button id="connectButton" type="submit">Connect</button>
-    </div>
-    <p></p>
-    <div>
-        <label for="sendMessage">Query:</label>
-        <input id="sendMessage" disabled />
-        <button id="sendButton" type="submit" disabled>Send</button>
-        <button id="closeButton" disabled>Close Socket</button>
-    </div>
-
-    <h2>Communication Log</h2>
-    <table style="width: 800px">
-        <thead>
-            <tr>
-                <td style="width: 100px">From</td>
-                <td style="width: 100px">To</td>
-                <td>Data</td>
-            </tr>
-        </thead>
-        <tbody id="commsLog">
-        </tbody>
-    </table>
-
-    <script>
-        var connectionUrl = document.getElementById("connectionUrl");
-        var connectButton = document.getElementById("connectButton");
-        var stateLabel = document.getElementById("stateLabel");
-        var sendMessage = document.getElementById("sendMessage");
-        var sendButton = document.getElementById("sendButton");
-        var commsLog = document.getElementById("commsLog");
-        var closeButton = document.getElementById("closeButton");
-        var socket;
-
-        var scheme = document.location.protocol === "https:" ? "wss" : "ws";
-        var port = document.location.port ? (":" + document.location.port) : "";
-
-        connectionUrl.value = scheme + "://" + document.location.hostname + port + "/ws" ;
-
-        function updateState() {
-            function disable() {
-                sendMessage.disabled = true;
-                sendButton.disabled = true;
-                closeButton.disabled = true;
-            }
-            function enable() {
-                sendMessage.disabled = false;
-                sendButton.disabled = false;
-                closeButton.disabled = false;
-            }
-
-            connectionUrl.disabled = true;
-            connectButton.disabled = true;
-
-            if (!socket) {
-                disable();
-            } else {
-                switch (socket.readyState) {
-                    case WebSocket.CLOSED:
-                        stateLabel.innerHTML = "Closed";
-                        disable();
-                        connectionUrl.disabled = false;
-                        connectButton.disabled = false;
-                        break;
-                    case WebSocket.CLOSING:
-                        stateLabel.innerHTML = "Closing...";
-                        disable();
-                        break;
-                    case WebSocket.CONNECTING:
-                        stateLabel.innerHTML = "Connecting...";
-                        disable();
-                        break;
-                    case WebSocket.OPEN:
-                        stateLabel.innerHTML = "Open";
-                        enable();
-                        break;
-                    default:
-                        stateLabel.innerHTML = "Unknown WebSocket State: " + htmlEscape(socket.readyState);
-                        disable();
-                        break;
-                }
-            }
-        }
-
-        closeButton.onclick = function () {
-            if (!socket || socket.readyState !== WebSocket.OPEN) {
-                alert("socket not connected");
-            }
-            socket.close(1000, "Closing from client");
-        };
-
-        sendButton.onclick = function () {
-            if (!socket || socket.readyState !== WebSocket.OPEN) {
-                alert("socket not connected");
-            }
-            const rpcReq = {
-              jsonrpc: "2.0",
-              method: "subscribe",
-              params: {
-                query: sendMessage.value,
-              },
-              id: 1,
-            };
-            const rpcReqStr = JSON.stringify(rpcReq);
-            socket.send(rpcReqStr);
-            commsLog.innerHTML += '<tr>' +
-                '<td class="commslog-client">Client</td>' +
-                '<td class="commslog-server">Server</td>' +
-                '<td class="commslog-data">' + htmlEscape(rpcReqStr) + '</td></tr>';
-        };
-
-        connectButton.onclick = function() {
-            stateLabel.innerHTML = "Connecting";
-            socket = new WebSocket(connectionUrl.value);
-            socket.onopen = function (event) {
-                updateState();
-                commsLog.innerHTML += '<tr>' +
-                    '<td colspan="3" class="commslog-data">Connection opened</td>' +
-                '</tr>';
-            };
-            socket.onclose = function (event) {
-                updateState();
-                commsLog.innerHTML += '<tr>' +
-                    '<td colspan="3" class="commslog-data">Connection closed. Code: ' + htmlEscape(event.code) + '. Reason: ' + htmlEscape(event.reason) + '</td>' +
-                '</tr>';
-            };
-            socket.onerror = updateState;
-            socket.onmessage = function (event) {
-                commsLog.innerHTML += '<tr>' +
-                    '<td class="commslog-server">Server</td>' +
-                    '<td class="commslog-client">Client</td>' +
-                    '<td class="commslog-data">' + htmlEscape(event.data) + '</td></tr>';
-            };
-        };
-
-        function htmlEscape(str) {
-            return str.toString()
-                .replace(/&/g, '&amp;')
-                .replace(/"/g, '&quot;')
-                .replace(/'/g, '&#39;')
-                .replace(/</g, '&lt;')
-                .replace(/>/g, '&gt;');
-        }
-    </script>
-</body>
-</html>

src/DataStreamSuite/ColumnMapExpressions/ColumnMapExpressions.csproj

@@ -1,7 +1,7 @@
 <Project Sdk="Microsoft.NET.Sdk">
 
   <PropertyGroup>
-    <TargetFramework>net8.0</TargetFramework>
+    <TargetFramework>net7.0</TargetFramework>
     <ImplicitUsings>enable</ImplicitUsings>
     <Nullable>enable</Nullable>
   </PropertyGroup>

src/DataStreamSuite/DataStream.Console/DataStream.Console.csproj

@@ -7,7 +7,7 @@
 
   <PropertyGroup>
     <OutputType>Exe</OutputType>
-    <TargetFramework>net8.0</TargetFramework>
+    <TargetFramework>net7.0</TargetFramework>
     <ImplicitUsings>enable</ImplicitUsings>
     <Nullable>enable</Nullable>
   </PropertyGroup>

src/DataStreamSuite/DataStream/DataStream.csproj

@@ -1,7 +1,7 @@
 <Project Sdk="Microsoft.NET.Sdk">
 
   <PropertyGroup>
-    <TargetFramework>net8.0</TargetFramework>
+    <TargetFramework>net7.0</TargetFramework>
     <Nullable>enable</Nullable>
   </PropertyGroup>
 
@@ -15,7 +15,6 @@
   </ItemGroup>
 
   <ItemGroup>
-    <PackageReference Include="HtmlAgilityPack" Version="1.11.57" />
     <PackageReference Include="MongoDB.Bson" Version="2.23.1" />
     <PackageReference Include="MongoDB.Driver" Version="2.23.1" />
   </ItemGroup>

src/DataStreamSuite/DataStream/Primitives/FileStreams/LoaderStream.cs

@@ -30,8 +30,8 @@ namespace BCG
            {
               using (FileStream fs = new FileStream(fileName, FileMode.Open, FileAccess.Read))
               {
-#pragma warning disable SYSLIB0011
                   BinaryFormatter bf = new BinaryFormatter();
+#pragma warning disable SYSLIB0011
                   BlueTable bt = (BlueTable)bf.Deserialize(fs);
 #pragma warning restore SYSLIB0011
                   mySetRealSource(new BlueRowCollectionStaticStream(bt));

src/DataStreamSuite/DataStream/Primitives/KeyValue/KeyValueToRelationalStream.cs

@@ -1,31 +0,0 @@
-namespace BCG {
-    using System.Collections.Generic;
-
-    public class KeyValueToRelationalStream : ViewStream {
-        private readonly ViewStream _kv_str;
-        private readonly string _doc_id_col, _key_col, _val_col;
-        public KeyValueToRelationalStream(ViewStream kv_str, string doc_id_col, string key_col, string val_col) {
-            _kv_str = kv_str;
-            _doc_id_col = doc_id_col;
-            _key_col = key_col;
-            _val_col = val_col;
-        }
-
-        protected override void myOnFirstSubscriber() {
-            _kv_str.onChange += _on_str_update;
-        }
-
-        protected override void myOnLastSubscriber() {
-            _kv_str.onChange -= _on_str_update;
-        }
-
-        private void _on_str_update(object dp_obj) {
-            DataPacket dp = dp_obj as DataPacket;
-            var rows = dp.newRows;
-            var rv = BlueDataUtility.keyvalue_to_relational<string>(rows, _doc_id_col, _key_col, _val_col);
-            var dp_out = BlueDataUtility.build_dp_inc(rv, myLastValue as DataPacket, true);
-            myFireEvent(dp_out);
-        }
-
-    }
-}

src/DataStreamSuite/DataStream/Primitives/Misc/WebScrapingStream.cs

@@ -1,121 +0,0 @@
-using System;
-using System.Collections.Generic;
-using System.IO;
-using System.Security.Cryptography.X509Certificates;
-
-using BCG;
-using BCG.Utilities.General;
-
-using HtmlAgilityPack;
-
-namespace BCG {
-    class Program {
-        static void Main() {
-            string url = "http://quotes.toscrape.com";
-            List<string> quotes = ScrapeQuotes(url);
-
-            Console.WriteLine("Quotes from http://quotes.toscrape.com:");
-            foreach (var quote in quotes) {
-                Console.WriteLine(quote);
-            }
-        }
-
-        static List<string> ScrapeQuotes(string url) {
-            List<string> quotes = new List<string>();
-
-            // Create a new HtmlWeb instance
-            HtmlWeb web = new HtmlWeb();
-
-            try {
-                // Load the HTML document from the specified URL
-                HtmlDocument document = web.Load(url);
-
-                // Extract quotes from the HTML document
-                var quoteNodes = document.DocumentNode.SelectNodes("//span[@class='text']");
-                if (quoteNodes != null) {
-                    foreach (var quoteNode in quoteNodes) {
-                        string quote = quoteNode.InnerText.Trim();
-                        quotes.Add(quote);
-                    }
-                }
-            } catch (Exception ex) {
-                Console.WriteLine($"Error: {ex.Message}");
-            }
-
-            return quotes;
-        }
-    }
-
-    public class WebScapperStream : ViewStream, IWorker {
-        private readonly string _url;
-        private readonly string _xpath;
-        private readonly ViewStreamDatabase _vsd;
-        private IDictionary<string, int> _line_id_map = new Dictionary<string, int>();
-        private readonly IBlueTableSchema _schema;
-
-        public WebScapperStream()
-        :
-        this("http://quotes.toscrape.com", "//span[@class='text']", ViewStreamDatabase.instance) { }
-        public WebScapperStream(string url, string xpath)
-        : this(url, xpath, ViewStreamDatabase.instance) { }
-        public WebScapperStream(string url, string xpath, ViewStreamDatabase vsd) {
-            bts.Log.Gen.info($"WebScapperStream cons url: {url} xpath: {xpath}");
-            _schema = BlueDataUtility.form_schema($"{url}_{xpath}",
-                                                Tuple.Create("id", TypeCode.Int32),
-                                                Tuple.Create("url", TypeCode.UInt32),
-                                                Tuple.Create("line", TypeCode.String));
-            _url = url;
-            _xpath = xpath;
-            _vsd = vsd;
-        }
-
-        protected override void myOnFirstSubscriber() {
-            MasterWorker.getInstance(false).addWorker(this, TimeSpan.FromSeconds(1));
-        }
-
-        protected override void myOnLastSubscriber() {
-            MasterWorker.getInstance(false).removeWorker(this);
-        }
-
-        public void doWork() {
-            var lines = scrape_lines(_url, _xpath);
-            Func<string, int> get_id = (line) => {
-                int rv = 0;
-                if (!_line_id_map.TryGetValue(line, out rv))
-                    _line_id_map.Add(line, _line_id_map.Count);
-                return rv;
-            };
-            BlueTable nrows = new BlueTable(_schema);
-            foreach (var l in lines) {
-                bts.Log.Gen.info($"line: {l}");
-                var id = get_id(l);
-                BlueRow nr = new BlueRow(_schema, get_id(l), _url, l);
-                nrows.addRow(nr);
-            }
-            var dp = BlueDataUtility.build_dp_inc(nrows, myLastValue as DataPacket, true);
-            myFireEvent(dp);
-        }
-
-        static List<string> scrape_lines(string url, string xpath) {
-            List<string> quotes = new List<string>();
-            // Create a new HtmlWeb instance
-            HtmlWeb web = new HtmlWeb();
-            try {
-                // Load the HTML document from the specified URL
-                HtmlDocument document = web.Load(url);
-                // Extract quotes from the HTML document
-                var quote_nodes = document.DocumentNode.SelectNodes(xpath);
-                if (quote_nodes != null) {
-                    foreach (var qn in quote_nodes) {
-                        string quote = qn.InnerText.Trim();
-                        quotes.Add(quote);
-                    }
-                }
-            } catch (Exception ex) {
-                Console.WriteLine($"Error: {ex.Message}");
-            }
-
-            return quotes;
-        }
-    }
-}

src/DataStreamSuite/DataStream/Primitives/MongoDB/MongoDBStream.cs

@@ -1,212 +1,37 @@
 using System;
-using System.Collections.Generic;
+
 using MongoDB.Driver;
 using MongoDB.Bson;
-using BCG;
-using BCG.Utilities.General;
-using ZstdSharp.Unsafe;
-using System.Runtime.CompilerServices;
-using mbts.spark;
 
 
-namespace BCG {
-    [DataStreamUserRespresentation("mongodb_sql")]
-    public class MongoDBSQLStream : ViewStream, IWorker {
-        private ViewStreamDatabase _vsd;
-        private readonly string _connection_string, _db_name, _collection_name;
-        private MongoClient _client;
-        private BlueTable _table;
+class Program {
+    static void Main() {
+        // Set the connection string and database name
+        string connectionString = "mongodb://localhost:27017";
+        string databaseName = "your_database_name";
+        string collectionName = "your_collection_name";
+
+        // Create a MongoClient
+        MongoClient client = new MongoClient(connectionString);
 
         // Access the database
-        private IMongoDatabase _database;
-        private IDictionary<string, int> _id_map = new Dictionary<string, int>();
-        // mongodb://192.168.1.80:27017
+        IMongoDatabase database = client.GetDatabase(databaseName);
 
-        public MongoDBSQLStream(string connection_string, string db_name, string collection_name)
-        : this(connection_string, db_name, collection_name, ViewStreamDatabase.instance) {
+        // Access the collection
+        IMongoCollection<BsonDocument> collection = database.GetCollection<BsonDocument>(collectionName);
 
-        }
-        public MongoDBSQLStream(string connection_string, string db_name, string collection_name, ViewStreamDatabase vsd) {
-            bts.Log.Gen.info($"MongoDBStream constructed connection_string: {connection_string} " +
-                                    $"db_name: {db_name}");
-            _connection_string = connection_string;
-            _db_name = db_name;
-            _collection_name = collection_name;
-            _vsd = vsd;
-            _client = new MongoClient(_connection_string);
-            _database = _client.GetDatabase(_db_name);
-        }
+        // Fetch all documents from the collection
+        var documents = collection.Find(Builders<BsonDocument>.Filter.Empty).ToCursor();
 
-        protected override void myOnFirstSubscriber() {
-            MasterWorker.getInstance(false).addWorker(this, TimeSpan.FromSeconds(1));
-        }
+        // Iterate through the documents and print values
+        foreach (var document in documents.ToEnumerable()) {
+            // Assuming your document structure has fields named "Field1", "Field2", etc.
+            Console.WriteLine("Field1: " + document["Field1"]);
+            Console.WriteLine("Field2: " + document["Field2"]);
+            // Add more fields as needed
 
-        protected override void myOnLastSubscriber() {
-            MasterWorker.getInstance(false).removeWorker(this);
-        }
-
-        public void doWork() {
-            _read_mogodb();
-        }
-
-        private void _read_mogodb() {
-            HighPerformanceTimer timer = new HighPerformanceTimer();
-            timer.start();
-            // Access the collection
-            IMongoCollection<BsonDocument> collection = _database.GetCollection<BsonDocument>(name: _collection_name);
-
-            // Fetch all documents from the collection
-            var documents = collection.Find(Builders<BsonDocument>.Filter.Empty).ToCursor();
-            _table = _table == null ? null : new BlueTable(_table.schema);
-            // Iterate through the documents and print values
-            foreach (var document in documents.ToEnumerable()) {
-                bts.Log.Gen.info($"document: {document} json: {document.ToJson()}");
-                if (_table == null) {
-                    _init_table(document);
-                }
-                _add_row(document);
-            }
-            timer.stop();
-            bts.Log.Gen.info($"MongoDBStream time to read {_collection_name} {timer.lastIntervalTime} msecs");
-            timer.start();
-            var dp = BlueDataUtility.build_dp_inc(_table, myLastValue == null ? null : myLastValue as DataPacket, true);
-            timer.stop();
-            bts.Log.Gen.info($"MongoDBStream time to compare tables :{timer.lastIntervalTime} msecs");
-            if (dp.addedRows.Count == 0 && dp.deletedRows.Count == 0)
-                return;
-            myFireEvent(dp);
-        }
-
-        private void _init_table(BsonDocument doc_row) {
-            bts.Log.assert(_table == null);
-            List<ColumnInfo> cis = new List<ColumnInfo>();
-            cis.Add((new ColumnInfo("id", 0, typeof(Int32))));
-            foreach (var n in doc_row.Names) {
-                cis.Add(new ColumnInfo(n, cis.Count, typeof(string)));
-            }
-            IBlueTableSchema schema = new BlueTableSchema(cis.ToArray(), _collection_name);
-            _table = new BlueTable(schema);
-        }
-
-        private void _add_row(BsonDocument doc) {
-            var key_val = doc["_id"];
-            var key = key_val.ToString();
-            int id;
-            if (!_id_map.TryGetValue(key, out id)) {
-                id = _id_map.Count;
-                _id_map.Add(key, id);
-            }
-            MutableBlueRow nr = new MutableBlueRow(_table.schema);
-            nr.setValue("id", id);
-            foreach (var n in doc.Names) {
-                var val = doc[n].ToString();
-                nr.setValue(n, val);
-            }
-            _table.addRow(nr);
+            // Print a separator between documents
+            Console.WriteLine(new string('-', 20));
         }
     }
-
-    [DataStreamUserRespresentation("mongodb")]
-    public class MongoDBStream : ViewStream, IWorker {
-        private ViewStreamDatabase _vsd;
-        private readonly string _connection_string, _db_name, _collection_name;
-        private MongoClient _client;
-        private BlueTable _table;
-
-        // Access the database
-        private IMongoDatabase _database;
-        private IDictionary<string, int> _id_map = new Dictionary<string, int>();
-        // mongodb://192.168.1.80:27017
-
-        public MongoDBStream(string connection_string, string db_name, string collection_name)
-        : this(connection_string, db_name, collection_name, ViewStreamDatabase.instance) {
-
-        }
-        public MongoDBStream(string connection_string, string db_name, string collection_name, ViewStreamDatabase vsd) {
-            bts.Log.Gen.info($"MongoDBStream constructed connection_string: {connection_string} " +
-                                    $"db_name: {db_name}");
-            _connection_string = connection_string;
-            _db_name = db_name;
-            _collection_name = collection_name;
-            _vsd = vsd;
-            _client = new MongoClient(_connection_string);
-            _database = _client.GetDatabase(_db_name);
-        }
-
-        protected override void myOnFirstSubscriber() {
-            MasterWorker.getInstance(false).addWorker(this, TimeSpan.FromSeconds(1));
-        }
-
-        protected override void myOnLastSubscriber() {
-            MasterWorker.getInstance(false).removeWorker(this);
-        }
-
-        public void doWork() {
-            _read_mogodb();
-        }
-
-        private void _read_mogodb() {
-            HighPerformanceTimer timer = new HighPerformanceTimer();
-            timer.start();
-            // Access the collection
-            IMongoCollection<BsonDocument> collection = _database.GetCollection<BsonDocument>(name: _collection_name);
-
-            // Fetch all documents from the collection
-            var documents = collection.Find(Builders<BsonDocument>.Filter.Empty).ToCursor();
-            _table = _table == null ? null : new BlueTable(_table.schema);
-            // Iterate through the documents and print values
-            foreach (var document in documents.ToEnumerable()) {
-                bts.Log.Gen.info($"document: {document} json: {document.ToJson()}");
-                if (_table == null) {
-                    _init_table(document);
-                }
-                _add_doc(document);
-            }
-            timer.stop();
-            bts.Log.Gen.info($"MongoDBStream time to read {_collection_name} {timer.lastIntervalTime} msecs");
-            timer.start();
-            var dp = BlueDataUtility.build_dp_inc(_table, myLastValue == null ? null : myLastValue as DataPacket, true);
-            timer.stop();
-            bts.Log.Gen.info($"MongoDBStream time to compare tables :{timer.lastIntervalTime} msecs");
-            if (dp.addedRows.Count == 0 && dp.deletedRows.Count == 0)
-                return;
-            myFireEvent(dp);
-        }
-
-        private void _init_table(BsonDocument doc_row) {
-            bts.Log.assert(_table == null);
-            var schema = BlueDataUtility.form_schema($"{_database}_{_table}",
-            Tuple.Create("id", TypeCode.Int32),
-            Tuple.Create("doc_id", TypeCode.String),
-            Tuple.Create("key", TypeCode.String),
-            Tuple.Create("value", TypeCode.String));
-            _table = new BlueTable(schema);
-        }
-
-        private void _add_doc(BsonDocument doc) {
-            var doc_id_bson = doc["_id"];
-            var doc_id = doc_id_bson.ToString();
-            Func<string, string, int> get_id = (string doc_id, string key) => {
-                string hash_key = $"{doc_id}_{key}";
-                int id = 0;
-                if (!_id_map.TryGetValue(hash_key, out id)) {
-                    id = _id_map.Count;
-                    _id_map.Add(hash_key, id);
-                }
-                return id;
-            };
-            foreach (var n in doc.Names) {
-                string key = n;
-                var id = get_id(doc_id, key);
-                MutableBlueRow nr = new MutableBlueRow(_table.schema);
-                var val = doc[n].ToString();
-                nr.setValue("id", id);
-                nr.setValue("doc_id", doc_id);
-                nr.setValue("key", key);
-                nr.setValue("value", val);
-                _table.addRow(nr);
-            }
-        }
-    }
-
-}
+}

src/DataStreamSuite/DataStreamCore/BlueData/ColumnInfo.cs

@@ -1,103 +1,95 @@
-using System;
-using System.Text.Json.Serialization;
-
-namespace BCG
-{
-   /// <summary>
-   /// Resprsents information about a column schema, ie its ordinal, the type, the name and constant value or expression. where applicable
-   /// </summary>
-   [Serializable]
-   public class ColumnInfo : IComparable
-   {
-      public static bool isAllowedType (Type t)
-      {
-         return 
-            t.Equals (typeof (System.Int32)) ||
-            t.Equals (typeof (System.Int64)) ||
-            t.Equals (typeof (System.Double)) ||
-            t.Equals (typeof (System.String)) ||
-            t.Equals (typeof (System.DateTime)) ? true :
-            false;
-      }
-
-      //the default column info does not cach values in a lookup table, therefore it is in group 0
-      public ColumnInfo(string colName, int colNo, Type colType)
-         : this(colName, colNo, colType, 0)
-      {
-
-
-      }
-
-      public ColumnInfo(string colName, int colNo, Type colType, short colGroup)
-      {
-         myColName = colName;
-         myColNo = colNo;
-         myColType = colType;
-         myLookupTableGroup = colGroup;
-         myColValue = null;
-      }
-
-
-      public ColumnInfo (string colName, int colNo, object colValue)
-         : this (colName, colNo, colValue.GetType())  {myColValue = colValue == null ? DBNull.Value : colValue;}
-
-      public ColumnInfo (string colName, int colNo, Type colType, object colValue)
-         : this (colName, colNo, colType) 
-      {
-
-         myColValue = colValue == null ? DBNull.Value : colValue;
-      
-      }
-
-
-      public int CompareTo (object obj)
-      {
-         int otherColNo = ((ColumnInfo)(obj)).colNo;
-         if (myColNo < otherColNo)
-            return -1;
-         else if (myColNo > otherColNo)
-            return 1;
-         return 0;
-      }
-
-      //public bool isCalculated{get{return myExpression != null;}}
-
-      public bool isConstant {get{return myColValue != null;}}
-
-      private int myColNo;
-      private string myColName;
-      private Type myColType;
-      //private string myExpression;
-      private object myColValue;
-
-      //no table will ever have more than 65,536 columns
-      //if it does, we are in serious trouble
-      private short myLookupTableGroup;
-
-      [JsonPropertyName("colNo")]
-      public int colNo{get{return myColNo;}}
-
-      [JsonPropertyName("colName")]
-      public string colName {get{return myColName;}}
-
-      [JsonPropertyName("colType")]
-      public Type colType {get{return myColType;}}
-
-
-      //this value corresponds to how the data in the column is serialized. 
-      //if the group number is 0, the values are not stored in a table
-      //if the vlaue is non-zero, all columns with the the same lookup table group are stored in the same lookup table together
-      //are stored together in a lookup table
-
-      
-      [JsonIgnore]
-      public short lookupTableGroup
-      {
-         get { return myLookupTableGroup; }
-         set { myLookupTableGroup = value; }
-      }
-      //public string expression {get{return myExpression;}}
-      [JsonIgnore]
-      public object colValue {get{return myColValue;}}
-   }
-}
+using System;
+
+namespace BCG
+{
+   /// <summary>
+   /// Resprsents information about a column schema, ie its ordinal, the type, the name and constant value or expression. where applicable
+   /// </summary>
+   [Serializable]
+   public class ColumnInfo : IComparable
+   {
+      public static bool isAllowedType (Type t)
+      {
+         return 
+            t.Equals (typeof (System.Int32)) ||
+            t.Equals (typeof (System.Int64)) ||
+            t.Equals (typeof (System.Double)) ||
+            t.Equals (typeof (System.String)) ||
+            t.Equals (typeof (System.DateTime)) ? true :
+            false;
+      }
+
+      //the default column info does not cach values in a lookup table, therefore it is in group 0
+      public ColumnInfo(string colName, int colNo, Type colType)
+         : this(colName, colNo, colType, 0)
+      {
+
+
+      }
+
+      public ColumnInfo(string colName, int colNo, Type colType, short colGroup)
+      {
+         myColName = colName;
+         myColNo = colNo;
+         myColType = colType;
+         myLookupTableGroup = colGroup;
+         myColValue = null;
+      }
+
+
+      public ColumnInfo (string colName, int colNo, object colValue)
+         : this (colName, colNo, colValue.GetType())  {myColValue = colValue == null ? DBNull.Value : colValue;}
+
+      public ColumnInfo (string colName, int colNo, Type colType, object colValue)
+         : this (colName, colNo, colType) 
+      {
+
+         myColValue = colValue == null ? DBNull.Value : colValue;
+      
+      }
+
+
+      public int CompareTo (object obj)
+      {
+         int otherColNo = ((ColumnInfo)(obj)).colNo;
+         if (myColNo < otherColNo)
+            return -1;
+         else if (myColNo > otherColNo)
+            return 1;
+         return 0;
+      }
+
+      //public bool isCalculated{get{return myExpression != null;}}
+
+      public bool isConstant {get{return myColValue != null;}}
+
+      private int myColNo;
+      private string myColName;
+      private Type myColType;
+      //private string myExpression;
+      private object myColValue;
+
+      //no table will ever have more than 65,536 columns
+      //if it does, we are in serious trouble
+      private short myLookupTableGroup;
+
+      public int colNo{get{return myColNo;}}
+      public string colName {get{return myColName;}}
+      public Type colType {get{return myColType;}}
+
+
+      //this value corresponds to how the data in the column is serialized. 
+      //if the group number is 0, the values are not stored in a table
+      //if the vlaue is non-zero, all columns with the the same lookup table group are stored in the same lookup table together
+      //are stored together in a lookup table
+
+      
+      public short lookupTableGroup
+      {
+         get { return myLookupTableGroup; }
+         set { myLookupTableGroup = value; }
+      }
+      //public string expression {get{return myExpression;}}
+      public object colValue {get{return myColValue;}}
+   }
+}

src/DataStreamSuite/DataStreamCore/BlueData/MetaDataField.cs

@@ -3,7 +3,8 @@ using System.Collections.Generic;
 using System.Linq;
 using System.Text;
 
-namespace BCG {
+namespace BCG
+{
    /*
       These fields are first going to be used by  BlueRay to help it determine what differnet tags mean, but eventually
     * the goal is that they will be used for all services in this system. 
@@ -11,12 +12,11 @@ namespace BCG {
     * not all metadata fields are present in all metadata; it depends upon the class. For example, a bluepages service
     * probably won't have a 'symbol' or 'exchange' field populated, but it'll probably have the 'type' of service populated
     */
-   public enum MetaDataField {
-      Protocol,       // is this a bluepages service, a pubsub tag, a blueray tag, a 29 west thing, or what
-      Type,           // if it's a multicast, is it option, fireprice, whatever. If it's bluepages, is it persistence, pubsub, whatever
-      Symbol,
-      Exchange,
-      MarketDataType, // BAM (bid, ask, mid), last, etc
-      RealName
-   }; //does this thing have a better name we could call it
+   public enum MetaDataField {   Protocol,  //is this a bluepages service, a pubsub tag, a blueray tag, a 29 west thing, or what
+                                 Type,   //if it's a multicast, is it option, fireprice, whatever. If it's bluepages, is it persistence, pubsub, whatever
+                                 Symbol, 
+                                 Exchange,
+                                 Platform,
+                                 MarketDataType, // BAM (bid, ask, mid), last, etc
+                                 RealName }; //does this thing have a better name we could call it
 }

src/DataStreamSuite/DataStreamCore/BlueRowJsonConverter.cs

@@ -1,61 +0,0 @@
-#nullable enable
-using BCG;
-
-using System;
-using System.Text.Json;
-using System.Text.Json.Serialization;
-
-namespace Bts.DataStreamCore;
-
-class BlueRowJsonConverter : JsonConverter<IBlueRow> {
-   IBlueTableSchema? schema = null;
-
-   public override IBlueRow Read(
-      ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options) {
-      if (schema is null) {
-         throw new InvalidOperationException("Cannot read row from json because schema is null");
-      }
-      var mbr = new MutableBlueRow(schema);
-      int colNum = 0;
-      switch (reader.TokenType) {
-         case JsonTokenType.StartArray:
-            while (reader.Read() && reader.TokenType != JsonTokenType.EndArray) {
-               mbr.setValue(colNum++, readCellValue(ref reader));
-               colNum++;
-            }
-            break;
-         case JsonTokenType.StartObject:
-            while (reader.Read() && reader.TokenType != JsonTokenType.EndObject) {
-               if (reader.TokenType != JsonTokenType.PropertyName) {
-                  throw new InvalidOperationException(
-                     "Expected 'PropertyName' immediately after 'StartObject' token in row json");
-               }
-               var colName = reader.GetString();
-               if (!reader.Read()) {
-                  throw new InvalidOperationException(
-                     "Failed to read token immediately after 'PropertyName' token in row json");
-               }
-               mbr.setValue(colName, readCellValue(ref reader));
-            }
-            break;
-      }
-      if (mbr.Count != schema.columnInfo.Length) {
-         throw new InvalidOperationException(
-            $"Row json array length '{mbr.Count}' does not match the schema's " +
-            $"column length '{schema.columnInfo.Length}'.");
-      }
-      return mbr;
-
-      static object? readCellValue(ref Utf8JsonReader reader) =>
-         reader.TokenType switch {
-            JsonTokenType.String => reader.GetString(),
-            JsonTokenType.Number => reader.GetDouble(),
-            JsonTokenType.True or JsonTokenType.False => reader.GetBoolean(),
-            _ => throw new InvalidOperationException("Failed to read cell value"),
-         };
-   }
-
-   public override void Write(Utf8JsonWriter writer, IBlueRow value, JsonSerializerOptions options) {
-      throw new NotImplementedException();
-   }
-}

src/DataStreamSuite/DataStreamCore/DataPacketJsonConverter.cs

@@ -1,154 +0,0 @@
-#nullable enable
-using BCG;
-
-using Spark.Core.Json;
-
-using System;
-using System.Collections.Generic;
-using System.Linq;
-using System.Text.Json;
-using System.Text.Json.Serialization;
-
-namespace Bts.DataStreamCore;
-
-class DataPacketJsonConverter : JsonConverter<DataPacket> {
-   /// <summary>
-   /// Whether or not to include a field for current rows (i.e., snapshot of current state). By
-   /// default, only incremental changes are included ('added' and 'deleted').
-   /// </summary>
-   public bool IncludeCurrent { get; set; }
-
-   /// <summary>
-   /// Whether to treat rows as objects (name/value pairs), or arrays (list of values). Arrays
-   /// result in smaller payloads, but aren't as easy to read as name/value pairs (for mere humans).
-   /// </summary>
-   public bool UseObjectRows { get; set; }
-
-   public override DataPacket? Read(ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options) {
-      if (!reader.ReadObject()) {
-         return null;
-      }
-      if (!reader.ReadPropertyName(out var prop)) {
-         return null;
-      }
-      if (!reader.Read()) {
-         return null;
-      }
-      if (prop == "added") {
-         var added = new List<MutableBlueRow>();
-         if (reader.TokenType != JsonTokenType.StartArray) {
-            return null;
-         }
-         while (true) {
-            if (!reader.Read()) {
-               break;
-            }
-            if (reader.TokenType != JsonTokenType.StartArray || !reader.Read()) {
-               return null;
-            }
-            switch (reader.TokenType) {
-               case JsonTokenType.String:
-                  // DAMNIT! Need the schema.
-                  break;
-            }
-         }
-      }
-   }
-
-   public override void Write(Utf8JsonWriter writer, DataPacket value, JsonSerializerOptions options) {
-      using var _ = writer.WriteObject();
-      using (writer.WriteArray("schema")) {
-         foreach (var col in value.schema.columnInfo) {
-            using (writer.WriteObject()) {
-               writer.WriteString("name", col.colName);
-               writer.WriteNumber("index", col.colNo);
-               // TODO: use language agnostic type names (e.g., 'string', 'integer', 'float', 'date' etc.)
-               writer.WriteString("type", col.colType.ToString());
-            }
-         }
-      }
-
-      var cellWriters = value.schema.columnInfo.Select(getCellWriter).ToList();
-      using (writer.WriteArray("added")) {
-         foreach (IBlueRow row in value.addedRows) {
-            using (writer.WriteArray()) {
-               for (int i = 0; i < cellWriters.Count; i++) {
-                  cellWriters[i](row[i]);
-               }
-            }
-         }
-      }
-      using (writer.WriteArray("deleted")) {
-         foreach (IBlueRow row in value.deletedRows) {
-            using (writer.WriteArray()) {
-               for (int i = 0; i < cellWriters.Count; i++) {
-                  cellWriters[i](row[i]);
-               }
-            }
-         }
-      }
-      if (IncludeCurrent) {
-         using (writer.WriteArray("current")) {
-            foreach (IBlueRow row in value.newRows) {
-               using (writer.WriteArray()) {
-                  for (int i = 0; i < cellWriters.Count; i++) {
-                     cellWriters[i](row[i]);
-                  }
-               }
-            }
-         }
-      }
-
-      Action<object?> getCellWriter(ColumnInfo colInfo) {
-         var writeVal = getCellValueWriter(colInfo);
-         if (!UseObjectRows) {
-            return writeVal;
-         }
-         return v => {
-            writer.WritePropertyName(colInfo.colName);
-            writeVal(v);
-         };
-      }
-
-      Action<object?> getCellValueWriter(ColumnInfo colInfo) {
-         var t = colInfo.colType;
-         if (t == typeof(int)) {
-            return nullCheckWrapper(v => writer.WriteNumberValue((int)v));
-         }
-         if (t == typeof(uint)) {
-            return nullCheckWrapper(v => writer.WriteNumberValue((uint)v));
-         }
-         if (t == typeof(long)) {
-            return nullCheckWrapper(v => writer.WriteNumberValue((long)v));
-         }
-         if (t == typeof(ulong)) {
-            return nullCheckWrapper(v => writer.WriteNumberValue((ulong)v));
-         }
-         if (t == typeof(short)) {
-            return nullCheckWrapper(v => writer.WriteNumberValue((short)v));
-         }
-         if (t == typeof(ushort)) {
-            return nullCheckWrapper(v => writer.WriteNumberValue((ushort)v));
-         }
-         if (t == typeof(double)) {
-            return nullCheckWrapper(v => writer.WriteNumberValue((double)v));
-         }
-         if (t == typeof(float)) {
-            return nullCheckWrapper(v => writer.WriteNumberValue((float)v));
-         }
-         if (t == typeof(bool)) {
-            return nullCheckWrapper(v => writer.WriteBooleanValue((bool)v));
-         }
-         return nullCheckWrapper(v => writer.WriteStringValue(v?.ToString()));
-
-         Action<object?> nullCheckWrapper(Action<object> writeNonNull) =>
-            v => {
-               if (v is null) {
-                  writer.WriteNullValue();
-               } else {
-                  writeNonNull(v);
-               }
-            };
-      }
-   }
-}

src/DataStreamSuite/DataStreamCore/DataStreamCore.csproj

@@ -2,7 +2,7 @@
 
   <PropertyGroup>
     <AllowUnsafeBlocks>true</AllowUnsafeBlocks>
-    <TargetFramework>net8.0</TargetFramework>
+    <TargetFramework>net7.0</TargetFramework>
     <Nullable>disable</Nullable>
   </PropertyGroup>
 

src/DataStreamSuite/DataStreamExpressions/DataStreamExpressions.csproj

@@ -1,7 +1,7 @@
 <Project Sdk="Microsoft.NET.Sdk">
 
   <PropertyGroup>
-    <TargetFramework>net8.0</TargetFramework>
+    <TargetFramework>net7.0</TargetFramework>
     <ImplicitUsings>enable</ImplicitUsings>
     <Nullable>enable</Nullable>
   </PropertyGroup>

src/DataStreamSuite/DataStreamPublisher/DataStreamPublisher.csproj

@@ -11,7 +11,7 @@
 
   <PropertyGroup>
     <OutputType>Exe</OutputType>
-    <TargetFramework>net8.0</TargetFramework>
+    <TargetFramework>net7.0</TargetFramework>
     <Nullable>enable</Nullable>
     <AllowUnsafeBlocks>true</AllowUnsafeBlocks>
   </PropertyGroup>

src/DataStreamSuite/DataStreamPublisher/Program.cs

@@ -11,7 +11,7 @@ using System.Threading;
 
 bts.Log.Gen.info($"Starting DSPublisher");
 Console.WriteLine("Starting DSPublisher...");
-bool run_echo_tcp_client_only = false;
+bool run_echo_tcp_client_only = true;
 static void init(string[] args) {
    Func<string, string, Task> login = async (l, p) => {
       var login = l;
@@ -31,7 +31,6 @@ else
 
 static ViewStream build_stream(string stream_expression) {
    try {
-      bts.Log.Gen.info($"build_stream stream_expression: {stream_expression}");
       var exp = DataStreamExpressionsInterpreter.Evaluate(stream_expression);
       var rv = exp.getView(new Dictionary<string, StatementExpression>(0));
       return rv;
@@ -59,16 +58,14 @@ void _publish_streams() {
    CloudStreamManager csm = new CloudStreamManager();
    // get it from config
    Dictionary<string, string> streams_to_publish = new Dictionary<string, string>() {
-        //{"bbo","bbo()"},
-        //{"instruments","instruments()"},
+        {"bbo","bbo()"},
+        {"instruments","instruments()"},
         //{"timesales","timesales(10000)"},
-        //{"books","books()"},
-       {"seconds_1_10","second(1,10)"},
+        {"books","books()"},
+      // {"seconds_1_10","second(1,10)"},
         {"literal_420","literal(\"x\", \"420.0\")"},
-        {"mongodb_testcollection", "mongodb(\"mongodb://192.168.1.80:27017\",\"local\",\"test_collection\")"}
-        };
-   //  {"bbo_ts","timeSeries(bbo())"}
-
+      //  {"bbo_ts","timeSeries(bbo())"}
+   };
    foreach (var kv in streams_to_publish) {
       var name = kv.Key;
       var exp = kv.Value;
@@ -80,7 +77,6 @@ void _publish_streams() {
       System.Threading.Thread.Sleep(TimeSpan.FromSeconds(1));
       ViewStreamDatabase.instance.doWork();
       csm.doWork();
-      MasterWorker.getInstance(false).doWork();
    }
 }
 

src/DataStreamSuite/DataStreamQueryAnalyzer/DataStreamQueryAnalyzer.csproj

@@ -2,7 +2,7 @@
 
   <PropertyGroup>
     <OutputType>Exe</OutputType>
-    <TargetFramework>net8.0-windows</TargetFramework>
+    <TargetFramework>net7.0-windows</TargetFramework>
     <Nullable>enable</Nullable>
     <UseWindowsForms>true</UseWindowsForms>
   </PropertyGroup>

src/DataStreamSuite/DataStreamSuite.sln

@@ -38,8 +38,6 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "DataStreamTests", "DataStre
 EndProject
 Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "SharedStreams", "SharedStreams\SharedStreams.csproj", "{C1EBC423-4668-43D4-B7E5-A291086BA823}"
 EndProject
-Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Bts.DataStreamWebApp.Server", "Bts.DataStreamWebApp.Server\Bts.DataStreamWebApp.Server.csproj", "{6DE5E4CE-423E-4571-81F7-539D268D6059}"
-EndProject
 Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Bts.DataStream.Websocket", "Bts.DataStream.Websocket\Bts.DataStream.Websocket.csproj", "{088655DA-3126-4F8A-9AC7-9061FDA891FF}"
 EndProject
 Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Bts.DataStream.WebsocketServer", "Bts.DataStream.WebsocketServer\Bts.DataStream.WebsocketServer.csproj", "{35823BE0-C883-476A-8B30-D6097CBA1A8D}"
@@ -133,10 +131,6 @@ Global
 		{C1EBC423-4668-43D4-B7E5-A291086BA823}.Debug|Any CPU.Build.0 = Debug|Any CPU
 		{C1EBC423-4668-43D4-B7E5-A291086BA823}.Release|Any CPU.ActiveCfg = Release|Any CPU
 		{C1EBC423-4668-43D4-B7E5-A291086BA823}.Release|Any CPU.Build.0 = Release|Any CPU
-		{6DE5E4CE-423E-4571-81F7-539D268D6059}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
-		{6DE5E4CE-423E-4571-81F7-539D268D6059}.Debug|Any CPU.Build.0 = Debug|Any CPU
-		{6DE5E4CE-423E-4571-81F7-539D268D6059}.Release|Any CPU.ActiveCfg = Release|Any CPU
-		{6DE5E4CE-423E-4571-81F7-539D268D6059}.Release|Any CPU.Build.0 = Release|Any CPU
 		{088655DA-3126-4F8A-9AC7-9061FDA891FF}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
 		{088655DA-3126-4F8A-9AC7-9061FDA891FF}.Debug|Any CPU.Build.0 = Debug|Any CPU
 		{088655DA-3126-4F8A-9AC7-9061FDA891FF}.Release|Any CPU.ActiveCfg = Release|Any CPU

src/DataStreamSuite/DataStreamTests/DataStreamTests.csproj

@@ -1,7 +1,7 @@
 <Project Sdk="Microsoft.NET.Sdk">
 
   <PropertyGroup>
-    <TargetFramework>net8.0</TargetFramework>
+    <TargetFramework>net7.0</TargetFramework>
     <ImplicitUsings>enable</ImplicitUsings>
     <Nullable>enable</Nullable>
 

src/DataStreamSuite/ExpressionCore/ExpressionCore.csproj

@@ -1,7 +1,7 @@
 <Project Sdk="Microsoft.NET.Sdk">
 
   <PropertyGroup>
-    <TargetFramework>net8.0</TargetFramework>
+    <TargetFramework>net7.0</TargetFramework>
     <ImplicitUsings>enable</ImplicitUsings>
     <Nullable>enable</Nullable>
   </PropertyGroup>

src/DataStreamSuite/ExpressionTester/ExpressionTester.csproj

@@ -6,7 +6,7 @@
 
   <PropertyGroup>
     <OutputType>Exe</OutputType>
-    <TargetFramework>net8.0</TargetFramework>
+    <TargetFramework>net7.0</TargetFramework>
     <ImplicitUsings>enable</ImplicitUsings>
     <Nullable>enable</Nullable>
   </PropertyGroup>

src/DataStreamSuite/ExpressionTests/ExpressionTests.csproj

@@ -1,7 +1,7 @@
 <Project Sdk="Microsoft.NET.Sdk">
 
   <PropertyGroup>
-    <TargetFramework>net8.0</TargetFramework>
+    <TargetFramework>net7.0</TargetFramework>
     <ImplicitUsings>enable</ImplicitUsings>
     <Nullable>enable</Nullable>
 

src/DataStreamSuite/GuiUtilities/GuiUtilities.csproj

@@ -1,7 +1,7 @@
 <Project Sdk="Microsoft.NET.Sdk">
 
   <PropertyGroup>
-    <TargetFramework>net8.0-windows</TargetFramework>
+    <TargetFramework>net7.0-windows</TargetFramework>
     <ImplicitUsings>enable</ImplicitUsings>
     <Nullable>enable</Nullable>
     <UseWindowsForms>true</UseWindowsForms>

src/DataStreamSuite/PubSub/Client/Communication/IPubSubServerConnection.cs

@@ -47,30 +47,30 @@ namespace BCG {
       /// of Int32.Max would return all the messages it has
       /// received between last getUpdates and this call.
       /// </summary
-      IList<byte[]> get_updates(TimeSpan timeout, int maxElements);
+      IList<byte[]> getUpdates(TimeSpan timeout, int maxElements);
       /// <summary>
       /// server URL to which the connection is made
       /// </summary>
-      string server_url { get; }
+      string serverURL { get; }
       /// <summary>
       /// client URL as seen by the server. if a connection has not
       /// been established (as can happen in TCP) an empty string 
       /// is returned
       /// </summary>
-      string client_url { get; }
+      string clientURL { get; }
 
       /// <summary>
       /// registers the callback to receive 
       /// connection notifications
       /// </summary>
-      void register_connection_callback(PubSubServerConnectionStatusChange del);
+      void registerCallback(ConnectionStatusChange del);
 
       /// <summary>
       /// registers callbacks to reveive data
       /// </summary>
       /// <param name="del"></param>
-      void register_data_callback(PubSubServerDataDelegate del);
+      void registerDataCallback(DataDelegate del);
 
-      void register_send_confirmation_callback(SendConfirmationDelegate del);
+      void registerSendConfirmationCallback(SendConfirmationDelegate del);
    }
 }

src/DataStreamSuite/PubSub/Client/Communication/ManagedTCP/PubSubServerManagedTCPConnection.cs

@@ -8,7 +8,6 @@ using System.IO;
 //using BCG.Utilities.General;
 using System.Threading;
 using mbts;
-using bts;
 
 namespace BCG {
    /// <summary>
@@ -21,27 +20,26 @@ namespace BCG {
    /// ITCPConnectionHandler.onConnection will be called on any arbitrary thread
    /// 
    /// </summary>
-   public class PubSubServerManagedTCPConnection : IPubSubServerConnection,// external facing interface to communicate msgs to Pubsubserver   
-                                                   ITCPMessageHandler,     // to get messages from ITCPCOnnection
-                                                   ITCPConnectionHandler { // to get connections from AsyncTCPConnector_APM
+   public class PubSubServerManagedTCPConnection : IPubSubServerConnection,
+                                                   ITCPMessageHandler,
+                                                   ITCPConnectionHandler {
       private ITCPConnection _connection, _new_connection;
       private readonly BlutoSubAddress _server_address;
       // if a message(with confirm) can not be sent , its id is queued here, 
       // on reconnection, _on_send_onfirm_callback is fired with a negative on these ids, so the user
       //can re-send it. The re-send mechanism is not in implemented in this class, but by the user of this class
       private ThreadSafeQueue<string> _negative_unsent_confs = new ThreadSafeQueue<string>();
-      // in onData, data is queued here, for polling
+      // in onData, data is queued here
       private SyncQueue<byte[]> _in_data_queue = new SyncQueue<byte[]>();
       private DateTime _last_hb = DateTime.Now;
-      private PubSubServerConnectionStatusChange _conn_callback;
-      private PubSubServerDataDelegate _data_callback;
-      private SendConfirmationDelegate _on_send_confirm_callback;
+      private ConnectionStatusChange _conn_callback;
+      private DataDelegate _data_callback;
+      private SendConfirmationDelegate _on_send_onfirm_callback;
       private readonly TimeSpan _connection_drop_threshold =
-         new TimeSpan(0, 0, 0, 0,
-                     PubSub3Config.instance.PS3_HeartBeatInterval * PubSub3Config.instance.PS3_MissingHeartBeatsForGC);
+         new TimeSpan(0, 0, 0, 0, PubSub3Config.instance.PS3_HeartBeatInterval * PubSub3Config.instance.PS3_MissingHeartBeatsForGC);
 
-      public PubSubServerManagedTCPConnection(string server_url)
-         : this(new BlutoSubAddress(server_url)) { }
+      public PubSubServerManagedTCPConnection(string serverURL)
+         : this(new BlutoSubAddress(serverURL)) { }
 
       public PubSubServerManagedTCPConnection(BlutoSubAddress server_address) {
          _server_address = server_address;
@@ -50,17 +48,17 @@ namespace BCG {
 
       private void _connect_to_ps_server() {
          bts.Log.Gen.info($"PubSubServerManagedTCPConnection._connect_to_ps_server server: {_server_address}");
-         AsyncTCPConnector_APM.connect_to(_server_address, this);
+         ClientToServerManagedTCPConnector.connect_to(_server_address, this, TimeSpan.FromSeconds(1));
       }
 
       #region IPubSubServerConnection Members
 
-      public void send_with_confirmation(byte[] data, ref string conf_id) {
-         bts.Log.Gen.info($"PubSubServerManagedTCPConnection.sendWithConfirmation server: {_server_address} data: {data.Length} _connection : {_connection != null}");
+      public void sendWithConfirmation(byte[] data, ref string conf_id) {
+         bts.Log.Gen.info($"PubSubServerManagedTCPConnection.sendWithConfirmation server: {_server_address} data: {data.Length} myConnection : {_connection != null}");
          if (_connection != null) {
             _connection.sendWithConfirmation(data, ref conf_id, (msg_id, confirmed) => {
-               if (_on_send_confirm_callback != null)
-                  _on_send_confirm_callback(msg_id, confirmed);
+               if (_on_send_onfirm_callback != null)
+                  _on_send_onfirm_callback(msg_id, confirmed);
             });
          } else {
             bts.Log.Gen.warn($"PubSubServerManagedTCPConnection.sendWithConfirmation server: {_server_address} no connection, buffering send");
@@ -88,18 +86,18 @@ namespace BCG {
 
       public void send_with_confirmation(PubSubMessage msg, ref string conf_id) {
          var bytes = msg.to_bytes();
-         send_with_confirmation(bytes, ref conf_id);
+         sendWithConfirmation(bytes, ref conf_id);
       }
 
-      public IList<byte[]> get_updates(TimeSpan timeout, int maxElements) {
+      public IList<byte[]> getUpdates(TimeSpan timeout, int maxElements) {
          return _in_data_queue.dequeueMultipleElements(Convert.ToInt32(timeout.TotalMilliseconds), maxElements);
       }
 
-      public string server_url {
+      public string serverURL {
          get { return _server_address.ToString(); }
       }
 
-      public string client_url {
+      public string clientURL {
          get {
             string rv = "";
             lock (this) {
@@ -120,16 +118,16 @@ namespace BCG {
          }
       }
 
-      public void register_connection_callback(PubSubServerConnectionStatusChange del) {
+      public void registerCallback(ConnectionStatusChange del) {
          _conn_callback = del;
       }
 
-      public void register_data_callback(PubSubServerDataDelegate del) {
+      public void registerDataCallback(DataDelegate del) {
          _data_callback = del;
       }
 
-      public void register_send_confirmation_callback(SendConfirmationDelegate del) {
-         _on_send_confirm_callback = del;
+      public void registerSendConfirmationCallback(SendConfirmationDelegate del) {
+         _on_send_onfirm_callback = del;
       }
 
       #endregion
@@ -187,6 +185,18 @@ namespace BCG {
                break;
 
          }
+         // this stupid shit, caused duplicating og messages and 2 days of wild goose chase
+         // if (message.messageType == RemoteMessageType.Data)
+         // {
+         //    if (_data_callback != null)
+         //       _data_callback(message.remoteConnectionInfo.ToString(), message.message);
+         //    else
+         //       _in_data_queue.enqueue(message.message);
+         // }
+         // else
+         // {
+         //    _last_hb = DateTime.Now;
+         // }
       }
 
       #endregion
@@ -194,7 +204,8 @@ namespace BCG {
       #region ITCPConnectionHandler Members
 
       /// <summary>
-      ///  will be called on ana rbitrary .Net thread
+      /// This function is called on the main thread (as all other functions
+      /// except onData)
       /// </summary>
       /// <param name="connection"></param>
       public void onConnection(ITCPConnection connection) {
@@ -235,10 +246,10 @@ namespace BCG {
             _connection.register(this);
             if (_conn_callback != null)
                _conn_callback(_server_address.ToString(), true);
-            if (_on_send_confirm_callback != null) {
+            if (_on_send_onfirm_callback != null) {
                var msg_ids = _negative_unsent_confs.dequeueAll();
                foreach (var id in msg_ids)
-                  _on_send_confirm_callback(id, false);
+                  _on_send_onfirm_callback(id, false);
             }
          }
       }
@@ -253,6 +264,47 @@ namespace BCG {
       public void doWork() {
          //_dispose_and_reconnect_if_missing_hb();
          _check_for_new_connection();
+         ///check for heartbeats and let clients know in 
+         ///case of missing hb (connection breaking)
+         // TimeSpan hb_interval = DateTime.Now - myLastHB;
+         // bool hb_missed = hb_interval > myConnectionDropThreshold;
+         // bool conn_disposed = myConnection != null || myConnection.isDisposed;
+         // if (conn_disposed || hb_missed)
+         // {
+         //    if (hb_missed)
+         //    {
+         //       // since conn breakage was diagnosed logically, lets physically close the connection
+         //       myConnection.Dispose();
+         //       string dispose_msg = $"PubSubServerManagedTCPConnection.doWork server: {myServerAddress} Disposing connection conn_disposed: {conn_disposed} hb_missed: {hb_missed}";
+         //       Console.WriteLine(dispose_msg);
+         //    }
+         //    myConnection = null; // this is used as a marker for a dead connection
+         //    if (myConnCallback != null)
+         //       myConnCallback(myServerAddress.ToString(), false);
+         //    string reconn_msg = $"PubSubServerManagedTCPConnection.doWork disconnecting client from server: {myServerAddress} conn_disposed: {conn_disposed} hb_missed: {hb_missed} " +
+         //    $"because of missing hbs last_hb: {myLastHB}, now: {DateTime.Now} curr_hb_interval: {hb_interval.TotalSeconds} threshold: {myConnectionDropThreshold.TotalSeconds} conn_disposed: {conn_disposed}";
+         //    ClientToServerManagedTCPConnector.connectTo(myServerAddress, this);
+         // }
+         // if (myTempNewConnection != null)
+         // {
+         //    if (myConnection != null)
+         //    {
+         //       // TODOds
+         //       // log error and close it
+         //       // BlueEventLog.logAssert(myConnection == null, "Connection has to be nulled out and disposed "
+         //       //                                                 + " before a new connection is established");
+         //    }
+         //    Console.WriteLine("PubSubServerManagedTCPConnection hooking new connection");
+         //    myConnection = myTempNewConnection;
+         //    myTempNewConnection = null;
+         //    myConnection.register(this);
+         //    myConfs.dequeueAll();
+         //    if (myConnCallback != null)
+         //       myConnCallback(myServerAddress.ToString(), true);
+         // }
+         // foreach (Pair<string, bool> p in myConfs.dequeueAll())
+         //    if (myOnSendConfirmCallback != null)
+         //       myOnSendConfirmCallback(p.first, p.second);
       }
 
       #endregion

src/DataStreamSuite/PubSub/Client/MetaData/MetaData.cs

@@ -16,7 +16,6 @@ namespace BCG {
    // SubMetaData
    // TransactionMetaData
    // UnSubMetaData
-   // 
    public class MetaData : ICloneable {
       private IDictionary _name_value_map = new Dictionary<string, string>();
       int mySize = -1;

src/DataStreamSuite/PubSub/Client/MetaData/SubMetaData.cs

@@ -1,8 +1,10 @@
 using System;
 
-namespace BCG {
-   //[Serializable]
-   public class SubMetaData {
+namespace BCG
+{
+   [Serializable]
+   public class SubMetaData 
+   {
       /// <summary>
       /// connection URL on which the connection will receive data
       /// </summary>
@@ -11,18 +13,20 @@ namespace BCG {
       /// identification of the client and the connection on which to receive data
       /// machine; ip; port; clientURL
       /// </summary>
-      public readonly string subscribing_app = ClientInfo.thisClient.processString;
+      public string subscribingApp;
       /// <summary>
       /// tags that are to be subscribed
       /// </summary>
       public string[] tags;
 
-      public SubMetaData() { }
+      public SubMetaData()
+      { }
 
-      public SubMetaData(string the_connection, params string[] the_tags) {
-         connection = the_connection;
-         tags = the_tags;
-         //subscribing_app = ClientInfo.thisClient.processString;
+      public SubMetaData(string theConnection, params string[] theTags)
+      {
+         connection = theConnection;
+         tags = theTags;
+         subscribingApp = ClientInfo.thisClient.processString;
       }
    }
 }

src/DataStreamSuite/PubSub/Client/MetaData/UnSubMetaData.cs

@@ -1,17 +1,21 @@
 using System;
 
-namespace BCG {
-   //[Serializable]
-   public class UnSubMetaData {
+namespace BCG
+{
+   [Serializable]
+   public class UnSubMetaData
+   {
       public string connection;
-      public readonly string[] tags;
-      public readonly string unsubscribing_app = ClientInfo.thisClient.processString;
+      public string[] tags;
+      public string unsubscribingApp = ClientInfo.thisClient.processString;
 
-      public UnSubMetaData() { }
+      public UnSubMetaData()
+      {}
 
-      public UnSubMetaData(string the_connection, params string[] the_tags) {
-         connection = the_connection;
-         tags = the_tags;
+      public UnSubMetaData(string theConnection, params string[] theTags)
+      {
+         connection = theConnection;
+         tags = theTags;
       }
    }
 }

src/DataStreamSuite/PubSub/Client/PublishingUtilities/IPublisher.cs

@@ -6,18 +6,12 @@ namespace BCG
 	public interface IPublisher
 	{
       void publish( IPubItem pi );
-#pragma warning disable SYSLIB0011 // Type or member is obsolete
       void publish( IPubItem pi, IFormatter f );
-#pragma warning restore SYSLIB0011 // Type or member is obsolete
       void publish( IPubItem[] pi );
-#pragma warning disable SYSLIB0011 // Type or member is obsolete
       void publish( IPubItem[] pi, IFormatter f );
-#pragma warning restore SYSLIB0011 // Type or member is obsolete
    }
 
-#pragma warning disable SYSLIB0011 // Type or member is obsolete
    public interface IPublisher<T> where T : IFormatter
-#pragma warning restore SYSLIB0011 // Type or member is obsolete
    {
       void publish (IPubItem pi);
       void publish (IPubItem[] pi);

src/DataStreamSuite/PubSub/Client/PublishingUtilities/Publisher.cs

@@ -8,9 +8,7 @@ namespace BCG
 	public class Publisher : IPublisher
 	{
       private PublisherCore myPubCore = new PublisherCore ();
-#pragma warning disable SYSLIB0011 // Type or member is obsolete
       private BinaryFormatter myBinFormatter = new BinaryFormatter ();
-#pragma warning restore SYSLIB0011 // Type or member is obsolete
       private const string myTagType = "DefaultTag";
 
       public Publisher()   {}
@@ -23,9 +21,7 @@ namespace BCG
       public void publish (IPubItem[] pis)
       { publish( pis, myBinFormatter ); }
 
-#pragma warning disable SYSLIB0011 // Type or member is obsolete
       public void publish( IPubItem pi, IFormatter f )
-#pragma warning restore SYSLIB0011 // Type or member is obsolete
       {
          MemoryStream ms = new MemoryStream();
          #pragma warning disable 
@@ -35,9 +31,7 @@ namespace BCG
          myPubCore.publish( myTagType, pi.getTag(), ms );
       }
 
-#pragma warning disable SYSLIB0011 // Type or member is obsolete
       public void publish( IPubItem[] pis, IFormatter f )
-#pragma warning restore SYSLIB0011 // Type or member is obsolete
       {
          myPubCore.beginTransaction();
          foreach ( IPubItem pi in pis )
@@ -48,14 +42,10 @@ namespace BCG
       #endregion
    }
 
-#pragma warning disable SYSLIB0011 // Type or member is obsolete
    public class Publisher<T> : IPublisher<T> where T : IFormatter, new ()
-#pragma warning restore SYSLIB0012 // Type or member is obsolete
    {
       private PublisherCore myPubCore = new PublisherCore ();
-#pragma warning disable SYSLIB0011 // Type or member is obsolete
       private IFormatter myFormatter = new T ();
-#pragma warning restore SYSLIB0011 // Type or member is obsolete
       private const string myTagType = "DefaultTag";
 
       public Publisher () { }
@@ -65,9 +55,9 @@ namespace BCG
       public void publish (IPubItem pi)
       {
          MemoryStream ms = new MemoryStream ();
-#pragma warning disable 
+         #pragma warning disable 
          myFormatter.Serialize (ms, pi);
-#pragma warning restore SYSLIB0011
+         #pragma warning restore SYSLIB0011
          ms.Position = 0;
          myPubCore.publish (myTagType, pi.getTag (), ms);
       }

src/DataStreamSuite/PubSub/ManagedTCP/AsyncTCPConnection_APM.cs

@@ -328,8 +328,7 @@ namespace bts {
             return;
          }
          _send_key_map.Add(message.key);
-         bts.Log.Gen.info($"AsyncTCPConnection_APM._on_send: pre endSend this: {ToString()} tid: {_tid()} " +
-                                 $"res: {_cb_txt(res)} message: {res.AsyncState} _is_alive: {_is_alive}");
+         bts.Log.Gen.info($"AsyncTCPConnection_APM._on_send: pre endSend this: {ToString()} tid: {_tid()} res: {_cb_txt(res)} message: {res.AsyncState} _is_alive: {_is_alive}");
          if (_is_alive) {
             try {
                SocketError send_err;

src/DataStreamSuite/PubSub/ManagedTCP/ClientToServerManagedTCPConnector.cs

@@ -1,82 +1,82 @@
-// using System;
-// using System.Collections.Generic;
-// using System.Text;
-// using System.Net;
-// using System.Net.Sockets;
-// using System.Threading;
-// using System.Threading.Tasks;
-// //using BCG.Utilities;
-// //using BCG.Utilities.General;
+using System;
+using System.Collections.Generic;
+using System.Text;
+using System.Net;
+using System.Net.Sockets;
+using System.Threading;
+using System.Threading.Tasks;
+//using BCG.Utilities;
+//using BCG.Utilities.General;
 
-// namespace BCG {
-//    internal class SocketConnectionArgs {
-//       public EndPoint ep;
-//       public Socket sock;
-//       public ITCPConnectionHandler handler;
-//       public string serverURL;
+namespace BCG {
+   internal class SocketConnectionArgs {
+      public EndPoint ep;
+      public Socket sock;
+      public ITCPConnectionHandler handler;
+      public string serverURL;
 
-//       internal SocketConnectionArgs(EndPoint theEp, Socket theSock, ITCPConnectionHandler theHandler,
-//          string theServerURL) {
-//          ep = theEp;
-//          sock = theSock;
-//          handler = theHandler;
-//          serverURL = theServerURL;
-//       }
-//    }
+      internal SocketConnectionArgs(EndPoint theEp, Socket theSock, ITCPConnectionHandler theHandler,
+         string theServerURL) {
+         ep = theEp;
+         sock = theSock;
+         handler = theHandler;
+         serverURL = theServerURL;
+      }
+   }
 
-//    public class ClientToServerManagedTCPConnector {
-//       private static Random myRnd = new Random(DateTime.Now.Millisecond);
-//       private static TimeSpan myHBTimeSpan = new TimeSpan(0, 0, 0, 0, PubSub3Config.instance.PS3_HeartBeatInterval);
-//       private static int myWaitTimeBeforeRetryingUpperLimit = 5000,
-//                            myWaitTimeBeforeRetryingLowerLimit = 1000;
+   public class ClientToServerManagedTCPConnector {
+      private static Random myRnd = new Random(DateTime.Now.Millisecond);
+      private static TimeSpan myHBTimeSpan = new TimeSpan(0, 0, 0, 0, PubSub3Config.instance.PS3_HeartBeatInterval);
+      private static int myWaitTimeBeforeRetryingUpperLimit = 5000,
+                           myWaitTimeBeforeRetryingLowerLimit = 1000;
 
-//       public static void connectTo(BlutoSubAddress serverAddress, ITCPConnectionHandler handler) {
-//          Console.WriteLine($"ClientToServerManagedTCPConnector.connectTo server: {serverAddress}");
-//          Socket sock = new Socket(AddressFamily.InterNetwork, SocketType.Stream, ProtocolType.Tcp);
-//          EndPoint ep = new IPEndPoint(IPAddress.Parse(serverAddress.ip), serverAddress.port);
-//          sock.BeginConnect(ep, new AsyncCallback(myOnConnect), new SocketConnectionArgs(ep, sock, handler, serverAddress.ToString()));
-//       }
+      public static void connectTo(BlutoSubAddress serverAddress, ITCPConnectionHandler handler) {
+         Console.WriteLine($"ClientToServerManagedTCPConnector.connectTo server: {serverAddress}");
+         Socket sock = new Socket(AddressFamily.InterNetwork, SocketType.Stream, ProtocolType.Tcp);
+         EndPoint ep = new IPEndPoint(IPAddress.Parse(serverAddress.ip), serverAddress.port);
+         sock.BeginConnect(ep, new AsyncCallback(myOnConnect), new SocketConnectionArgs(ep, sock, handler, serverAddress.ToString()));
+      }
 
-//       public static async Task connect_to_async(BlutoSubAddress serverAddress, ITCPConnectionHandler handler, TimeSpan wait_before_retrying, int max_n_attempts = 1024) {
-//          bts.Log.Gen.info($"ClientToServerManagedTCPConnector.connectTo server: {serverAddress}");
-//          await TCPSocketUtils.connect_to_server_async(serverAddress.ip, serverAddress.port, handler, wait_before_retrying, max_n_attempts);
-//       }
+      public static async Task connect_to_async(BlutoSubAddress serverAddress, ITCPConnectionHandler handler, TimeSpan wait_before_retrying, int max_n_attempts = 1024) {
+         bts.Log.Gen.info($"ClientToServerManagedTCPConnector.connectTo server: {serverAddress}");
+         await TCPSocketUtils.connect_to_server_async(serverAddress.ip, serverAddress.port, handler, wait_before_retrying, max_n_attempts);
+      }
 
-//       public static void connect_to(BlutoSubAddress serverAddress, ITCPConnectionHandler handler, TimeSpan wait_before_retrying, int max_n_attempts = 1024) {
-//          bts.Log.Gen.info($"ClientToServerManagedTCPConnector.connectTo server: {serverAddress}");
-//          Task.Run(async () => {
-//             await TCPSocketUtils.connect_to_server_async(serverAddress.ip, serverAddress.port, handler, wait_before_retrying, max_n_attempts);
-//          });
-//       }
+      public static void connect_to(BlutoSubAddress serverAddress, ITCPConnectionHandler handler, TimeSpan wait_before_retrying, int max_n_attempts = 1024) {
+         bts.Log.Gen.info($"ClientToServerManagedTCPConnector.connectTo server: {serverAddress}");
+         Task.Run(async () => {
+            await TCPSocketUtils.connect_to_server_async(serverAddress.ip, serverAddress.port, handler, wait_before_retrying, max_n_attempts);
+         });
+      }
 
-//       private static void myOnConnect(IAsyncResult res) {
-//          Console.WriteLine("ClientToServerManagedTCPConnector.myOnConnect " + res.ToString() + " " + res.AsyncState.ToString());
-//          SocketConnectionArgs args = res.AsyncState as SocketConnectionArgs;
-//          try {
-//             args.sock.EndConnect(res);
-//          } catch (SocketException sex) {
-//             string errMsg = String.Format("{0}: Connection to {1} failed.", DateTime.Now.ToString(), args.serverURL.ToString());
-//             //BlueEventLog.logEvent(sex.Message + ";" + errMsg, PS3LogLevels.disconnection);
-//             //Console.WriteLine(errMsg);
-//             if (args.handler.isDisposed) {
-//                string doneMsg = String.Format("{0}: Connection to {1} was disposed. Not Retrying.", DateTime.Now.ToString(), args.serverURL.ToString());
-//                //Console.WriteLine(doneMsg);
-//                //               BlueEventLog.logEvent(doneMsg, PS3LogLevels.disconnection);
-//                return;
-//             } else {
-//                string retryMsg = String.Format("{0}: Retrying Connection to {1}.", DateTime.Now.ToString(), args.serverURL.ToString());
-//                //Console.WriteLine(retryMsg);
-//                //BlueEventLog.logEvent(retryMsg, PS3LogLevels.disconnection);
+      private static void myOnConnect(IAsyncResult res) {
+         Console.WriteLine("ClientToServerManagedTCPConnector.myOnConnect " + res.ToString() + " " + res.AsyncState.ToString());
+         SocketConnectionArgs args = res.AsyncState as SocketConnectionArgs;
+         try {
+            args.sock.EndConnect(res);
+         } catch (SocketException sex) {
+            string errMsg = String.Format("{0}: Connection to {1} failed.", DateTime.Now.ToString(), args.serverURL.ToString());
+            //BlueEventLog.logEvent(sex.Message + ";" + errMsg, PS3LogLevels.disconnection);
+            //Console.WriteLine(errMsg);
+            if (args.handler.isDisposed) {
+               string doneMsg = String.Format("{0}: Connection to {1} was disposed. Not Retrying.", DateTime.Now.ToString(), args.serverURL.ToString());
+               //Console.WriteLine(doneMsg);
+               //               BlueEventLog.logEvent(doneMsg, PS3LogLevels.disconnection);
+               return;
+            } else {
+               string retryMsg = String.Format("{0}: Retrying Connection to {1}.", DateTime.Now.ToString(), args.serverURL.ToString());
+               //Console.WriteLine(retryMsg);
+               //BlueEventLog.logEvent(retryMsg, PS3LogLevels.disconnection);
 
-//                int waitTime = myRnd.Next(myWaitTimeBeforeRetryingLowerLimit, myWaitTimeBeforeRetryingUpperLimit);
-//                Thread.Sleep(waitTime);
-//                args.sock.BeginConnect(args.ep, new AsyncCallback(myOnConnect), args);
-//                return;
-//             }
-//          }
-//          //BlueEventLog.logEvent(String.Format("Successful connection made to {0}.", args.serverURL.ToString()), PS3LogLevels.connection);
-//          args.handler.onConnection(new bts.AsyncTCPConnection_APM(args.sock, myHBTimeSpan, ITCPConnection.ConnectionSide.Client));
-//       }
+               int waitTime = myRnd.Next(myWaitTimeBeforeRetryingLowerLimit, myWaitTimeBeforeRetryingUpperLimit);
+               Thread.Sleep(waitTime);
+               args.sock.BeginConnect(args.ep, new AsyncCallback(myOnConnect), args);
+               return;
+            }
+         }
+         //BlueEventLog.logEvent(String.Format("Successful connection made to {0}.", args.serverURL.ToString()), PS3LogLevels.connection);
+         args.handler.onConnection(new bts.AsyncTCPConnection_APM(args.sock, myHBTimeSpan, ITCPConnection.ConnectionSide.Client));
+      }
 
-//    }
-// }
+   }
+}

src/DataStreamSuite/PubSub/Misc/GenericFormatter.cs

@@ -5,21 +5,23 @@ using System.Runtime.Serialization.Formatters.Binary;
 
 namespace BCG
 {
-#pragma warning disable SYSLIB0011 // Type or member is obsolete
    public class GenericFormatter<F> : IGenericFormatter where F : IFormatter, new ()
    {
       IFormatter myFormatter = new F ();
 
       public T Deserialize<T> (Stream serializationStream)
       {
+         #pragma warning disable 
          return (T) myFormatter.Deserialize (serializationStream);
+         #pragma warning restore SYSLIB0011
       }
       public void Serialize<T> (Stream serializationStream, T graph)
       {
+         #pragma warning disable 
          myFormatter.Serialize (serializationStream, graph);
+         #pragma warning restore SYSLIB0011
       }
    }
 
    public class GenericBinaryFormatter : GenericFormatter<BinaryFormatter> { }
-#pragma warning restore SYSLIB0011 // Type or member is obsolete
 }

src/DataStreamSuite/PubSub/PubSub.csproj

@@ -6,7 +6,7 @@
 
   <PropertyGroup>
       <AllowUnsafeBlocks>true</AllowUnsafeBlocks>
-    <TargetFramework>net8.0</TargetFramework>
+    <TargetFramework>net7.0</TargetFramework>
     <Nullable>enable</Nullable>
   </PropertyGroup>
 

src/DataStreamSuite/PubSub/PubSubUpdate.cs

@@ -4,22 +4,19 @@ using System.IO;
 
 namespace BCG {
    /// <summary>
-   /// this is the data returned from the pubsub server
+   /// this is the data returned from the
+   /// pubsub server
    /// </summary>
    public class PubSubUpdate {
 
-      public PubSubUpdate(string tag, string the_tag_type, Stream body) {
-         this.tag = tag;
-         this.body = body;
-         this.tag_type = the_tag_type;
-      }
+      public PubSubUpdate(string tag, string tagType, Stream body) { this.tag = tag; this.body = body; this.tagType = tagType; }
 
       public string tag;
       public Stream body;
-      public string tag_type;
+      public string tagType;
 
       public override string ToString() {
-         return $"PubSubUpdate tag: {tag} tag_type: {tag_type} len: {body.Position}";
+         return $"PubSubUpdate tag: {tag} tag_type: {tagType} len: {body.Position}";
       }
    };
 }

src/DataStreamSuite/PubSub/Server/Communication/IPubsubMessageHubBase.cs

@@ -5,16 +5,18 @@ using System.IO;
 using System.Collections.Generic;
 using mbts;
 
-namespace BCG {
-   public delegate void PubSubServerConnectionStatusChange(string peerURL, bool connection);
-   public delegate void PubSubServerDataDelegate(string sender, byte[] data);
+namespace BCG
+{
+   public delegate void ConnectionStatusChange(string peerURL, bool connection);
+   public delegate void DataDelegate(string sender, byte[] data);
    /// <summary>
    /// Encapsulates PubsubServer's communication with the
    /// rest of the world. Has send and get methods, and also
    /// an event to tell if a client connects or disconnects.
    /// this event is prompted by the doWork.
    /// </summary>
-   public interface IPubsubMessageHub : IWorker {
+   public interface IPubsubMessageHub : IWorker
+   {
 
       /// <summary>
       /// The port that the message hub is listening on.  Return -1 if you're not listening on a port, i.e. for hte case of multicast
@@ -27,14 +29,14 @@ namespace BCG {
       /// getUpdates. The delegate will be fired on an arbitrary thread.
       /// </summary>
       /// <param name="?"></param>
-      void registerDataCallback(PubSubServerDataDelegate del);
+      void registerDataCallback(DataDelegate del);
       /// <summary>
       /// registers a connection callback. this callback
       /// will be fired within the doWork call.
       /// </summary>
       /// <param name="del"></param>
       /// <param name="register"></param>
-      void registerConnectionCallback(PubSubServerConnectionStatusChange del);
+      void registerConnectionCallback(ConnectionStatusChange del);
       /// <summary>
       /// this is where this interface starts communication with the
       /// rest of the world. Obvioulsy, the resgiters should be

src/DataStreamSuite/PubSub/Server/Communication/ManagedTCP/ManagedTCPMessageHub.cs

@@ -22,8 +22,8 @@ namespace BCG {
 
       }
       private bool myActivated = false;
-      private PubSubServerConnectionStatusChange myConnCb;
-      private PubSubServerDataDelegate myDataCb;
+      private ConnectionStatusChange myConnCb;
+      private DataDelegate myDataCb;
 
       /// <summary>
       /// client to connection
@@ -77,12 +77,12 @@ namespace BCG {
 
       #region IPubsubMessageHub Members
 
-      public void registerDataCallback(PubSubServerDataDelegate del) {
+      public void registerDataCallback(DataDelegate del) {
          // BlueEventLog.logAssert(!myActivated, "connection callbacks should be registered before the call to activate");
          myDataCb = del;
       }
 
-      public void registerConnectionCallback(PubSubServerConnectionStatusChange del) {
+      public void registerConnectionCallback(ConnectionStatusChange del) {
          //BlueEventLog.logAssert(!myActivated, "data callbacks should be registered before the call to activate");
          myConnCb = del;
       }

src/DataStreamSuite/PubSub/Server/Communication/ServerMessageHub.cs

@@ -4,22 +4,25 @@ using System.Threading;
 using System.IO;
 using mbts;
 
-namespace BCG {
+namespace BCG
+{
    /// <summary>
    /// Wraps around all the IPubsubMessageHub's that pubsub supports.
    /// ie MSMQ and TCP to begin with.
    /// </summary>
-   public class ServerMessageHub : IPubsubMessageHub {
+   public class ServerMessageHub : IPubsubMessageHub
+   {
       private static ServerMessageHub mySelf = null;
       private static object myInstanceLock = new object();
       private Dictionary<string, IPubsubMessageHub> myHubs = new Dictionary<string, IPubsubMessageHub>();
       private SyncQueue<Pair<string, byte[]>> myReceivingQueue = new SyncQueue<Pair<string, byte[]>>();
-      private PubSubServerConnectionStatusChange myConnectionCb;
-      private PubSubServerDataDelegate myDataCb;
+      private ConnectionStatusChange myConnectionCb;
+      private DataDelegate myDataCb;
       private MasterWorker myMasterWorker = MasterWorker.getInstance(false);
 
       protected ServerMessageHub() : this(PubSub3Config.instance.PS3_ServerPort) { }
-      protected ServerMessageHub(int portNumber) {
+      protected ServerMessageHub(int portNumber)
+      {
          // TODO: no more MSMQ
          // if (!PubSub3Config.instance.PS3_DisableMSMQ)
          //    myHubs.Add(MSMQSubAddress.prefix, new MSMQMessageHub());
@@ -32,14 +35,19 @@ namespace BCG {
                                                       as IPubsubMessageHub);
       }
 
-      public int port {
+      public int port
+      {
          get { return myHubs[BlutoSubAddress.prefix].port; }
 
       }
-      public static ServerMessageHub instance {
-         get {
-            if (mySelf == null) {
-               lock (myInstanceLock) {
+      public static ServerMessageHub instance
+      {
+         get
+         {
+            if (mySelf == null)
+            {
+               lock (myInstanceLock)
+               {
                   if (mySelf == null)
                      mySelf = new ServerMessageHub();
                }
@@ -52,39 +60,47 @@ namespace BCG {
 
       /* Builds the server message hub with a specified port number. Only works if the server message hub has not already been created. 
        * */
-      public static ServerMessageHub getInstance(int portNumber) {
-         if (mySelf == null) {
-            lock (myInstanceLock) {
+      public static ServerMessageHub getInstance(int portNumber)
+      {
+         if (mySelf == null)
+         {
+            lock (myInstanceLock)
+            {
                if (mySelf == null)
                   mySelf = new ServerMessageHub(portNumber);
             }
 
-         } else
+         }
+         else
             throw new ArgumentException("Error: Cannot specify a port number for the ServerMessageHub instance if it has already been created.");
          return mySelf;
       }
       #region IPubsubMessageHub Members
 
-      public void registerDataCallback(PubSubServerDataDelegate del) { myDataCb = del; }
+      public void registerDataCallback(DataDelegate del) { myDataCb = del; }
 
-      public void registerConnectionCallback(PubSubServerConnectionStatusChange del) { myConnectionCb = del; }
+      public void registerConnectionCallback(ConnectionStatusChange del) { myConnectionCb = del; }
 
-      public void activate() {
+      public void activate()
+      {
          string msg = "PubSubServer started .. initializing TCP transport components .. ";
          Console.WriteLine(msg);
-         foreach (IPubsubMessageHub hub in myHubs.Values) {
-            hub.registerDataCallback(new PubSubServerDataDelegate(myOnData));
-            hub.registerConnectionCallback(new PubSubServerConnectionStatusChange(myOnConnection));
+         foreach (IPubsubMessageHub hub in myHubs.Values)
+         {
+            hub.registerDataCallback(new DataDelegate(myOnData));
+            hub.registerConnectionCallback(new ConnectionStatusChange(myOnConnection));
             hub.activate();
          }
          myMasterWorker.addWorker(this, new TimeSpan(0, 0, 0, 0, 250));
       }
 
-      public List<Pair<string, byte[]>> getUpdates(TimeSpan timeout, int maxElements) {
+      public List<Pair<string, byte[]>> getUpdates(TimeSpan timeout, int maxElements)
+      {
          return myReceivingQueue.dequeueMultipleElements(timeout, maxElements);
       }
 
-      public bool send(string receiver, byte[] data) {
+      public bool send(string receiver, byte[] data)
+      {
          if (receiver.StartsWith(BlutoSubAddress.prefix))
             return myHubs[BlutoSubAddress.prefix].send(receiver, data);
          else if (receiver.StartsWith(MSMQSubAddress.prefix))
@@ -93,7 +109,8 @@ namespace BCG {
             return false;
       }
 
-      public bool send(string receiver, Stream str) {
+      public bool send(string receiver, Stream str)
+      {
          if (receiver.StartsWith(BlutoSubAddress.prefix))
             return myHubs[BlutoSubAddress.prefix].send(receiver, str);
          else if (receiver.StartsWith(MSMQSubAddress.prefix))
@@ -109,7 +126,8 @@ namespace BCG {
       /// <summary>
       /// connection callback will be called within this doWork
       /// </summary>
-      public void doWork() {
+      public void doWork()
+      {
          foreach (IPubsubMessageHub hub in myHubs.Values)
             hub.doWork();
       }
@@ -120,18 +138,21 @@ namespace BCG {
       public static string internalPublisherConnection = "InternalPublisher";
 
 
-      public void internalSend(byte[] data) {
+      public void internalSend(byte[] data)
+      {
          myReceivingQueue.enqueue(new Pair<string, byte[]>(internalPublisherConnection, data));
       }
 
-      private void myOnData(string sender, byte[] msg) {
+      private void myOnData(string sender, byte[] msg)
+      {
          if (myDataCb != null)
             myOnData(sender, msg);
          else
             myReceivingQueue.enqueue(new Pair<string, byte[]>(sender, msg));
       }
 
-      private void myOnConnection(string clientURL, bool connected) {
+      private void myOnConnection(string clientURL, bool connected)
+      {
          if (myConnectionCb != null)
             myConnectionCb(clientURL, connected);
       }

src/DataStreamSuite/PubSub/Server/PubSubMessageReactor.cs

@@ -58,7 +58,7 @@ namespace BCG {
 
          myReceivingProcThread = new Thread(new ThreadStart(myReceivingThreadProc));
          myMessageHub = ServerMessageHub.instance;
-         myMessageHub.registerConnectionCallback(new PubSubServerConnectionStatusChange(myHandleConnections));
+         myMessageHub.registerConnectionCallback(new ConnectionStatusChange(myHandleConnections));
 
 
       }

src/DataStreamSuite/PubSub/Server/PubSubState.cs

@@ -55,7 +55,7 @@ namespace BCG {
             myTagToPubsubMap[tag].onSub(sub);
 
             if (onSingleTagChange != null)
-               onSingleTagChange(new PubSubSingleTagStateEvent(tag, PubSubSingleTagStateEvent.EventType.subscription, sub.connection, sub.subscribing_app));
+               onSingleTagChange(new PubSubSingleTagStateEvent(tag, PubSubSingleTagStateEvent.EventType.subscription, sub.connection, sub.subscribingApp));
          }
          if (onTagMapChanged != null)
             onTagMapChanged(sub.tags);
@@ -68,10 +68,10 @@ namespace BCG {
                ///and publishers for the tag then remove it.
                myTagToPubsubMap[tag].onUnsub(unsub);
                if (onSingleTagChange != null)
-                  onSingleTagChange(new PubSubSingleTagStateEvent(tag, PubSubSingleTagStateEvent.EventType.unsubscription, unsub.connection, unsub.unsubscribing_app));
+                  onSingleTagChange(new PubSubSingleTagStateEvent(tag, PubSubSingleTagStateEvent.EventType.unsubscription, unsub.connection, unsub.unsubscribingApp));
             } else {
                string warningMsg = String.Format("{0} was never published or subscribed on this pubsub, hence" +
-                  " the unsub call by app {1} is voided", tag, unsub.unsubscribing_app);
+                  " the unsub call by app {1} is voided", tag, unsub.unsubscribingApp);
                //BlueEventLog.logEvent(warningMsg, BlueEventLevel.Warning);
             }
          }

src/DataStreamSuite/PubSub/Server/SingleTagPubSubMap.cs

@@ -1,19 +1,19 @@
 using System;
 using System.Collections.Generic;
 using System.Text;
-
 using BCG.Utilities;
-
 using mbts;
 
-namespace BCG {
+namespace BCG
+{
    /// <summary>
    /// SingleTagPubSubMap consists of all the state w.r.t to
    /// publishing and subscription of a single tag. this information
    /// consists of the publisher, and a map of the subscriber
    /// connection vs the apps they're coming from 
    /// </summary>
-   internal class SingleTagPubSubMap {
+   internal class SingleTagPubSubMap
+   {
       private string myTag;
       /// <summary>
       /// map of clientURL to client app (machine;app;pid;clientURL)
@@ -29,7 +29,8 @@ namespace BCG {
       /// </summary>
       List<Pair<string, string>> myDeadPubConnToApp = new List<Pair<string, string>>();
 
-      public SingleTagPubSubMap(string tag) {
+      public SingleTagPubSubMap(string tag)
+      {
          myTag = tag;
       }
 
@@ -37,7 +38,8 @@ namespace BCG {
       /// Returns a list all the active connections that 
       /// have subscribed to this tag
       /// </summary>
-      public ICollection<string> subscriberConnections {
+      public ICollection<string> subscriberConnections
+      {
          get { return mySubConnToApp.Keys; }
       }
 
@@ -45,7 +47,8 @@ namespace BCG {
       /// Returns a list of all the applications that have
       /// subscribed to this tag
       /// </summary>
-      public ICollection<string> subscripingApps {
+      public ICollection<string> subscripingApps
+      {
          get { return mySubConnToApp.Values; }
       }
 
@@ -54,7 +57,8 @@ namespace BCG {
       /// connections as key and the application
       /// as value
       /// </summary>
-      public Dictionary<string, string> subscribers {
+      public Dictionary<string, string> subscribers
+      {
          get { return mySubConnToApp; }
       }
 
@@ -62,7 +66,8 @@ namespace BCG {
       /// Returns a dictionary for all active publishing
       /// connection as key and the application as value
       /// </summary>
-      public Dictionary<string, string> activePublishers {
+      public Dictionary<string, string> activePublishers
+      {
          get { return myActivePubConnToApp; }
       }
 
@@ -70,7 +75,8 @@ namespace BCG {
       /// Returns a list of pair of dead connections and
       /// theur corresponding applications
       /// </summary>
-      public ICollection<Pair<string, string>> deadPublishers {
+      public ICollection<Pair<string, string>> deadPublishers
+      {
          get { return myDeadPubConnToApp; }
       }
 
@@ -81,14 +87,18 @@ namespace BCG {
       /// </summary>
       /// <param name="sub"></param>
       /// <returns></returns>
-      public bool onSub(SubMetaData sub) {
-         if (mySubConnToApp.ContainsKey(sub.connection)) {
+      public bool onSub(SubMetaData sub)
+      {
+         if (mySubConnToApp.ContainsKey(sub.connection))
+         {
             string warningMsg = String.Format("tag {0} has already been subscribed by app {0}, subscriber {1}"
-               + "it should unsubscribe before resubscribing", myTag, sub.connection, sub.subscribing_app);
+               + "it should unsubscribe before resubscribing", myTag, sub.connection, sub.subscribingApp);
             //BlueEventLog.logEvent(warningMsg, BlueEventLevel.Warning);
             return false;
-         } else {
-            mySubConnToApp.Add(sub.connection, sub.subscribing_app);
+         }
+         else
+         {
+            mySubConnToApp.Add(sub.connection, sub.subscribingApp);
             return true;
          }
       }
@@ -100,13 +110,17 @@ namespace BCG {
       /// </summary>
       /// <param name="unsub"></param>
       /// <returns></returns>
-      public bool onUnsub(UnSubMetaData unsub) {
-         if (mySubConnToApp.ContainsKey(unsub.connection)) {
+      public bool onUnsub(UnSubMetaData unsub)
+      {
+         if (mySubConnToApp.ContainsKey(unsub.connection))
+         {
             mySubConnToApp.Remove(unsub.connection);
             return true;
-         } else {
+         }
+         else
+         {
             string warningMsg = String.Format("tag {0} has already was either unsubscribed or never subscribed by app {0}, subscriber {1}",
-               myTag, unsub.connection, unsub.unsubscribing_app);
+               myTag, unsub.connection, unsub.unsubscribingApp);
             //BlueEventLog.logEvent(warningMsg, BlueEventLevel.Warning);
             return false;
          }
@@ -119,20 +133,25 @@ namespace BCG {
       /// </summary>
       /// <param name="subscriber"></param>
       /// <returns></returns>
-      public bool onConnectionRemoveIfExists(string connection, out bool wasPublisher, out bool wasSubscriber) {
+      public bool onConnectionRemoveIfExists(string connection, out bool wasPublisher, out bool wasSubscriber)
+      {
          bool rv = false;
-         if (mySubConnToApp.ContainsKey(connection)) {
+         if (mySubConnToApp.ContainsKey(connection))
+         {
             mySubConnToApp.Remove(connection);
             wasSubscriber = true;
             rv = true;
-         } else
+         }
+         else
             wasSubscriber = false;
-         if (myActivePubConnToApp.ContainsKey(connection)) {
+         if (myActivePubConnToApp.ContainsKey(connection))
+         {
             myDeadPubConnToApp.Add(new Pair<string, string>(connection, myActivePubConnToApp[connection]));
             myActivePubConnToApp.Remove(connection);
             rv = true;
             wasPublisher = true;
-         } else
+         }
+         else
             wasPublisher = false;
          return rv;
       }
@@ -143,10 +162,14 @@ namespace BCG {
       /// </summary>
       /// <param name="pub"></param>
       /// <param name="publisherChanged"></param>
-      public void onPub(PubMetaData pub, out bool newPublisher) {
-         if (myActivePubConnToApp.ContainsKey(pub.connection)) {
+      public void onPub(PubMetaData pub, out bool newPublisher)
+      {
+         if (myActivePubConnToApp.ContainsKey(pub.connection))
+         {
             newPublisher = false;
-         } else {
+         }
+         else
+         {
             myActivePubConnToApp.Add(pub.connection, pub.publishingApp);
             newPublisher = true;
          }

src/DataStreamSuite/PubSub/SingleServerSubscriberCore.cs

@@ -6,79 +6,92 @@ using BCG.Utilities;
 using BCG.Utilities.General;
 using mbts;
 using mbts.spark;
-using System.Linq;
 
 namespace BCG {
    /// <summary>
-   /// Deals with the subscriptions (and getting data) from one sole
-   /// pubsub server. Theres no sharing of conenctions
+   /// Deals with the subscriptions (and getting data) from one
+   /// pubsub server. Once again, theres no sharing of conenctions
    /// from this point. Every new instance of SingleServerSubscriberCore
    /// will open its new dedicated connection to the serverURL that
    /// its concerned with (passed as a conctructor arg).
    /// </summary>
 
    internal interface ISingleServerSubscriberCore : IDisposable {
-      IList<PubSubUpdate> get_updates(int timeout);
+      IList<PubSubUpdate> getUpdates(int timeout);
       event PubSubConnectionChange onPubSubConnectionChange;
       void subscribe(params string[] tags);
       void unsubscribe(params string[] tags);
    }
-
    internal class SingleServerSubscriberCore : IDisposable, ISingleServerSubscriberCore {
-      private readonly IPubSubServerConnection _connection;
-      private HashSet<string> _tags = new HashSet<string>();
+      private HighPerformanceTimer myTimer = new HighPerformanceTimer();
+      private PubSubServerConnectionFactory myConnectionFactory;
+      private IPubSubServerConnection myConnection;
+      private List<string> myTags = new List<string>();
 
       public event PubSubConnectionChange onPubSubConnectionChange;
 
-      public SingleServerSubscriberCore(string server_url, PubSubServerConnectionFactory connection_factory) {
-         _connection = connection_factory.getConnection(server_url);
-         _connection.register_connection_callback(new PubSubServerConnectionStatusChange(_handle_server_reboot));
+      public SingleServerSubscriberCore(string serverURL, PubSubServerConnectionFactory connectionFactory) {
+         myConnectionFactory = connectionFactory;
+         myConnection = myConnectionFactory.getConnection(serverURL);
+         myConnection.registerCallback(new ConnectionStatusChange(myHandleServerReboot));
       }
 
-      // gets as many data items that it can get in
-      // timeout milliseconds
-      public IList<PubSubUpdate> get_updates(int timeout) {
-         _connection.doWork();
-         List<PubSubUpdate> rv = new List<PubSubUpdate>();
-         IList<byte[]> data_items = _connection.get_updates(new TimeSpan(0, 0, 0, 0, timeout), Int32.MaxValue);
-         foreach (byte[] data in data_items) {
-            process_message(data, ref rv);
-         }
-         return rv;
+      public IList<PubSubUpdate> getUpdates() {
+         return getUpdates(1);
+      }
 
+      /// <summary>
+      /// gets as many data items that it can get in
+      /// timeout milliseconds
+      /// </summary>
+      /// <param name="timeout"></param>
+      /// <returns></returns>
+      public IList<PubSubUpdate> getUpdates(int timeout) {
+         myConnection.doWork();
+         List<PubSubUpdate> retVal = new List<PubSubUpdate>();
+         IList<byte[]> dataItems = myConnection.getUpdates(new TimeSpan(0, 0, 0, 0, timeout), Int32.MaxValue);
+         foreach (byte[] data in dataItems) {
+            processMessage(data, retVal);
+         }
+         return retVal;
+
+      }
+
+      public List<string> getSubscribedTags() {
+         return myTags;
       }
 
       public void subscribe(params string[] tags) {
-         HashSet<string> this_call_tags = new HashSet<string>();
          foreach (string tag in tags) {
-            if (_tags.Contains(tag)) {
-               bts.Log.Gen.warn($"SingleServerSubscriberCore.subscribe tag: {tag} has already " +
-                                 $"been subscribed to server: {_connection.server_url}");
+            if (myTags.Contains(tag)) {
+               string errMsg = String.Format("tag {0} was already subscribed from server {1}",
+                  tag, myConnection.serverURL);
+               // BlueEventLog.logEvent(errMsg, BlueEventLevel.Warning);
             } else {
-               _tags.Add(tag);
-               this_call_tags.Add(tag);
+               myTags.Add(tag);
             }
          }
-         _subscribe(this_call_tags.ToArray());
+         mySubscribe(tags);
       }
 
       public void unsubscribe(params string[] tags) {
-         List<string> unsub_tags = new List<string>();
+         List<string> unsubTags = new List<string>();
          foreach (string tag in tags) {
-            if (_tags.Contains(tag)) {
-               _tags.Remove(tag);
-               unsub_tags.Add(tag);
+            if (myTags.Contains(tag)) {
+               myTags.Remove(tag);
+               unsubTags.Add(tag);
             } else {
-               bts.Log.Gen.warn($"tag {tag} is being unsubscribed from server {_connection.server_url}, eventhough " +
-                                 "it was never subscribed");
+               string errMsg = String.Format("tag {0} is being unsubscribed from server {1}, eventhough " +
+                  "it was never subscribed", tag, myConnection.serverURL);
+               // BlueEventLog.logEvent(errMsg, BlueEventLevel.Warning);
             }
          }
-         UnSubMetaData unsub = new UnSubMetaData(_connection.client_url, unsub_tags.ToArray());
+         UnSubMetaData unsub = new UnSubMetaData(myConnection.clientURL, unsubTags.ToArray());
          Stream str = new MemoryStream();
          new MetaData(unsub).save(str);
          str.Position = 0;
          // this is where data is actually sent out
-         _connection.send(str);
+         myConnection.send(str);
       }
 
       #region IDisposable Members
@@ -86,40 +99,44 @@ namespace BCG {
       public void Dispose() {
          ///remove the subscriber
          //myConnection.send(new PubSubMessage(new MetaData(new RemoveSubscriberMetaData(myConnection.clientURL))).toStream());
-         _connection.Dispose();
+         myConnection.Dispose();
 
          //clear the list of tags you've subscribed to. This way, if you're ever used again, 
          //you have to send all the subscription metadata out again.
-         _tags.Clear();
+         myTags.Clear();
       }
 
       #endregion
 
-      // The only place where a tag is subsribed from the Pubsub server. the subscriptions are wrapped into SubMetaData,
-      // which has a to_stream method
-      private void _subscribe(params string[] tags) {
-         SubMetaData sub = new SubMetaData(_connection.client_url, tags);
+      private void mySubscribe(params string[] tags) {
+         SubMetaData sub = new SubMetaData(myConnection.clientURL, tags);
          Stream str = new MemoryStream();
          new MetaData(sub).save(str);
          str.Position = 0;
-         _connection.send(str);
+         myConnection.send(str);
       }
 
-      private void _handle_server_reboot(string server_url, bool connection) {
-         bts.Log.Gen.info($"SingleServerSubscriberCore._handle_server_reboot connection: {connection} server_url: {server_url}");
+      private void myHandleServerReboot(string serverURL, bool connection) {
          if (connection) {
-            _subscribe(_tags.ToArray());
+            //ie connection established again, so has to
+            //resubscribe to all the tags
+            // BlueEventLog.logEvent("Connection to " + serverURL + " established....resubscribing to all the tags",
+            //    PS3LogLevels.disconnection);
+            mySubscribe(myTags.ToArray());
+
+
          } else {
-            bts.Log.Gen.warn($"SingleServerSubscriberCore._handle_server_reboot connection to {server_url}  broke....attempting to reconnect...");
+            // BlueEventLog.logEvent("Connection to " + serverURL + " broke....attempting to reconnect...",
+            //    PS3LogLevels.disconnection);
          }
+
          //let anyone watching you know that your connection has changed. 
          if (onPubSubConnectionChange != null)
-            onPubSubConnectionChange(connection, server_url);
+            onPubSubConnectionChange(connection, serverURL);
       }
 
-      // processes one message
-      internal static void process_message(byte[] data, ref List<PubSubUpdate> updates) {
-         bts.Log.Gen.info($"SingleServerSubscriberCore.process_message data_len: {data.Length}");
+      internal static void processMessage(byte[] data, List<PubSubUpdate> updates) {
+         bts.Log.Gen.info($"SingleServerSubscriberCore.processMessage data_len: {data.Length}");
          if (data != null && data.Length > 0) {
             MemoryStream str = new MemoryStream(data);
             MetaData md = new MetaData(str);
@@ -139,8 +156,9 @@ namespace BCG {
                if (p.isCompressed) {
                   long compressedSize = bodyStream.Length - bodyStream.Position;
                   if (compressedSize == 0) {
-                     string error_msg = "Error: tag " + p.tag + " had compressed size of 0 bytes. This shouldnt' happen. Not decompressing tag or sending out update.";
-                     bts.Log.Gen.error(error_msg);
+                     string errorMsg = "Error: tag " + p.tag + " had compressed size of 0 bytes. This shouldnt' happen. Not decompressing tag or sending out update.";
+                     Console.WriteLine(errorMsg);
+                     //BlueEventLog.logEvent(errorMsg, BlueEventLevel.Error);
                      return;
                   }
                   realBody = StreamDecompressor.decompressStream(bodyStream);
@@ -156,9 +174,8 @@ namespace BCG {
                   bodyStream.Read(trConstituent, 0, trConstituent.Length);
                   updates.Add(new PubSubUpdate(pub.tag, pub.type, new MemoryStream(trConstituent)));
                }
-            } else {
-               bts.Log.assert_d(false, $"Unknown message received: {md} only Pub and TransactionMetaData are legal");
-            }
+            } else { }
+            //BlueEventLog.logAssert(false, "Unknown message received: " + md.ToString() + " , only Pub and TransactionMetaData are legal");
          }
       }
    }

src/DataStreamSuite/PubSub/StaticPublisherCore.cs

@@ -387,18 +387,18 @@ namespace BCG {
             SharpZipCompressor Kompressor = new SharpZipCompressor();
             Kompressor.compress(content, contentToPublish);
             contentToPublish.Position = 0;
-            pub_md = new PubMetaData(tagType, tag, (int)contentToPublish.Length, (int)content.Length, serverConn.server_url, ClientInfo.thisClient.processString);
+            pub_md = new PubMetaData(tagType, tag, (int)contentToPublish.Length, (int)content.Length, serverConn.serverURL, ClientInfo.thisClient.processString);
             if (PubSub3Config.instance.shouldReportTagsCompressionRatio(tag)) {
                string msg = String.Format("Compressed tag {0} from {1} to {2} bytes, for a ratio of {3}.", tag, content.Length, contentToPublish.Length, (float)content.Length / (float)contentToPublish.Length);
                // BlueEventLog.logEvent(msg, PS3LogLevels.compressionRatio);
             }
          } else
-            pub_md = new PubMetaData(tagType, tag, (int)content.Length, serverConn.server_url, ClientInfo.thisClient.processString);
+            pub_md = new PubMetaData(tagType, tag, (int)content.Length, serverConn.serverURL, ClientInfo.thisClient.processString);
 
          content.Position = initPos;
          if (CrushedTagManager.instance.is_tag_crushable(tag)) {
             string refId = String.Empty;
-            Console.WriteLine("StaticPublisherCore.mySend serverConn.sendWithConfirmation tag: " + tag + " server: " + serverConn.server_url);
+            Console.WriteLine("StaticPublisherCore.mySend serverConn.sendWithConfirmation tag: " + tag + " server: " + serverConn.serverURL);
             //  this is where the data is actually sent out (there are multiple sends)
             PubSubMessage ps_msg = new PubSubMessage(new MetaData(pub_md), contentToPublish);
             //Console.WriteLine($"serverConn.send_with_confirmation metadata: {ps_msg.meta_data}");
@@ -428,8 +428,8 @@ namespace BCG {
             myTagToSender.Add(tag, sender);
             if (!mySenders.Contains(sender)) {
                mySenders.Add(sender);
-               sender.register_connection_callback(new PubSubServerConnectionStatusChange(myHandleServerReboot));
-               sender.register_send_confirmation_callback(new SendConfirmationDelegate(myOnSendConfirm));
+               sender.registerCallback(new ConnectionStatusChange(myHandleServerReboot));
+               sender.registerSendConfirmationCallback(new SendConfirmationDelegate(myOnSendConfirm));
             }
 
          }
@@ -443,7 +443,7 @@ namespace BCG {
          if (connection) {
             string thisPublisherInfo = ClientInfo.thisClient.processString;
             foreach (KeyValuePair<string, IPubSubServerConnection> p in myTagToSender) {
-               if (p.Value.server_url == serverURL) {
+               if (p.Value.serverURL == serverURL) {
                   string tagType;
                   string tag = p.Key;
                   myCrushableTagsWaitingOnAck.Clear();

src/DataStreamSuite/PubSub/StaticSubscriberCore.cs

@@ -80,7 +80,7 @@ namespace BCG {
                int timeShare = timeout / myServerURLToSubCore.Count;
                List<PubSubUpdate> retVal = new List<PubSubUpdate>();
                foreach (ISingleServerSubscriberCore subCore in myServerURLToSubCore.Values)
-                  retVal.AddRange(subCore.get_updates(timeShare));
+                  retVal.AddRange(subCore.getUpdates(timeShare));
                _log_n_updates(retVal);
                myTimer.stop();
                //if (myTimer.lastIntervalTime > 10)

src/DataStreamSuite/PubSubServer/Program.cs

@@ -15,7 +15,7 @@ namespace BCG {
       /// </summary>
       [STAThread]
       public static void Main(string[] args) {
-         bool run_echo_tcp_server_only = false;
+         bool run_echo_tcp_server_only = true;
          bts.utils.BTSNativeMethods.provide_command_line(new string[] { }, 0);
          mbts.config.Configuration.initialize();
          if (run_echo_tcp_server_only) {

src/DataStreamSuite/PubSubServer/PubSubServer.csproj

@@ -8,7 +8,7 @@
   </ItemGroup>
   <PropertyGroup>
     <OutputType>Exe</OutputType>
-    <TargetFramework>net8.0</TargetFramework>
+    <TargetFramework>net7.0</TargetFramework>
     <Nullable>enable</Nullable>
   </PropertyGroup>
 

src/DataStreamSuite/SharedStreams/Business/MarketData/TimSeriesOperations/TimeSeriesGroupByStream.cs

@@ -1,20 +1,26 @@
-namespace BCG {
+namespace BCG
+{
    using bts.api.orderman;
 
-   enum TSGroupByOperation {
+   enum TSGroupByOperation
+   {
       OHLC = 0,
       Statistics
    }
 
    [DataStreamUserRespresentation("timeseriesanalysis")]
-   public class TimeSeriesGroupByStream : ViewStream {
-      public TimeSeriesGroupByStream(ViewStream md_str, string timebucket_col, string output_col, string operation) {
+   public class TimeSeriesGroupByStream : ViewStream
+   {
+      public TimeSeriesGroupByStream(ViewStream md_str, string timebucket_col, string output_col, string operation)
+      {
          ViewStream rs = new NonIncGroupByStream(md_str, new OHLCNonIncGroupByEvaluator(timebucket_col, output_col));
          mySetRealSource(rs);
       }
    }
-   public class OHLC {
-      public OHLC(double open_args, double high_args, double low_args, double close_args) {
+   public class OHLC
+   {
+      public OHLC(double open_args, double high_args, double low_args, double close_args)
+      {
          open = open_args;
          high = high_args;
          low = low_args;
@@ -22,36 +28,47 @@ namespace BCG {
       }
       public readonly double open, high, low, close;
    }
-   public class OHLCNonIncGroupByEvaluator : INonIncGroupByEvaluator {
+   public class OHLCNonIncGroupByEvaluator : INonIncGroupByEvaluator
+   {
       private string _timestamp_col;
       private string _output_col;
       private IBlueTableSchema _out_schema;
-      public OHLCNonIncGroupByEvaluator(string timestamp_col, string output_col) {
+      public OHLCNonIncGroupByEvaluator(string timestamp_col, string output_col)
+      {
          _timestamp_col = timestamp_col;
          _output_col = output_col;
       }
 
-      string open_col {
-         get {
+      string open_col
+      {
+         get
+         {
             return $"{_output_col}_open"; ;
          }
       }
-      string high_col {
-         get {
+      string high_col
+      {
+         get
+         {
             return $"{_output_col}_high"; ;
          }
       }
-      string low_col {
-         get {
+      string low_col
+      {
+         get
+         {
             return $"{_output_col}_low"; ;
          }
       }
-      string close_col {
-         get {
+      string close_col
+      {
+         get
+         {
             return $"{_output_col}_close"; ;
          }
       }
-      private void _build_schema(IBlueTableSchema input_schema) {
+      private void _build_schema(IBlueTableSchema input_schema)
+      {
          List<Tuple<Type, string>> new_cols = new List<Tuple<Type, string>>();
          ColumnInfo ci = input_schema[_output_col];
          new_cols.Add(Tuple.Create(ci.colType, open_col));
@@ -62,25 +79,32 @@ namespace BCG {
          _out_schema = BlueDataUtility.add_columns(input_schema, new_cols.ToArray());
       }
 
-      public IBlueRowCollection evaluate(IDictionary<int, List<IBlueRow>> grouped_rows) {
-         if (_out_schema == null) {
-            if (grouped_rows.Count > 0) {
-               foreach (var kv in grouped_rows) {
+      public IBlueRowCollection evaluate(IDictionary<int, List<IBlueRow>> grouped_rows)
+      {
+         if (_out_schema == null)
+         {
+            if (grouped_rows.Count > 0)
+            {
+               foreach (var kv in grouped_rows)
+               {
                   var rows = kv.Value;
-                  if (rows != null && rows.Count > 0) {
+                  if (rows != null && rows.Count > 0)
+                  {
                      _build_schema(rows.ElementAt(0).schema);
                   }
                }
             }
          }
          BlueTable rv = new BlueTable(_out_schema);
-         foreach (var kv in grouped_rows) {
+         foreach (var kv in grouped_rows)
+         {
             int id = kv.Key;
             IEnumerable<IBlueRow> rows = kv.Value;
             // sort rows on time to compute o,c correctly??
-            if (rows != null && rows.Count() > 0) {
+            if (rows != null && rows.Count() > 0)
+            {
                IBlueRow reprow = rows.ElementAt(0);
-               IEnumerable<double> vals = BlueDataUtility.get_colvalues<double>(rows, _output_col);
+               IEnumerable<double> vals = BlueDataUtility.get_covalues<double>(rows, _output_col);
                OHLC ohlc = compute_ohlc(vals);
                MutableBlueRow nr = new MutableBlueRow(_out_schema, reprow);
                nr.setValue("id", kv.Key);
@@ -94,11 +118,13 @@ namespace BCG {
          return rv;
       }
 
-      public string[] get_group_by_cols() {
+      public string[] get_group_by_cols()
+      {
          return new string[] { _timestamp_col };
       }
 
-      static OHLC compute_ohlc(IEnumerable<double> prices) {
+      static OHLC compute_ohlc(IEnumerable<double> prices)
+      {
          double o = prices.ElementAt(0);
          double c = prices.ElementAt(prices.Count() - 1);
          double l = prices.Min();
@@ -107,22 +133,28 @@ namespace BCG {
       }
    }
    [DataStreamUserRespresentation("RSI")]
-   public class RSIStream : ViewStream {
+   public class RSIStream : ViewStream
+   {
       // for ms_str expects a timeseries with a timestamp col and an output col (price col) to calculate the ohlc for
-      public RSIStream(ViewStream md_str, string timebucket_column, string output_col) {
+      public RSIStream(ViewStream md_str, string timebucket_column, string output_col)
+      {
 
       }
    }
    [DataStreamUserRespresentation("Bollinger")]
-   public class BollingerBandStream : ViewStream {
+   public class BollingerBandStream : ViewStream
+   {
       // for ms_str expects a timeseries with a timestamp col and an output col (price col) to calculate the ohlc for
-      public BollingerBandStream(ViewStream md_str, string timebucket_column, string output_col) {
+      public BollingerBandStream(ViewStream md_str, string timebucket_column, string output_col)
+      {
       }
    }
    [DataStreamUserRespresentation("OHLC")]
-   public class OHLCStream : ViewStream {
+   public class OHLCStream : ViewStream
+   {
       // for ms_str expects a timeseries with a timestamp col and an output col (price col) to calculate the ohlc for
-      public OHLCStream(ViewStream md_str, string timebucket_column, string output_col) {
+      public OHLCStream(ViewStream md_str, string timebucket_column, string output_col)
+      {
          ViewStream rs = new TimeSeriesGroupByStream(md_str, timebucket_column, output_col, TSGroupByOperation.OHLC.ToString());
          mySetRealSource(rs);
       }

src/DataStreamSuite/SharedStreams/SharedStreams.csproj

@@ -1,7 +1,7 @@
 <Project Sdk="Microsoft.NET.Sdk">
 
   <PropertyGroup>
-    <TargetFramework>net8.0</TargetFramework>
+    <TargetFramework>net7.0</TargetFramework>
     <ImplicitUsings>enable</ImplicitUsings>
     <Nullable>enable</Nullable>
   </PropertyGroup>

src/DataStreamSuite/TransportUtilities/TransportUtilities.csproj

@@ -1,7 +1,7 @@
 <Project Sdk="Microsoft.NET.Sdk">
 
   <PropertyGroup>
-    <TargetFramework>net8.0</TargetFramework>
+    <TargetFramework>net7.0</TargetFramework>
     <ImplicitUsings>enable</ImplicitUsings>
     <Nullable>enable</Nullable>
   </PropertyGroup>