fix build: Add a few includes, suppress some werrors

Not sure how this slipped through the cracks, but I'm guessing the build
broke when we upgraded the cpp compilers (to clang 17 and gcc 13).

* Include <cstdint> in a few places
* Suppress -Wstringop-overflow in sparkd-gcc-release build
* Change const char * param to explicit std::string for proper type
  deduction in call to 'std::make_tuple'. For some reason, the gcc build
  was failing with an array-bounds werror (changing the arg fixed it).

.gitmodules

@@ -1,6 +1,6 @@
 [submodule "externals"]
 	path = externals
-	url = https://x-token-auth:ATCTT3xFfGN0h_ENnUSfEAIzbfVzkQPLG26cIli50MtcTwZ7VEc8lin5tJYqibjlBJ9PM9R642Ra5GjiL52TBfbO8KeUBT0UoMT7gLQHL2Y1TkW9yZr6NkvsoDkSl49DeD4Ijiu4_0u4qEm4oMVDDg2_fB059-uOWOo7ZL94tGZET8CSVpFIGvs=B0E03660@bitbucket.org/bluetradesys/externals.git
+	url = git@bitbucket.org:bluetradesys/externals.git
 [submodule "vcpkg"]
 	path = vcpkg
 	url = https://github.com/microsoft/vcpkg.git

.gitpod.yml

@@ -1,49 +0,0 @@
-# This configuration file was automatically generated by Gitpod.
-# Please adjust to your needs (see https://www.gitpod.io/docs/introduction/learn-gitpod/gitpod-yaml)
-# and commit this file to your remote git repository to share the goodness with others.
-
-# Learn more from ready-to-use templates: https://www.gitpod.io/docs/introduction/getting-started/quickstart
-
-image: gitpod/workspace-dotnet-8
-tasks:
-  - name: Install Deps
-    init: |
-      (
-        set -e
-        sudo install-packages rapidjson-dev
-        sudo install-packages libssh2-1-dev
-        sudo install-packages openssh-server
-        sudo install-packages liblttng-ust-dev
-        sudo install-packages libpcap0.8-dev
-        sudo install-packages ccache
-        sudo ln -s $(which ccache) /usr/local/bin/clang
-        sudo ln -s $(which ccache) /usr/local/bin/clang++
-        sudo ln -s $(which ccache) /usr/local/bin/clang-17
-        sudo ln -s $(which ccache) /usr/local/bin/clang++-17
-        sudo ln -s $(which ccache) /usr/local/bin/gcc
-        sudo ln -s $(which ccache) /usr/local/bin/g++
-        sudo ln -s $(which ccache) /usr/local/bin/gcc-12
-        sudo ln -s $(which ccache) /usr/local/bin/g++-12
-        if [ -e externals/src/emumba/install_emumba-deribit-connector.sh ]; then
-          externals/src/emumba/install_emumba-deribit-connector.sh
-        fi
-      )
-  - name: Setup Python venv
-    init: |
-      python -m pip install --upgrade pip
-      python -m venv .venv
-      . .venv/bin/activate
-      python -m pip install --upgrade pip
-      pip install -r requirements.txt
-
-ports:
-  - name: sparkd config
-    port: 8000
-    onOpen: notify
-  - name: sparkd bus
-    port: 8001
-
-gitConfig:
-  alias.st: "status"
-  status.short: "true"
-  core.pager: "less -FXRS"

CMakePresets.json

@@ -143,7 +143,7 @@
       "cacheVariables": {
         "CMAKE_C_COMPILER": "gcc",
         "CMAKE_CXX_COMPILER": "g++",
-        "CMAKE_CXX_FLAGS_RELEASE": "-fPIC -Wall -pedantic -Wextra -Werror -Wno-missing-braces -Wno-ignored-qualifiers --std=c++20 -fdiagnostics-color -Wno-stringop-overread -Wno-maybe-uninitialized -fno-omit-frame-pointer -DNDEBUG -O3",
+        "CMAKE_CXX_FLAGS_RELEASE": "-fPIC -Wall -pedantic -Wextra -Werror -Wno-missing-braces -Wno-ignored-qualifiers -Wno-stringop-overflow --std=c++20 -fdiagnostics-color -Wno-stringop-overread -Wno-maybe-uninitialized -fno-omit-frame-pointer -DNDEBUG -O3",
         "CMAKE_INSTALL_PREFIX": "${sourceDir}/install/gcc/release/sparkd"
       }
     },

docker/sparkd/dev/Dockerfile

@@ -2,40 +2,55 @@ FROM ubuntu:22.04
 
 ARG DEBIAN_FRONTEND=noninteractive
 
-# Foundational Deps
-RUN apt-get update && apt-get install -y \
-    build-essential \
-    ca-certificates \
-    curl \
-    gnupg \
-    liblttng-ust-dev \
-    libpcap0.8-dev \
-    libssh2-1-dev \
-    locales \
-    openssh-server \
-    software-properties-common \
-    sudo \
-    tzdata
+# Note installations that take a while (more than 10s)
 
-# Clang 17
+RUN apt-get update
+RUN apt-get install -y ca-certificates
+RUN apt-get install -y ccache
+RUN apt-get install -y curl
+RUN apt-get install -y fish
+RUN apt-get install -y gdb
+# git-all: ~250s
+RUN apt-get install -y git-all
+RUN apt-get install -y gnupg
+# liblttng-ust-dev: ~30s
+RUN apt-get install -y liblttng-ust-dev
+RUN apt-get install -y libssh2-1-dev
+RUN apt-get install -y locales
+RUN apt-get install -y ninja-build
+# openssh-server: ~40s
+RUN apt-get install -y openssh-server
+RUN apt-get install -y pkg-config
+RUN apt-get install -y software-properties-common
+RUN apt-get install -y sudo
+RUN apt-get install -y tzdata
+RUN apt-get install -y vim
+RUN apt-get install -y zip
+
+# Python: ~30s
+RUN apt-get install -y \
+    python3-pip \
+    python3-venv \
+    && python3 -m pip install --upgrade pip
+
+# GCC
+ARG gcc_major=13
+RUN add-apt-repository ppa:ubuntu-toolchain-r/test \
+    && apt-get update \
+    && apt-get install -y g++-$gcc_major
+
+# Clang
+ARG clang_major=17
 RUN curl -fsSL https://apt.llvm.org/llvm-snapshot.gpg.key 2> /dev/null \
     | gpg --dearmor - \
     | tee /usr/share/keyrings/llvm-snapshot.gpg > /dev/null \
-    && echo "deb [signed-by=/usr/share/keyrings/llvm-snapshot.gpg] http://apt.llvm.org/jammy/ llvm-toolchain-jammy-17 main" \
+    && echo "deb [signed-by=/usr/share/keyrings/llvm-snapshot.gpg] http://apt.llvm.org/$(lsb_release -cs)/ llvm-toolchain-$(lsb_release -cs)-$clang_major main" \
     | tee -a "/etc/apt/sources.list.d/llvm.list" \
     && apt-get update \
     && apt-get install -y \
-    clang-17 \
-    clang-format-17 \
-    clangd-17
-
-# G++
-RUN apt-get update && apt-get install -y \
-    g++
-
-# Ninja
-RUN apt-get update && apt-get install -y \
-    ninja-build
+    clang-$clang_major \
+    clang-format-$clang_major \
+    clangd-$clang_major
 
 # CMake
 RUN curl -fsSL https://apt.kitware.com/keys/kitware-archive-latest.asc 2> /dev/null \
@@ -53,12 +68,6 @@ RUN curl -fsSLO "https://packages.microsoft.com/config/ubuntu/$(lsb_release -rs)
     && apt-get update \
     && apt-get install -y dotnet-sdk-8.0
 
-# Python
-RUN apt-get update && apt-get install -y \
-    python3-pip \
-    python3-venv \
-    && python3 -m pip install --upgrade pip
-
 # Node
 ARG node_major=18
 RUN mkdir -p /etc/apt/keyrings \
@@ -69,41 +78,35 @@ RUN mkdir -p /etc/apt/keyrings \
     && apt-get update \
     && apt-get install -y nodejs
 
-# Sqlite3
+# Emumba deps
 RUN apt-get update && apt-get install -y \
-    sqlite3
-
-# Dev Utils (install last so they can be added/removed with minimal rebuild)
-RUN apt-get update && apt-get install -y \ 
-    ccache \
-    fish \
-    gdb \
-    git-all \
-    vim
+    build-essential \
+    doxygen \
+    graphviz \
+    libgtest-dev \
+    libssl-dev \
+    pkg-config \
+    python3-sphinx \
+    rapidjson-dev \
+    zlib1g-dev
 
 # Set up default executables (update-alternatives)
-RUN update-alternatives --install /usr/bin/clang++ clang++ /usr/bin/clang++-17 100; \
-    update-alternatives --install /usr/bin/clang clang /usr/bin/clang-17 100; \
-    update-alternatives --install /usr/bin/clang-format clang-format /usr/bin/clang-format-17 100; \
-    update-alternatives --install /usr/bin/clangd clangd /usr/bin/clangd-17 100; \
-    update-alternatives --install /usr/bin/g++ g++ /usr/bin/g++-12 100; \
-    update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-12 100
+RUN update-alternatives --install /usr/bin/clang clang /usr/bin/clang-$clang_major 100; \
+    update-alternatives --install /usr/bin/clang++ clang++ /usr/bin/clang++-$clang_major 100; \
+    update-alternatives --install /usr/bin/clang-format clang-format /usr/bin/clang-format-$clang_major 100; \
+    update-alternatives --install /usr/bin/clangd clangd /usr/bin/clangd-$clang_major 100; \
+    update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-$gcc_major 100; \
+    update-alternatives --install /usr/bin/g++ g++ /usr/bin/g++-$gcc_major 100
 
 # Add compiler symlinks for ccache 
 RUN ln -s /usr/bin/ccache /usr/local/bin/clang; \
     ln -s /usr/bin/ccache /usr/local/bin/clang++; \
-    ln -s /usr/bin/ccache /usr/local/bin/clang-13; \
-    ln -s /usr/bin/ccache /usr/local/bin/clang++-13; \
-    ln -s /usr/bin/ccache /usr/local/bin/clang-15; \
-    ln -s /usr/bin/ccache /usr/local/bin/clang++-15; \
-    ln -s /usr/bin/ccache /usr/local/bin/clang-17; \
-    ln -s /usr/bin/ccache /usr/local/bin/clang++-17; \
+    ln -s /usr/bin/ccache /usr/local/bin/clang-$clang_major; \
+    ln -s /usr/bin/ccache /usr/local/bin/clang++-$clang_major; \
     ln -s /usr/bin/ccache /usr/local/bin/gcc; \
     ln -s /usr/bin/ccache /usr/local/bin/g++; \
-    ln -s /usr/bin/ccache /usr/local/bin/gcc-11; \
-    ln -s /usr/bin/ccache /usr/local/bin/g++-11; \
-    ln -s /usr/bin/ccache /usr/local/bin/gcc-12; \
-    ln -s /usr/bin/ccache /usr/local/bin/g++-12
+    ln -s /usr/bin/ccache /usr/local/bin/gcc-$gcc_major; \
+    ln -s /usr/bin/ccache /usr/local/bin/g++-$gcc_major
 
 RUN locale-gen en_US.UTF-8
 

global.json

@@ -1,6 +0,0 @@
-{
-  "sdk": {
-    "version": "6.0.200",
-    "rollForward": "latestMajor"
-  }
-}

src/DataStreamSuite/BooleanExpressions/BooleanExpressionTokens.lex

@@ -42,7 +42,6 @@ OpenParenthesis          \(
 CloseParenthesis         \)
 Comma                    ,
 Negate                   '!'
-MapsTo                   ':'
 IsOfType                 '/'
 
 %%
@@ -98,10 +97,6 @@ IsOfType                 '/'
    return (int)Token.NEGATE;
 }
 
-{MapsTo} {
-   return (int)Token.MAPS_TO;
-}
-
 {Equal} {
    return (int)Token.EQUAL;
 }

src/DataStreamSuite/Bts.DataStream.Websocket/DataSync.cs

@@ -5,9 +5,9 @@ using BCG;
 using bts;
 using bts.api.core;
 using bts.collections;
+using bts.websocket_core.json_rpc;
 
 using Bts.DataStreamExpressions;
-using Bts.Websocket;
 
 using mbts.utils;
 

src/DataStreamSuite/Bts.DataStream.Websocket/JsonParseResult.cs

@@ -1,24 +0,0 @@
-namespace Bts.DataStream.Websocket;
-
-using Newtonsoft.Json.Linq;
-
-class JsonParseResult {
-   public string? MessageId { get; init; }
-   public RequestType Type { get; init; }
-   public JObject? Parameters { get; init; }
-   public string? Error { get; init; }
-   public bool Successful => Error is not null;
-
-   public static JsonParseResult CreateError(string error) =>
-      new() {
-         Error = error,
-         MessageId = "missing"
-      };
-
-   public static JsonParseResult Create(string messageId, RequestType type, JObject parameters) =>
-      new() {
-         MessageId = messageId,
-         Parameters = parameters,
-         Type = type
-      };
-}

src/DataStreamSuite/Bts.DataStream.Websocket/JsonRpcErrorCode.cs

@@ -1,6 +0,0 @@
-namespace Bts.DataStream.Websocket;
-
-public enum JsonRpcErrorCode {
-   None = 0,
-   InvalidRequest = 1,
-}

src/DataStreamSuite/Bts.DataStream.Websocket/Session.cs

@@ -2,8 +2,6 @@ namespace Bts.DataStream.Websocket;
 
 using bts;
 
-using Bts.Websocket;
-
 using Google.Protobuf;
 
 using System.Text.Json;
@@ -15,7 +13,7 @@ public static class Session {
       WebsocketServerConfig config, JsonSerializerOptions? opts) {
       opts ??= JsonSerializerOptions.Default;
       var sessionId = Interlocked.Increment(ref sessionIdCounter);
-      var gateway = new WebsocketGatewayObservable(config);
+      var gateway = new WebsocketClient(config);
       var sync = new DataSync(opts);
       using (gateway.Subscribe(sync))
       using (sync.Subscribe(gateway)) {

src/DataStreamSuite/Bts.DataStream.Websocket/SubscriptionParams.cs

@@ -1,30 +1,30 @@
 namespace Bts.DataStream.Websocket;
 
+using bts.utils;
+
 using Spark.Core;
+using Spark.Core.Json;
 
 using System;
 using System.Diagnostics.CodeAnalysis;
 using System.Text.Json;
 using System.Text.Json.Serialization;
 
-public class SubscriptionParams : IEquatable<SubscriptionParams> {
+public class SubscriptionParams(string? query = null) : IEquatable<SubscriptionParams> {
    static class PropNames {
       public const string QUERY = "query";
    }
 
-   public SubscriptionParams(string? query) {
-      Query = query;
-   }
-
    // Serializer supports plain string, or an array of lines for convenience
    [MaybeNull]
    [JsonPropertyName(PropNames.QUERY)]
    [JsonConverter(typeof(MultilineStringJsonConverter))]
-   public string? Query { get; set; }
+   public string? Query { get; set; } = query;
 
    public bool Equals(SubscriptionParams? other) =>
       other is not null &&
-      Query == other.Query;
+      // Not the most elegant equals impl, but it'll work
+      Query.safe_equals(other.Query, (a, b) => a?.ToString() == b?.ToString());
 
    class SubscriptionParamsConverter : JsonConverter<SubscriptionParams> {
       public override bool CanConvert(Type typeToConvert) =>
@@ -38,19 +38,19 @@ public class SubscriptionParams : IEquatable<SubscriptionParams> {
          return read(ref reader);
       }
 
-      SubscriptionParams read(ref Utf8JsonReader reader) {
+      static SubscriptionParams read(ref Utf8JsonReader reader) {
          return reader.TokenType switch {
-            JsonTokenType.String => stringImpl(ref reader),
-            JsonTokenType.StartArray => arrayImpl(ref reader),
-            JsonTokenType.StartObject => objectImpl(ref reader),
+            JsonTokenType.String => StringImpl(ref reader),
+            JsonTokenType.StartArray => ArrayImpl(ref reader),
+            JsonTokenType.StartObject => ObjectImpl(ref reader),
             _ => throw new ArgumentException(
                $"Unable to parse {typeof(SubscriptionParams)} from {reader.TokenType}"),
          };
 
-         static SubscriptionParams stringImpl(ref Utf8JsonReader reader) =>
+         static SubscriptionParams StringImpl(ref Utf8JsonReader reader) =>
             new(reader.GetString()!);
 
-         static SubscriptionParams arrayImpl(ref Utf8JsonReader reader) {
+         static SubscriptionParams ArrayImpl(ref Utf8JsonReader reader) {
             var lines = new List<string>(2);
             while (reader.Read()) {
                switch (reader.TokenType) {
@@ -65,7 +65,7 @@ public class SubscriptionParams : IEquatable<SubscriptionParams> {
                $"Failed to parse {typeof(SubscriptionParams)} (did not reach end of array)");
          }
 
-         static SubscriptionParams objectImpl(ref Utf8JsonReader reader) {
+         static SubscriptionParams ObjectImpl(ref Utf8JsonReader reader) {
             var lines = new List<string>(2);
             while (reader.Read()) {
                switch (reader.TokenType) {
@@ -74,9 +74,9 @@ public class SubscriptionParams : IEquatable<SubscriptionParams> {
                      if (propName == PropNames.QUERY) {
                         switch (reader.TokenType) {
                            case JsonTokenType.StartArray:
-                              return arrayImpl(ref reader);
+                              return ArrayImpl(ref reader);
                            case JsonTokenType.String:
-                              return stringImpl(ref reader);
+                              return StringImpl(ref reader);
                         }
                      }
                      break;
@@ -100,11 +100,13 @@ public class SubscriptionParams : IEquatable<SubscriptionParams> {
             writer.WriteString(PropNames.QUERY, value.Query);
             return;
          }
-         writer.WriteStartArray(PropNames.QUERY);
-         foreach (var l in lines) {
-            writer.WriteStringValue(l);
-         }
-         writer.WriteEndArray();
+         writer.WriteArray(PropNames.QUERY, lines);
       }
    }
+
+   public override bool Equals(object? obj) =>
+      Equals(obj as SubscriptionParams);
+
+   public override int GetHashCode() =>
+      Query?.Length ?? typeof(SubscriptionParams).GetHashCode();
 }

src/DataStreamSuite/Bts.DataStream.Websocket/ViewStreamUpdate.cs

@@ -1,41 +1,41 @@
-#nullable enable
-using BCG;
-
-using Spark.Core.Json;
-
-using System;
-using System.Collections.Generic;
-using System.Linq;
-using System.Text.Json;
-using System.Text.Json.Serialization;
-
-namespace Bts.DataStreamCore;
-
-public class DataStreamUpdate {
-   [JsonPropertyName("tableName")]
-   public string? TableName { get; set; }
-
-   [JsonPropertyName("columns")]
-   public List<ColumnInfo>? Columns { get; set; }
-
-   [JsonPropertyName("addedRows")]
-   public List<IBlueRow>? AddedRows { get; set; }
-
-   [JsonPropertyName("deletedRows")]
-   public List<IBlueRow>? DeletedRows { get; set; }
-
-   public DataPacket ToDataPacket(BlueTable state, bool applyChanges = true) {
-      var schema = new BlueTableSchema([.. Columns!], TableName);
-      var addedRowsTable = new BlueTable(schema);
-      var deletedRowsTable = new BlueTable(schema);
-      if (applyChanges) {
-         state.deleteRows(deletedRowsTable);
-         state.addRows(addedRowsTable);
-      }
-      return new(state, addedRowsTable, deletedRowsTable);
-   }
-
-   public static string ToJson() {
-      return "";
-   }
-}
+#nullable enable
+using BCG;
+
+using Spark.Core.Json;
+
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text.Json;
+using System.Text.Json.Serialization;
+
+namespace Bts.DataStreamCore;
+
+class ViewStreamUpdate {
+   [JsonPropertyName("tableName")]
+   public string? TableName { get; set; }
+
+   [JsonPropertyName("columns")]
+   public List<ColumnInfo>? Columns { get; set; }
+
+   [JsonPropertyName("addedRows")]
+   public List<Dictionary<string, object?>>? AddedRows { get; set; }
+
+   [JsonPropertyName("deletedRows")]
+   public List<Dictionary<string, object?>>? DeletedRows { get; set; }
+
+   public DataPacket ToDataPacket(BlueTable state, bool applyChanges = true) {
+      var schema = new BlueTableSchema([.. Columns!], TableName);
+      var addedRowsTable = new BlueTable(schema);
+      var deletedRowsTable = new BlueTable(schema);
+      if (applyChanges) {
+         state.deleteRows(deletedRowsTable);
+         state.addRows(addedRowsTable);
+      }
+      return new(state, addedRowsTable, deletedRowsTable);
+   }
+
+   public static string ToJson() {
+      return "";
+   }
+}

src/DataStreamSuite/Bts.DataStream.WebsocketClient/Bts.DataStream.WebsocketClient.csproj

@@ -1,10 +0,0 @@
-<Project Sdk="Microsoft.NET.Sdk.Web">
-
-  <PropertyGroup>
-    <TargetFramework>net7.0</TargetFramework>
-    <Nullable>enable</Nullable>
-    <ImplicitUsings>enable</ImplicitUsings>
-    <EnableDefaultContentItems>True</EnableDefaultContentItems>
-  </PropertyGroup>
-
-</Project>

src/DataStreamSuite/Bts.DataStream.WebsocketClient/Pages/Error.cshtml

@@ -1,26 +0,0 @@
-@page
-@model ErrorModel
-@{
-    ViewData["Title"] = "Error";
-}
-
-<h1 class="text-danger">Error.</h1>
-<h2 class="text-danger">An error occurred while processing your request.</h2>
-
-@if (Model.ShowRequestId)
-{
-    <p>
-        <strong>Request ID:</strong> <code>@Model.RequestId</code>
-    </p>
-}
-
-<h3>Development Mode</h3>
-<p>
-    Swapping to the <strong>Development</strong> environment displays detailed information about the error that occurred.
-</p>
-<p>
-    <strong>The Development environment shouldn't be enabled for deployed applications.</strong>
-    It can result in displaying sensitive information from exceptions to end users.
-    For local debugging, enable the <strong>Development</strong> environment by setting the <strong>ASPNETCORE_ENVIRONMENT</strong> environment variable to <strong>Development</strong>
-    and restarting the app.
-</p>

src/DataStreamSuite/Bts.DataStream.WebsocketClient/Pages/Error.cshtml.cs

@@ -1,27 +0,0 @@
-using System.Diagnostics;
-using Microsoft.AspNetCore.Mvc;
-using Microsoft.AspNetCore.Mvc.RazorPages;
-
-namespace Bts.DataStream.WebsocketClient.Pages;
-
-[ResponseCache(Duration = 0, Location = ResponseCacheLocation.None, NoStore = true)]
-[IgnoreAntiforgeryToken]
-public class ErrorModel : PageModel
-{
-    public string? RequestId { get; set; }
-
-    public bool ShowRequestId => !string.IsNullOrEmpty(RequestId);
-
-    private readonly ILogger<ErrorModel> _logger;
-
-    public ErrorModel(ILogger<ErrorModel> logger)
-    {
-        _logger = logger;
-    }
-
-    public void OnGet()
-    {
-        RequestId = Activity.Current?.Id ?? HttpContext.TraceIdentifier;
-    }
-}
-

src/DataStreamSuite/Bts.DataStream.WebsocketClient/Pages/Index.cshtml

@@ -1,15 +0,0 @@
-@page
-@model IndexModel
-@{
-  ViewData["Title"] = "Home page";
-}
-
-<div class="text-center">
-  <h1 class="display-4">Welcome</h1>
-  <p>Learn about <a href="https://docs.microsoft.com/aspnet/core">building Web apps with ASP.NET Core</a>.</p>
-  <button id="connectButton">Connect to Websocket</button>
-  <div id="output"></div>
-  <input type="text" id="methodInput" placeholder="Enter method name" />
-  <input type="text" id="paramsInput" placeholder="Enter method parameters (if any)" />
-  <button id="sendMessageButton">Send JSON-RPC Request</button>
-</div>

src/DataStreamSuite/Bts.DataStream.WebsocketClient/Pages/Index.cshtml.cs

@@ -1,16 +0,0 @@
-using Microsoft.AspNetCore.Mvc;
-using Microsoft.AspNetCore.Mvc.RazorPages;
-
-namespace Bts.DataStream.WebsocketClient.Pages;
-
-public class IndexModel : PageModel {
-   private readonly ILogger<IndexModel> _logger;
-
-   public IndexModel(ILogger<IndexModel> logger) {
-      _logger = logger;
-   }
-
-   public void OnGet() {
-
-   }
-}

src/DataStreamSuite/Bts.DataStream.WebsocketClient/Pages/Privacy.cshtml

@@ -1,8 +0,0 @@
-@page
-@model PrivacyModel
-@{
-    ViewData["Title"] = "Privacy Policy";
-}
-<h1>@ViewData["Title"]</h1>
-
-<p>Use this page to detail your site's privacy policy.</p>

src/DataStreamSuite/Bts.DataStream.WebsocketClient/Pages/Privacy.cshtml.cs

@@ -1,19 +0,0 @@
-using Microsoft.AspNetCore.Mvc;
-using Microsoft.AspNetCore.Mvc.RazorPages;
-
-namespace Bts.DataStream.WebsocketClient.Pages;
-
-public class PrivacyModel : PageModel
-{
-    private readonly ILogger<PrivacyModel> _logger;
-
-    public PrivacyModel(ILogger<PrivacyModel> logger)
-    {
-        _logger = logger;
-    }
-
-    public void OnGet()
-    {
-    }
-}
-

src/DataStreamSuite/Bts.DataStream.WebsocketClient/Pages/Shared/_Layout.cshtml

@@ -1,51 +0,0 @@
-<!DOCTYPE html>
-<html lang="en">
-<head>
-    <meta charset="utf-8" />
-    <meta name="viewport" content="width=device-width, initial-scale=1.0" />
-    <title>@ViewData["Title"] - Bts.DataStream.WebsocketClient</title>
-    <link rel="stylesheet" href="~/lib/bootstrap/dist/css/bootstrap.min.css" />
-    <link rel="stylesheet" href="~/css/site.css" asp-append-version="true" />
-    <link rel="stylesheet" href="~/Bts.DataStream.WebsocketClient.styles.css" asp-append-version="true" />
-</head>
-<body>
-    <header>
-        <nav class="navbar navbar-expand-sm navbar-toggleable-sm navbar-light bg-white border-bottom box-shadow mb-3">
-            <div class="container">
-                <a class="navbar-brand" asp-area="" asp-page="/Index">Bts.DataStream.WebsocketClient</a>
-                <button class="navbar-toggler" type="button" data-bs-toggle="collapse" data-bs-target=".navbar-collapse" aria-controls="navbarSupportedContent"
-                        aria-expanded="false" aria-label="Toggle navigation">
-                    <span class="navbar-toggler-icon"></span>
-                </button>
-                <div class="navbar-collapse collapse d-sm-inline-flex justify-content-between">
-                    <ul class="navbar-nav flex-grow-1">
-                        <li class="nav-item">
-                            <a class="nav-link text-dark" asp-area="" asp-page="/Index">Home</a>
-                        </li>
-                        <li class="nav-item">
-                            <a class="nav-link text-dark" asp-area="" asp-page="/Privacy">Privacy</a>
-                        </li>
-                    </ul>
-                </div>
-            </div>
-        </nav>
-    </header>
-    <div class="container">
-        <main role="main" class="pb-3">
-            @RenderBody()
-        </main>
-    </div>
-
-    <footer class="border-top footer text-muted">
-        <div class="container">
-            &copy; 2023 - Bts.DataStream.WebsocketClient - <a asp-area="" asp-page="/Privacy">Privacy</a>
-        </div>
-    </footer>
-
-    <script src="~/lib/jquery/dist/jquery.min.js"></script>
-    <script src="~/lib/bootstrap/dist/js/bootstrap.bundle.min.js"></script>
-    <script src="~/js/site.js" asp-append-version="true"></script>
-
-    @await RenderSectionAsync("Scripts", required: false)
-</body>
-</html>

src/DataStreamSuite/Bts.DataStream.WebsocketClient/Pages/Shared/_Layout.cshtml.css

@@ -1,48 +0,0 @@
-/* Please see documentation at https://docs.microsoft.com/aspnet/core/client-side/bundling-and-minification
-for details on configuring this project to bundle and minify static web assets. */
-
-a.navbar-brand {
-  white-space: normal;
-  text-align: center;
-  word-break: break-all;
-}
-
-a {
-  color: #0077cc;
-}
-
-.btn-primary {
-  color: #fff;
-  background-color: #1b6ec2;
-  border-color: #1861ac;
-}
-
-.nav-pills .nav-link.active, .nav-pills .show > .nav-link {
-  color: #fff;
-  background-color: #1b6ec2;
-  border-color: #1861ac;
-}
-
-.border-top {
-  border-top: 1px solid #e5e5e5;
-}
-.border-bottom {
-  border-bottom: 1px solid #e5e5e5;
-}
-
-.box-shadow {
-  box-shadow: 0 .25rem .75rem rgba(0, 0, 0, .05);
-}
-
-button.accept-policy {
-  font-size: 1rem;
-  line-height: inherit;
-}
-
-.footer {
-  position: absolute;
-  bottom: 0;
-  width: 100%;
-  white-space: nowrap;
-  line-height: 60px;
-}

src/DataStreamSuite/Bts.DataStream.WebsocketClient/Pages/Shared/_ValidationScriptsPartial.cshtml

@@ -1,2 +0,0 @@
-<script src="~/lib/jquery-validation/dist/jquery.validate.min.js"></script>
-<script src="~/lib/jquery-validation-unobtrusive/jquery.validate.unobtrusive.min.js"></script>

src/DataStreamSuite/Bts.DataStream.WebsocketClient/Pages/_ViewImports.cshtml

@@ -1,3 +0,0 @@
-@using Bts.DataStream.WebsocketClient
-@namespace Bts.DataStream.WebsocketClient.Pages
-@addTagHelper *, Microsoft.AspNetCore.Mvc.TagHelpers

src/DataStreamSuite/Bts.DataStream.WebsocketClient/Pages/_ViewStart.cshtml

@@ -1,3 +0,0 @@
-@{
-    Layout = "_Layout";
-}

src/DataStreamSuite/Bts.DataStream.WebsocketClient/Program.cs

@@ -1,26 +0,0 @@
-using Microsoft.AspNetCore.Mvc;
-
-var builder = WebApplication.CreateBuilder(args);
-
-// Add services to the container.
-builder.Services.AddRazorPages();
-
-var app = builder.Build();
-
-// Configure the HTTP request pipeline.
-if (!app.Environment.IsDevelopment()) {
-   app.UseExceptionHandler("/Error");
-   // The default HSTS value is 30 days. You may want to change this for production scenarios, see https://aka.ms/aspnetcore-hsts.
-   app.UseHsts();
-}
-
-app.UseHttpsRedirection();
-app.UseStaticFiles();
-
-app.UseRouting();
-
-app.UseAuthorization();
-
-app.MapRazorPages();
-
-app.Run();

src/DataStreamSuite/Bts.DataStream.WebsocketClient/Properties/launchSettings.json

@@ -1,37 +0,0 @@
-{
-  "iisSettings": {
-    "windowsAuthentication": false,
-    "anonymousAuthentication": true,
-    "iisExpress": {
-      "applicationUrl": "http://localhost:29312",
-      "sslPort": 44390
-    }
-  },
-  "profiles": {
-    "http": {
-      "commandName": "Project",
-      "dotnetRunMessages": true,
-      "launchBrowser": true,
-      "applicationUrl": "http://localhost:5214",
-      "environmentVariables": {
-        "ASPNETCORE_ENVIRONMENT": "Development"
-      }
-    },
-    "https": {
-      "commandName": "Project",
-      "dotnetRunMessages": true,
-      "launchBrowser": true,
-      "applicationUrl": "https://localhost:7176;http://localhost:5214",
-      "environmentVariables": {
-        "ASPNETCORE_ENVIRONMENT": "Development"
-      }
-    },
-    "IIS Express": {
-      "commandName": "IISExpress",
-      "launchBrowser": true,
-      "environmentVariables": {
-        "ASPNETCORE_ENVIRONMENT": "Development"
-      }
-    }
-  }
-}

src/DataStreamSuite/Bts.DataStream.WebsocketClient/appsettings.Development.json

@@ -1,9 +0,0 @@
-{
-  "DetailedErrors": true,
-  "Logging": {
-    "LogLevel": {
-      "Default": "Information",
-      "Microsoft.AspNetCore": "Warning"
-    }
-  }
-}

src/DataStreamSuite/Bts.DataStream.WebsocketClient/appsettings.json

@@ -1,9 +0,0 @@
-{
-  "Logging": {
-    "LogLevel": {
-      "Default": "Information",
-      "Microsoft.AspNetCore": "Warning"
-    }
-  },
-  "AllowedHosts": "*"
-}

src/DataStreamSuite/Bts.DataStream.WebsocketClient/wwwroot/css/site.css

@@ -1,22 +0,0 @@
-html {
-  font-size: 14px;
-}
-
-@media (min-width: 768px) {
-  html {
-    font-size: 16px;
-  }
-}
-
-.btn:focus, .btn:active:focus, .btn-link.nav-link:focus, .form-control:focus, .form-check-input:focus {
-  box-shadow: 0 0 0 0.1rem white, 0 0 0 0.25rem #258cfb;
-}
-
-html {
-  position: relative;
-  min-height: 100%;
-}
-
-body {
-  margin-bottom: 60px;
-}

src/DataStreamSuite/Bts.DataStream.WebsocketClient/wwwroot/js/site.js

@@ -1,67 +0,0 @@
-// Please see documentation at https://docs.microsoft.com/aspnet/core/client-side/bundling-and-minification
-// for details on configuring this project to bundle and minify static web assets.
-
-// Write your JavaScript code.
-const connectButton = document.getElementById('connectButton');
-const output = document.getElementById('output');
-const methodInput = document.getElementById('methodInput');
-const paramsInput = document.getElementById('paramsInput');
-const sendMessageButton = document.getElementById('sendMessageButton');
-
-let socket;
-
-connectButton.addEventListener('click', () => {
-    if (socket && socket.readyState === WebSocket.OPEN) {
-        return;
-    }
-
-    socket = new WebSocket('ws://localhost:8080'); // Replace with your WebSocket server URL
-
-    socket.onopen = (event) => {
-        output.innerHTML += 'Connected to WebSocket server<br>';
-    };
-
-    socket.onmessage = (event) => {
-        const message = JSON.parse(event.data);
-        output.innerHTML += `Received JSON-RPC response: ${JSON.stringify(message)}<br>`;
-    };
-
-    socket.onclose = (event) => {
-        if (event.wasClean) {
-            output.innerHTML += `Connection closed cleanly, code=${event.code}, reason=${event.reason}<br>`;
-        } else {
-            output.innerHTML += 'Connection abruptly closed<br>';
-        }
-    };
-
-    socket.onerror = (error) => {
-        output.innerHTML += `WebSocket Error: ${error.message}<br>`;
-    };
-});
-
-sendMessageButton.addEventListener('click', () => {
-    // const method = methodInput.value;
-    // const params = paramsInput.value;
-    const method = 'subscribe';
-    const params = `{
-"expression": "literal("n", "2.0")"
-}'
-`
-    if (socket && socket.readyState === WebSocket.OPEN && method) {
-        // const reqParams = {
-        //   'expression': message
-        // }
-        const request = {
-            jsonrpc: '2.0',
-            method: method,
-            params: {
-              query: `literal("n", "2.0")`
-            },
-            id: 1, // You can set a unique ID for each request
-        };
-
-        socket.send(JSON.stringify(request));
-        methodInput.value = ''; // Clear the input fields
-        paramsInput.value = '';
-    }
-});

src/DataStreamSuite/Bts.DataStream.WebsocketServer/Bts.DataStream.WebsocketServer.csproj

@@ -2,7 +2,7 @@
 
   <PropertyGroup>
     <OutputType>Exe</OutputType>
-    <TargetFramework>net7.0</TargetFramework>
+    <TargetFramework>net8.0</TargetFramework>
     <ImplicitUsings>enable</ImplicitUsings>
     <Nullable>enable</Nullable>
   </PropertyGroup>

src/DataStreamSuite/Bts.DataStream.WebsocketServer/Usings.cs

@@ -1,6 +1,5 @@
-global using Bts.Websocket;
-
 global using bts.websocket_core;
+global using bts.websocket_core.json_rpc;
 
 global using System.CommandLine;
 

src/DataStreamSuite/Bts.DataStreamWebApp.Server/Controllers/DataStreamSubscriptionController.cs

@@ -3,9 +3,9 @@ namespace Bts.DataStreamWebApp.Server.Controllers;
 using BCG;
 
 using bts.api.core;
+using bts.websocket_core.json_rpc;
 
 using Bts.DataStreamExpressions;
-using Bts.Websocket;
 
 using Microsoft.AspNetCore.Mvc;
 
@@ -17,13 +17,13 @@ using System.Text.Json;
 
 [Route("/ws")]
 public class DataStreamSubscriptionController(ILogger<DataStreamSubscriptionController> logger) : ControllerBase {
-   readonly ILogger<DataStreamSubscriptionController> _logger = logger;
+   readonly ILogger<DataStreamSubscriptionController> logger = logger;
    IDisposable? streamSubscription;
 
    [Route("/ws")]
    public async Task Get() {
       if (!HttpContext.WebSockets.IsWebSocketRequest) {
-         _logger.LogError("Expected websocket request on path '{}'", HttpContext.Request.Path);
+         logger.LogError("Expected websocket request on path '{}'", HttpContext.Request.Path);
          return;
       }
       Console.WriteLine("Establishing websocket connection...");
@@ -67,16 +67,17 @@ public class DataStreamSubscriptionController(ILogger<DataStreamSubscriptionCont
                 new ArraySegment<byte>(buffer), CancellationToken.None);
             var msg = Encoding.UTF8.GetString(buffer, 0, receiveResult.Count);
             var rpcReq = JsonSerializer.Deserialize<JsonRpcRequest<DataStreamSubscription>>(msg);
-            var queryArgs = rpcReq!.Params;
-            var exp = DataStreamExpressionsInterpreter.Evaluate(queryArgs.Query!);
+            var query = rpcReq?.Params?.Query ?? throw new InvalidOperationException(
+               "Failed to parse JsonRpcRequest.Params");
+            var exp = DataStreamExpressionsInterpreter.Evaluate(query);
             if (exp is null) {
-               Console.WriteLine($"Invalid query expression: {queryArgs.Query}.");
+               Console.WriteLine($"Invalid query expression: {query}.");
                continue;
             }
             var vs = exp.getView(new Dictionary<string, StatementExpression>(0));
             Disposable.dispose(ref streamSubscription);
-            vs.onChange += onStreamChanged;
-            streamSubscription = Disposable.Create(() => vs.onChange -= onStreamChanged);
+            vs.onChange += OnStreamChanged;
+            streamSubscription = Disposable.Create(() => vs.onChange -= OnStreamChanged);
          }
       } catch (Exception ex) {
          Console.WriteLine($"Exception on WebsocketServer thread: {ex}");
@@ -85,7 +86,7 @@ public class DataStreamSubscriptionController(ILogger<DataStreamSubscriptionCont
 
       return 0;
 
-      async void onStreamChanged(object newValue) {
+      async void OnStreamChanged(object newValue) {
          if (newValue is not DataPacket dp) {
             Console.WriteLine(
                $"OnStreamChanged delegate expected DataPacket (actually received {newValue?.GetType()}.");

src/DataStreamSuite/Bts.DataStreamWebApp.Server/Program.cs

@@ -3,6 +3,7 @@ using BCG;
 using bts;
 using bts.api.core;
 using bts.utils;
+using bts.websocket_core.json_rpc;
 
 using Bts.DataStreamExpressions;
 using Bts.DataStreamWebApp.Server;
@@ -111,7 +112,7 @@ async Task<int> runWebSocketServer(string? sparkd, string user, string password,
          receiveResult = await webSocket.ReceiveAsync(
              new ArraySegment<byte>(buffer), CancellationToken.None);
          var msg = Encoding.UTF8.GetString(buffer, 0, receiveResult.Count);
-         var rpcReq = JsonSerializer.Deserialize<Bts.Websocket.JsonRpcRequest<DataStreamSubscription>>(msg);
+         var rpcReq = JsonSerializer.Deserialize<JsonRpcRequest<DataStreamSubscription>>(msg);
          var queryArgs = rpcReq!.Params!;
          // var queryArgs = new DataStreamSubscription { Query = msg };
          var exp = DataStreamExpressionsInterpreter.Evaluate(queryArgs.Query!);

src/DataStreamSuite/DataStream.Console/ColumnMapTests.cs

@@ -1,34 +0,0 @@
-// namespace ColumnMapTests;
-
-// using Bts.ExpressionCore;
-// using Bts.ColumnMapExpressions;
-// using Bts.DataStreamExpressions;
-
-// using DataStream.Console;
-
-// public class TempColumnMapTests {
-//    public static void Run(ExpressionTest test) {
-//       Run(test.Expression, test.ExpectedResults);
-//    }
-
-//    public static void Run(string expression, IDictionary<string, string> expectedResults) {
-//       var interpreter = ColumnMapExpression.Parse(expression);
-//       var env = new Dictionary<string, object?>();
-//       var result = interpreter.Evaluate(env);
-//       foreach (var (key, expected) in expectedResults) {
-//          if (!result.TryGetValue(key, out var actual)) {
-//             throw new Exception($"ExpectedResults did not contain key '{key}'");
-//          }
-//          if (!expected.Equals(actual?.ToString() ?? string.Empty)) {
-//             throw new Exception($"Expected '{key}' to be {expected} (actually {actual})");
-//          }
-//          ConsoleHelper.WriteLine($"PASS: {key}={actual}={expected}", ConsoleColor.Green);
-//       }
-//    }
-
-//    static void assert(bool condition) {
-//       if (!condition) {
-//          throw new Exception();
-//       }
-//    }
-// }

src/DataStreamSuite/DataStream.Console/ConsoleHelper.cs

@@ -1,29 +0,0 @@
-namespace DataStream.Console;
-
-using Spark.Core.Disposables;
-
-using System;
-
-static class ConsoleHelper {
-   public static IDisposable PushBg(ConsoleColor value) {
-      var orig = Console.BackgroundColor;
-      Console.BackgroundColor = value;
-      return Disposable.Create(pop);
-      void pop() =>
-         Console.BackgroundColor = orig;
-   }
-
-   public static IDisposable PushFg(ConsoleColor value) {
-      var orig = Console.ForegroundColor;
-      Console.ForegroundColor = value;
-      return Disposable.Create(pop);
-      void pop() =>
-         Console.ForegroundColor = orig;
-   }
-   
-   public static void WriteLine(string text, ConsoleColor? fg = null, ConsoleColor? bg = null) {
-      using IDisposable? fgPush = fg is null ? null : PushFg(fg.Value);
-      using IDisposable? bgPush = bg is null ? null : PushBg(bg.Value);
-      Console.WriteLine(text);
-   }
-}

src/DataStreamSuite/DataStream.Console/DataStream.Console.csproj

@@ -1,15 +0,0 @@
-<Project Sdk="Microsoft.NET.Sdk">
-
-  <ItemGroup>
-    <ProjectReference Include="..\ArithmeticExpressions\ArithmeticExpressions.csproj" />
-    <ProjectReference Include="..\DataStreamExpressions\DataStreamExpressions.csproj" />
-  </ItemGroup>
-
-  <PropertyGroup>
-    <OutputType>Exe</OutputType>
-    <TargetFramework>net8.0</TargetFramework>
-    <ImplicitUsings>enable</ImplicitUsings>
-    <Nullable>enable</Nullable>
-  </PropertyGroup>
-
-</Project>

src/DataStreamSuite/DataStream.Console/ExpressionTest.cs

@@ -1,15 +0,0 @@
-namespace DataStream.Console;
-
-using Spark.Core;
-
-using System.Text;
-
-public class ExpressionTest {
-   public ExpressionTest(string expression, IDictionary<string, string> expectedResults) {
-      Expression = expression;
-      ExpectedResults = expectedResults;
-   }
-
-   public string Expression { get; }
-   public IDictionary<string, string> ExpectedResults { get; }
-}

src/DataStreamSuite/DataStream.Console/ExpressionTestParser.cs

@@ -1,25 +0,0 @@
-namespace DataStream.Console;
-
-using Spark.Core;
-
-using System.Text;
-
-public class ExpressionTestParser {
-   public static ExpressionTest Parse(IEnumerable<string> lines) {
-      var expressionLines = lines.SkipWhile(l => l != "BEGIN_TEST").Skip(1);
-      var expressionBd = new StringBuilder();
-      Dictionary<string, string>? expectedResults = null;
-      foreach (var l in expressionLines) {
-         const string END_SENTINEL = "END_TEST";
-         if (l.StartsWith(END_SENTINEL)) {
-            expectedResults = l[END_SENTINEL.Length..].Trim()
-               .Split(" ")
-               .Select(kv => kv.Split(":"))
-               .ToDictionary(kv => kv[0], kv => kv[1]);
-            break;
-         }
-         expressionBd.AppendLine(l);
-      }
-      return new(expressionBd.ToString(), expectedResults ?? new(0));
-   }
-}

src/DataStreamSuite/DataStream.Console/Program.cs

@@ -1,32 +0,0 @@
-// See https://aka.ms/new-console-template for more information
-using DataStream.Console;
-
-using ExpressionTests;
-
-using Spark.Core.Collections;
-
-using System.Reactive.Disposables;
-
-
-try {
-   foreach (var arg in args) {
-      if (File.Exists(arg)) {
-         using var fs = File.OpenRead(arg) ??
-            throw new Exception("File.OpenRead returned null. LIES!");
-         var expressionLines = new StringStreamEnumerable(new(fs));
-         var test = ExpressionTestParser.Parse(expressionLines);
-         TempExpressionTests.Run(test);
-      }
-   }
-   if (args.Length == 0) {
-      TempExpressionTests.Run("rv = 3 + 4", new Dictionary<string, string> { { "rv", "7" } });
-   }
-} catch (Exception ex) {
-   using (ConsoleHelper.PushBg(ConsoleColor.Red)) {
-      Console.WriteLine($"Failed! {ex}");
-   }
-   return;
-}
-
-using var _ = ConsoleHelper.PushFg(ConsoleColor.Green);
-Console.WriteLine($"Passed!");

src/DataStreamSuite/DataStream.Console/TempExpressionTests.cs

@@ -1,32 +0,0 @@
-namespace ExpressionTests;
-
-using Bts.ArithmeticExpressions;
-
-using DataStream.Console;
-
-public class TempExpressionTests {
-   public static void Run(ExpressionTest test) {
-      Run(test.Expression, test.ExpectedResults);
-   }
-
-   public static void Run(string expression, IDictionary<string, string> expectedResults) {
-      var prg = ArithmeticProgramParser.Parse(expression);
-      var env = new Dictionary<string, object?>();
-      var result = prg.Evaluate(env);
-      foreach (var (key, expected) in expectedResults) {
-         if (!result.TryGetValue(key, out var actual)) {
-            throw new Exception($"ExpectedResults did not contain key '{key}'");
-         }
-         if (!expected.Equals(actual?.ToString() ?? string.Empty)) {
-            throw new Exception($"Expected '{key}' to be {expected} (actually {actual})");
-         }
-         ConsoleHelper.WriteLine($"PASS: {key}={actual}={expected}", ConsoleColor.Green);
-      }
-   }
-
-   static void assert(bool condition) {
-      if (!condition) {
-         throw new Exception();
-      }
-   }
-}

src/DataStreamSuite/DataStream.Console/TestExpressions.txt

@@ -1,6 +0,0 @@
-BEGIN_TEST
-x=3
-y=4
-xySum= x+y
-constSum = 3 + 4
-END_TEST constSum:7 xySum:7

src/DataStreamSuite/DataStream/DataStreamExtensions.cs

@@ -0,0 +1,26 @@
+namespace Bts.DataStream;
+
+using BCG;
+
+using Spark.Core.Disposables;
+
+using System;
+using System.Reactive.Linq;
+
+public static class DataStreamExtensions {
+   public static ViewStream AsViewStream(this IObservable<DataPacket> source) =>
+      source as ViewStream ?? new ObservableViewStream(source);
+
+   public static IObservable<DataPacket> AsObservable(this ViewStream source) {
+      return source as IObservable<DataPacket> ?? Observable.Create<DataPacket>(Subscribe);
+
+      IDisposable Subscribe(IObserver<DataPacket> observer) {
+         source.onChange += OnChange;
+         return Disposable.Create(() => source.onChange -= OnChange);
+
+         void OnChange(object? newValue) =>
+            observer.OnNext(newValue as DataPacket ?? throw new ArgumentException(
+               $"Expected DataPacket, got {newValue?.GetType()}", nameof(newValue)));
+      }
+   }
+}

src/DataStreamSuite/DataStream/Misc/PandasStream.cs

@@ -0,0 +1,62 @@
+// using System;
+// using System.Diagnostics;
+
+// using BCG;
+// using BCG.Utilities.General;
+
+
+// namespace bts {
+
+//     public class PandasStream : ViewStream, IWorker {
+//         //C:\Users\Owner\AppData\Local\Microsoft\WindowsApps\python.exe
+
+//         public PandasStream(ViewStream vs, ViewStreamDatabase vsd, string pandas_code) {
+
+//         }
+
+//         public PandasStream(ViewStream vs, string pandas_code) :
+//         this(vs, ViewStreamDatabase.instance, pandas_code) {
+//         }
+
+
+//         protected override void myOnFirstSubscriber() {
+
+//             // string python_code = @"
+//             //    import winsound
+//             //    frequency = 2500  # Set frequency (in Hz)
+//             //    duration = 1000   # Set duration (in milliseconds)
+//             //    winsound.Beep(frequency, duration)
+//             // ";
+//             string python_code = @"
+//             file_path = \"C:\\Temp\\test2.txt\\"  # r prefix for raw string to avoid escape characters
+//             with open(file_path, \"w\") as f:
+//                 f.write(\"Hello, world!\n\")
+//             ";
+
+//             execute_python_code(python_code);
+//         }
+
+//         static void execute_python_code(string python_code) {
+//             ProcessStartInfo start = new ProcessStartInfo();
+//             start.FileName = "python"; // Python executable
+//             start.Arguments = "-c \"" + python_code.Replace("\"", "\\\"") + "\""; // Pass code as argument
+//             start.UseShellExecute = false;
+//             start.RedirectStandardOutput = true;
+
+//             using (Process process = Process.Start(start)) {
+//                 using (System.IO.StreamReader reader = process.StandardOutput) {
+//                     string result = reader.ReadToEnd();
+//                     Console.Write(result);
+//                 }
+//             }
+//         }
+
+//         protected override void myOnLastSubscriber() {
+//         }
+
+//         public void doWork() {
+
+//         }
+
+//     }
+// }

src/DataStreamSuite/DataStream/Misc/test.py

@@ -0,0 +1,3 @@
+file_path = r"C:\Temp\test2.txt"  # r prefix for raw string to avoid escape characters
+with open(file_path, "w") as f:
+    f.write("Hello, world!\n")

src/DataStreamSuite/DataStream/ObservableViewStream.cs

@@ -0,0 +1,34 @@
+namespace Bts.DataStream;
+
+using BCG;
+
+using bts;
+
+using System;
+
+public class ObservableViewStream : ViewStream, IObservable<DataPacket>, IDisposable {
+   readonly IObservable<DataPacket> source;
+   IDisposable? subscription;
+
+   public ObservableViewStream(IObservable<DataPacket> source) =>
+      this.source = source;
+
+   public void Dispose() {
+      subscription?.Dispose();
+      subscription = null;
+   }
+
+   protected override void myOnFirstSubscriber() {
+      Log.isnull(subscription);
+      subscription = source.Subscribe(myFireEvent);
+   }
+
+   protected override void myOnLastSubscriber() {
+      Log.notnull(subscription);
+      subscription.Dispose();
+      subscription = null;
+   }
+
+   IDisposable IObservable<DataPacket>.Subscribe(IObserver<DataPacket> observer) =>
+      source.Subscribe(observer);
+}

src/DataStreamSuite/DataStream/Primitives/AddColumns/AddConstantColumnRow.cs

@@ -0,0 +1,12 @@
+namespace bts {
+    using BCG;
+
+    using System;
+    using System.Collections;
+
+    public class AddConstantColumnRow {
+        AddConstantColumnRow(IBlueRow r, Tuple<string, Type, object>[] constant_cols) {
+
+        }
+    }
+}

src/DataStreamSuite/DataStream/Primitives/FileStreams/PersistorStream.cs

@@ -15,6 +15,11 @@ namespace BCG {
    // data:
    // epochtime, size, serialized data
    [DataStreamUserRespresentation("pers")]
+   // Saves each udpate of a stream into a file. The token is the the unique logical name of the stream,
+   // which maps the name of the directory where the streams are saved. The directory (3rd arg) is the 
+   // parent directory where the token folder is stored. Each file goes up to rolloevr_size after which a new
+   // file is created. The file names are token_epoch_datetime
+   // the first entry in a file is the complete state
    public class PersistorStream : ViewStream {
       private ViewStream _source_stream;
       private string _token;

src/DataStreamSuite/DataStream/Primitives/FileStreams/SnapshotStream.cs

@@ -11,13 +11,27 @@ using System.Reflection.Metadata;
 
 namespace BCG {
    public class SnapshotStream : ViewStream {
-
+      public enum SpecificSnapshot {
+         FirstOfDay,
+         LastOfDay
+      };
       private BlueTable _current_table;
       private IClientDeserializer _deserilializer = new ClientUniversalDeserializer();
       public DateTime _snapshot_time;
       public string _snapshot_file;
       public IEnumerable<PlaybackPacket> _packets;
-
+      public SnapshotStream(string snapshot_date, string specific_snapshot, string directory_name, string token) {
+         DateTime dt;
+         var parsed = DateTime.TryParse(snapshot_date, out dt);
+         dt = dt.Date;
+         bts.Log.assert(parsed);
+         SpecificSnapshot specific_snapshot_enum;
+         parsed = Enum.TryParse<SpecificSnapshot>(specific_snapshot, out specific_snapshot_enum);
+         bts.Log.assert(parsed);
+         string snapshot_dt = StreamPersistorUtils.find_snapshot_file($"{directory_name}\\{token}", dt, specific_snapshot_enum);
+         ViewStream rs = new SnapshotStream(snapshot_dt, directory_name, token);
+         mySetRealSource(rs);
+      }
 
       public SnapshotStream(string snapshot_dt, string directory_name, string token) {
          _snapshot_time = PersistorStream.parse_datetime(snapshot_dt);

src/DataStreamSuite/DataStream/Primitives/FileStreams/StreamPersistorUtils.cs

@@ -37,6 +37,20 @@ namespace BCG {
       public IEnumerable<PlaybackPacket> packets = new List<PlaybackPacket>();
    }
    public static class StreamPersistorUtils {
+      public static string find_snapshot_file(string directory_name, DateTime snapshot_date, SnapshotStream.SpecificSnapshot specific_snapshot) {
+         DateTime max_dt = DateTime.MinValue;
+         string rv = String.Empty;
+         string[] file_names = Directory.GetFiles(directory_name);
+         for (int i = 0; i < file_names.Count() - 1; i++) {
+            string fn = file_names[i];
+            var dt = parse_datetime_from_filename(fn);
+            if (dt > max_dt) {
+               max_dt = dt;
+               rv = fn;
+            }
+         }
+         return rv;
+      }
       public static string find_snapshot_file(string directory_name, DateTime snapshot_time) {
          string[] file_names = Directory.GetFiles(directory_name);
          for (int i = 0; i < file_names.Count() - 1; i++) {

src/DataStreamSuite/DataStream/Primitives/GroupBy/MaxGroupByEvaluator.cs

@@ -1,47 +1,42 @@
-// using System;
-// using System.Collections;
-// using BCG.Utilities;
+using System;
+using System.Collections;
 
-// namespace BCG
-// {
-// 	/// <summary>
-// 	/// Computes the max for a given column
-// 	/// </summary>
-// 	public class MaxGroupByEvaluator : GroupByEvaluator
-// 	{
+using BCG.Utilities;
 
-//       public MaxGroupByEvaluator (string outputCol, string[] groupByCols) : base (outputCol, groupByCols)
-//       {}
+namespace BCG {
+    /// <summary>
+    /// Computes the max for a given column
+    /// </summary>
+    public class MaxGroupByEvaluator : GroupByEvaluator {
 
-//       public MaxGroupByEvaluator (string colMapEpression, string outputCol, string[] groupByCols)
-//          : base (colMapEpression, outputCol, groupByCols)   {}
+        public MaxGroupByEvaluator(string outputCol, string[] groupByCols) : base(outputCol, groupByCols) { }
 
-//       public override IBlueRow evaluate (DataPacket dp)  
-//       {
-//          myOutputTable = myOutputTable == null ? new GroupByOutputTable (dp.schema) :
-//             myOutputTable;
-//          string hashString = BlueDataUtility.rowToString (GroupByStream.getRepRow (dp), myGroupByCols);
-//          IBlueRow currMaxRow = myOutputTable.getRow (hashString);
-//          IBlueRow maxRow = BlueDataUtility.computeMaxRow (dp.deletedRows, myOutputColumn);
-//          if (maxRow != null)
-//          {
-//             BlueEventLog.logAssert (currMaxRow != null);
-//             BlueEventLog.logAssert (BlueDataUtility.compareRows (maxRow, currMaxRow, myOutputColumn) <= 0);
-//             if (BlueDataUtility.compareRows (maxRow, currMaxRow, myOutputColumn) == 0)
-//             {
-//                maxRow = BlueDataUtility.computeMaxRow (dp.newRows, myOutputColumn);
-//                if (maxRow == null)
-//                   myOutputTable.removeRow (hashString);
-//                else
-//                   myOutputTable.addRow (maxRow, hashString);
-//             }
-//          }
-//          maxRow = BlueDataUtility.computeMaxRow (dp.addedRows, myOutputColumn);
-//          // if the row was replaced in the above lines, then currMaxRow < maxRow, so the if below would never return a true,
-//          // otherwise if maxRow > currMaxRow then its replaced
-//          if (maxRow != null && (currMaxRow == null || BlueDataUtility.compareRows (maxRow, currMaxRow, myOutputColumn) > 0))
-//             myOutputTable.addRow (maxRow, hashString);
-//          return myProject (myOutputTable.getRow (hashString));
-//       }
-// 	}
-// }
+        public MaxGroupByEvaluator(string colMapEpression, string outputCol, string[] groupByCols)
+           : base(colMapEpression, outputCol, groupByCols) { }
+
+        public override IBlueRow evaluate(DataPacket dp) {
+            myOutputTable = myOutputTable == null ? new GroupByOutputTable(dp.schema) :
+               myOutputTable;
+            string hashString = BlueDataUtility.rowToString(GroupByStream.getRepRow(dp), myGroupByCols);
+            IBlueRow currMaxRow = myOutputTable.getRow(hashString);
+            IBlueRow maxRow = BlueDataUtility.computeMaxRow(dp.deletedRows, myOutputColumn);
+            if (maxRow != null) {
+                bts.Log.assert(currMaxRow != null);
+                bts.Log.assert(BlueDataUtility.compareRows(maxRow, currMaxRow, myOutputColumn) <= 0);
+                if (BlueDataUtility.compareRows(maxRow, currMaxRow, myOutputColumn) == 0) {
+                    maxRow = BlueDataUtility.computeMaxRow(dp.newRows, myOutputColumn);
+                    if (maxRow == null)
+                        myOutputTable.removeRow(hashString);
+                    else
+                        myOutputTable.addRow(maxRow, hashString);
+                }
+            }
+            maxRow = BlueDataUtility.computeMaxRow(dp.addedRows, myOutputColumn);
+            // if the row was replaced in the above lines, then currMaxRow < maxRow, so the if below would never return a true,
+            // otherwise if maxRow > currMaxRow then its replaced
+            if (maxRow != null && (currMaxRow == null || BlueDataUtility.compareRows(maxRow, currMaxRow, myOutputColumn) > 0))
+                myOutputTable.addRow(maxRow, hashString);
+            return myProject(myOutputTable.getRow(hashString));
+        }
+    }
+}

src/DataStreamSuite/DataStream/Primitives/GroupBy/MaxStream.cs

@@ -1,20 +1,15 @@
-// using System;
+using System;
 
-// namespace BCG
-// {
-// 	public class MaxStream : ViewStream
-// 	{
-//       public MaxStream(ViewStream stream, string outputCol, string inputCols) : this(stream, outputCol, inputCols.Split(' '))
-//       {
-//       }
-// 		public MaxStream (ViewStream stream, string outputCol, string[] inputCols)
-// 		{
-//          mySetRealSource (new GroupByStream (stream, 
-//                                              new MaxGroupByEvaluator (outputCol, inputCols)));
-// 		}
+namespace BCG {
+    public class MaxStream : ViewStream {
+        public MaxStream(ViewStream stream, string outputCol, string inputCols) : this(stream, outputCol, inputCols.Split(' ')) {
+        }
+        public MaxStream(ViewStream stream, string outputCol, string[] inputCols) {
+            mySetRealSource(new GroupByStream(stream,
+                                                new MaxGroupByEvaluator(outputCol, inputCols)));
+        }
 
-//       public MaxStream (ViewStream stream, string outputCol)
-//          : this (new AddColumnsStream (stream, "fakeCol=0"), outputCol, new string[]{"fakeCol"})
-//       {}
-// 	}
-// }
+        public MaxStream(ViewStream stream, string outputCol)
+           : this(new AddColumnsStream(stream, "fakeCol=0"), outputCol, new string[] { "fakeCol" }) { }
+    }
+}

src/DataStreamSuite/DataStream/Primitives/GroupBy/MinGroupByEvaluator.cs

@@ -1,40 +1,36 @@
-// using System;
-// using BCG.Utilities;
+using System;
 
-// namespace BCG
-// {
-//    public class MinGroupByEvaluator : MaxGroupByEvaluator
-//    {
-//       public MinGroupByEvaluator (string outputCol, string[] groupByCols) 
-//          : base (outputCol, groupByCols)  {}
+using BCG.Utilities;
 
-//       public MinGroupByEvaluator (string colMapEpression, string outputCol, string[] groupByCols) :
-//          base (colMapEpression, outputCol, groupByCols)  {}
+namespace BCG {
+    public class MinGroupByEvaluator : MaxGroupByEvaluator {
+        public MinGroupByEvaluator(string outputCol, string[] groupByCols)
+           : base(outputCol, groupByCols) { }
 
-//       public override IBlueRow evaluate (DataPacket dp)  
-//       {
-//          myOutputTable = myOutputTable == null ? new GroupByOutputTable (dp.schema) : myOutputTable;
-//          string stringRep = BlueDataUtility.rowToString (GroupByStream.getRepRow (dp), myGroupByCols);
-//          IBlueRow currMinRow = myOutputTable.getRow (stringRep);
-//          IBlueRow minRow = BlueDataUtility.computeMinRow (dp.deletedRows, myOutputColumn);
-//          if (minRow != null)
-//          {
-//             // BlueEventLog.logAssert (currMinRow != null);
-//             // BlueEventLog.logAssert (BlueDataUtility.compareRows (minRow, currMinRow, myOutputColumn) >= 0);
-//             if (BlueDataUtility.compareRows (minRow, currMinRow, myOutputColumn) == 0)
-//             {
-//                minRow = BlueDataUtility.computeMinRow (dp.newRows, myOutputColumn);
-//                if (minRow == null)
-//                   myOutputTable.removeRow (stringRep);
-//                else
-//                   myOutputTable.addRow (minRow, stringRep);
-//             }
-//          }
-//          minRow = BlueDataUtility.computeMinRow (dp.addedRows, myOutputColumn);
-//          if (minRow != null && (currMinRow == null || BlueDataUtility.compareRows (minRow, currMinRow, myOutputColumn) < 0))
-//             myOutputTable.addRow (minRow, stringRep);
-//          return myOutputTable.getRow (stringRep);
-//       }
+        public MinGroupByEvaluator(string colMapEpression, string outputCol, string[] groupByCols) :
+           base(colMapEpression, outputCol, groupByCols) { }
 
-//    }
-// }
+        public override IBlueRow evaluate(DataPacket dp) {
+            myOutputTable = myOutputTable == null ? new GroupByOutputTable(dp.schema) : myOutputTable;
+            string stringRep = BlueDataUtility.rowToString(GroupByStream.getRepRow(dp), myGroupByCols);
+            IBlueRow currMinRow = myOutputTable.getRow(stringRep);
+            IBlueRow minRow = BlueDataUtility.computeMinRow(dp.deletedRows, myOutputColumn);
+            if (minRow != null) {
+                // BlueEventLog.logAssert (currMinRow != null);
+                // BlueEventLog.logAssert (BlueDataUtility.compareRows (minRow, currMinRow, myOutputColumn) >= 0);
+                if (BlueDataUtility.compareRows(minRow, currMinRow, myOutputColumn) == 0) {
+                    minRow = BlueDataUtility.computeMinRow(dp.newRows, myOutputColumn);
+                    if (minRow == null)
+                        myOutputTable.removeRow(stringRep);
+                    else
+                        myOutputTable.addRow(minRow, stringRep);
+                }
+            }
+            minRow = BlueDataUtility.computeMinRow(dp.addedRows, myOutputColumn);
+            if (minRow != null && (currMinRow == null || BlueDataUtility.compareRows(minRow, currMinRow, myOutputColumn) < 0))
+                myOutputTable.addRow(minRow, stringRep);
+            return myOutputTable.getRow(stringRep);
+        }
+
+    }
+}

src/DataStreamSuite/DataStream/Primitives/GroupBy/MinStream.cs

@@ -1,17 +1,13 @@
-// using System;
+using System;
 
-// namespace BCG
-// {
-//    public class MinStream : ViewStream
-//    {
-//       public MinStream (ViewStream stream, string outputCol, string[] inputCol)
-//       {
-//          mySetRealSource (new GroupByStream (stream, 
-//                            new MinGroupByEvaluator (outputCol, inputCol)));
-//       }
+namespace BCG {
+    public class MinStream : ViewStream {
+        public MinStream(ViewStream stream, string outputCol, string[] inputCol) {
+            mySetRealSource(new GroupByStream(stream,
+                              new MinGroupByEvaluator(outputCol, inputCol)));
+        }
 
-//       public MinStream (ViewStream stream, string outputCol)
-//                      : this (new AddColumnsStream (stream, "fakeCol=0"), outputCol, new string[]{"fakeCol"})
-//       {}
-//    }
-// }
+        public MinStream(ViewStream stream, string outputCol)
+                       : this(new AddColumnsStream(stream, "fakeCol=0"), outputCol, new string[] { "fakeCol" }) { }
+    }
+}

src/DataStreamSuite/DataStream/Primitives/GroupBy/SumGroupByEvaluator.cs

@@ -1,94 +1,82 @@
-// using System;
+using System;
 
-// namespace BCG
-// {
-//    /// <summary>
-//    /// Computes the sum for a given column
-//    /// </summary>
-//    public class SumGroupByEvaluator : GroupByEvaluator
-//    {
-//       public SumGroupByEvaluator (string outputCol, string[] groupByCols) : base (outputCol, groupByCols)
-//       {}
+namespace BCG {
+    /// <summary>
+    /// Computes the sum for a given column
+    /// </summary>
+    public class SumGroupByEvaluator : GroupByEvaluator {
+        public SumGroupByEvaluator(string outputCol, string[] groupByCols) : base(outputCol, groupByCols) { }
 
-//       public SumGroupByEvaluator (string colMapEpression, string outputCol, string[] groupByCols)
-//          : base (colMapEpression, outputCol, groupByCols)   {}
+        public SumGroupByEvaluator(string colMapEpression, string outputCol, string[] groupByCols)
+           : base(colMapEpression, outputCol, groupByCols) { }
 
-//       public override IBlueRow evaluate (DataPacket dp)  
-//       {
-//          myOutputTable = myOutputTable == null ? new GroupByOutputTable (dp.schema) : myOutputTable;
-         
-//          IBlueRow sumRow = BlueDataUtility.computeSum(dp.newRows, myOutputColumn);
-//          return sumRow;
-//       }
-//    }
+        public override IBlueRow evaluate(DataPacket dp) {
+            myOutputTable = myOutputTable == null ? new GroupByOutputTable(dp.schema) : myOutputTable;
+
+            IBlueRow sumRow = BlueDataUtility.computeSum(dp.newRows, myOutputColumn);
+            return sumRow;
+        }
+    }
 
 
-//    public class MultiColumnSumGroupByEvaluator : IGroupByEvaluator
-//    {
-//       private string[] myOutputCols, myGroupByCols;
-//       public MultiColumnSumGroupByEvaluator(string spaceSeperatedOutputCols, string spaceSeperatedGroupByCols)
-//       {
-//          myOutputCols = spaceSeperatedOutputCols.Split(' ');
-//          myGroupByCols = spaceSeperatedGroupByCols.Split(' ');
-//       }
+    public class MultiColumnSumGroupByEvaluator : IGroupByEvaluator {
+        private string[] myOutputCols, myGroupByCols;
+        public MultiColumnSumGroupByEvaluator(string spaceSeperatedOutputCols, string spaceSeperatedGroupByCols) {
+            myOutputCols = spaceSeperatedOutputCols.Split(' ');
+            myGroupByCols = spaceSeperatedGroupByCols.Split(' ');
+        }
 
-//       public MultiColumnSumGroupByEvaluator(string[] output_columns, string[] groupby_columns)
-//       {
-//          myOutputCols = output_columns;
-//          myGroupByCols = groupby_columns;
-//       }
+        public MultiColumnSumGroupByEvaluator(string[] output_columns, string[] groupby_columns) {
+            myOutputCols = output_columns;
+            myGroupByCols = groupby_columns;
+        }
 
-//       #region IGroupByEvaluator Members
+        #region IGroupByEvaluator Members
 
-//       public IBlueRow evaluate(DataPacket packet)  {return BlueDataUtility.computeSum(packet.newRows, myOutputCols);}
+        public IBlueRow evaluate(DataPacket packet) { return BlueDataUtility.computeSum(packet.newRows, myOutputCols); }
 
-//       public string[] groupByColumns { get { return myGroupByCols; } }
+        public string[] groupByColumns { get { return myGroupByCols; } }
 
-//       public void clear()  {/*stateless*/}
+        public void clear() {/*stateless*/}
 
-//       public IBlueTableSchema getOutputSchema(IBlueTableSchema schema)  {return schema;}
+        public IBlueTableSchema getOutputSchema(IBlueTableSchema schema) { return schema; }
 
-//       #endregion
-//    }
+        #endregion
+    }
 
-//    public class SumGroupByEvaluator<T> : GroupByEvaluator
-//    {
-//       public SumGroupByEvaluator(string outputCol, string[] groupByCols)
-//          : base(outputCol, groupByCols)
-//       { }
+    public class SumGroupByEvaluator<T> : GroupByEvaluator {
+        public SumGroupByEvaluator(string outputCol, string[] groupByCols)
+           : base(outputCol, groupByCols) { }
 
-//       public SumGroupByEvaluator(string colMapEpression, string outputCol, string[] groupByCols)
-//          : base(colMapEpression, outputCol, groupByCols) { }
+        public SumGroupByEvaluator(string colMapEpression, string outputCol, string[] groupByCols)
+           : base(colMapEpression, outputCol, groupByCols) { }
 
-//       public override IBlueRow evaluate(DataPacket dp)
-//       {
-//          myOutputTable = myOutputTable == null ? new GroupByOutputTable(dp.schema) : myOutputTable;
+        public override IBlueRow evaluate(DataPacket dp) {
+            myOutputTable = myOutputTable == null ? new GroupByOutputTable(dp.schema) : myOutputTable;
 
-//          IBlueRow sumRow = BlueDataUtility.computeSum<T>(dp.newRows, myOutputColumn);
-//          return sumRow;
-//       }
-//    }
+            IBlueRow sumRow = BlueDataUtility.computeSum<T>(dp.newRows, myOutputColumn);
+            return sumRow;
+        }
+    }
 
-//    public class MultiColumnSumGroupByEvaluator<T> : IGroupByEvaluator
-//    {
-//       private string[] myOutputCols, myGroupByCols;
-//       public MultiColumnSumGroupByEvaluator(string spaceSeperatedOutputCols, string spaceSeperatedGroupByCols)
-//       {
-//          myOutputCols = spaceSeperatedOutputCols.Split(' ');
-//          myGroupByCols = spaceSeperatedGroupByCols.Split(' ');
-//       }
+    public class MultiColumnSumGroupByEvaluator<T> : IGroupByEvaluator {
+        private string[] myOutputCols, myGroupByCols;
+        public MultiColumnSumGroupByEvaluator(string spaceSeperatedOutputCols, string spaceSeperatedGroupByCols) {
+            myOutputCols = spaceSeperatedOutputCols.Split(' ');
+            myGroupByCols = spaceSeperatedGroupByCols.Split(' ');
+        }
 
-//       #region IGroupByEvaluator Members
+        #region IGroupByEvaluator Members
 
-//       public IBlueRow evaluate(DataPacket packet)  {return BlueDataUtility.computeSum<T>(packet.newRows, myOutputCols);}
+        public IBlueRow evaluate(DataPacket packet) { return BlueDataUtility.computeSum<T>(packet.newRows, myOutputCols); }
 
-//       public string[] groupByColumns { get { return myGroupByCols; } }
+        public string[] groupByColumns { get { return myGroupByCols; } }
 
-//       public void clear()  {/*stateless*/}
+        public void clear() {/*stateless*/}
 
-//       public IBlueTableSchema getOutputSchema(IBlueTableSchema schema)  {return schema;}
+        public IBlueTableSchema getOutputSchema(IBlueTableSchema schema) { return schema; }
 
-//       #endregion
-//    }
+        #endregion
+    }
 
-// }
+}

src/DataStreamSuite/DataStream/Primitives/GroupBy/SumStream.cs

@@ -1,107 +1,89 @@
-// using System;
-// using System.Collections.Generic;
+using System;
+using System.Collections.Generic;
 
-// namespace BCG
-// {
-//    public class NullMapper : IStreamMapper
-//    {
-//       private Dictionary<string, bool> _valid_cols = new Dictionary<string, bool>();
+namespace BCG {
+    public class NullMapper : IStreamMapper {
+        private Dictionary<string, bool> _valid_cols = new Dictionary<string, bool>();
 
-//       public NullMapper(string[] valid_cols)
-//       {
-//          foreach (string c in valid_cols)
-//             _valid_cols[c] = true;
-//          _valid_cols["id"] = true;
-//       }
+        public NullMapper(string[] valid_cols) {
+            foreach (string c in valid_cols)
+                _valid_cols[c] = true;
+            _valid_cols["id"] = true;
+        }
 
-//       public IBlueTableSchema getOutputSchema(IBlueTableSchema input_schema) { return input_schema; }
+        public IBlueTableSchema getOutputSchema(IBlueTableSchema input_schema) { return input_schema; }
 
-//       public IBlueRow map(IBlueRow row)
-//       {
-//          MutableBlueRow out_row = new MutableBlueRow(row);
-//          foreach (string c in row.schema.columnNames)
-//             if (!_valid_cols.ContainsKey(c))
-//                out_row.setValue(c, DBNull.Value);
-//          return out_row;
-//       }
-//    }
+        public IBlueRow map(IBlueRow row) {
+            MutableBlueRow out_row = new MutableBlueRow(row);
+            foreach (string c in row.schema.columnNames)
+                if (!_valid_cols.ContainsKey(c))
+                    out_row.setValue(c, DBNull.Value);
+            return out_row;
+        }
+    }
 
-//    public class SumStream : ViewStream
-//    {
-//       public SumStream(ViewStream stream, string spaceSeparatedOutputCols, string spaceSeparatedGroupByCols)
-//       {
-//          string[] outputCols = spaceSeparatedOutputCols.Split(' ');
-//          string[] groupByCols = spaceSeparatedGroupByCols.Split(' ');
-//          mySetRealSource(new GroupByStream(stream,
-//                                            outputCols.Length == 1 ? new SumGroupByEvaluator(outputCols[0], groupByCols) as IGroupByEvaluator :
-//                                            new MultiColumnSumGroupByEvaluator(spaceSeparatedOutputCols, spaceSeparatedGroupByCols)));
-//       }
+    public class SumStream : ViewStream {
+        public SumStream(ViewStream stream, string spaceSeparatedOutputCols, string spaceSeparatedGroupByCols) {
+            string[] outputCols = spaceSeparatedOutputCols.Split(' ');
+            string[] groupByCols = spaceSeparatedGroupByCols.Split(' ');
+            mySetRealSource(new GroupByStream(stream,
+                                              outputCols.Length == 1 ? new SumGroupByEvaluator(outputCols[0], groupByCols) as IGroupByEvaluator :
+                                              new MultiColumnSumGroupByEvaluator(spaceSeparatedOutputCols, spaceSeparatedGroupByCols)));
+        }
 
-//       public SumStream(ViewStream stream, string[] output_columns)
-//       {
-//          ViewStream rs = new AddColumnsStream(stream, "fake_col=0");
-//          rs = new SumStream(rs, output_columns, new string[] { "fake_col" }, false);
-//          rs = new ProjectStream(rs, "!fake_col");
-//          mySetRealSource(rs);
-//       }
+        public SumStream(ViewStream stream, string[] output_columns) {
+            ViewStream rs = new AddColumnsStream(stream, "fake_col=0");
+            rs = new SumStream(rs, output_columns, new string[] { "fake_col" }, false);
+            rs = new ProjectStream(rs, "!fake_col");
+            mySetRealSource(rs);
+        }
 
-//       public SumStream(ViewStream stream, string space_seperated_output_columns, string space_seperated_groupby_columns, bool force_null_into_other_columns)
-//       {
-//          string[] output_columns = space_seperated_output_columns.Split(' ');
-//          string[] groupby_columns = String.IsNullOrEmpty (space_seperated_groupby_columns) ? new string[]{} : space_seperated_groupby_columns.Split (' ');
-//          mySetRealSource (new SumStream (stream, output_columns, groupby_columns, force_null_into_other_columns));
-//       }
+        public SumStream(ViewStream stream, string space_seperated_output_columns, string space_seperated_groupby_columns, bool force_null_into_other_columns) {
+            string[] output_columns = space_seperated_output_columns.Split(' ');
+            string[] groupby_columns = String.IsNullOrEmpty(space_seperated_groupby_columns) ? new string[] { } : space_seperated_groupby_columns.Split(' ');
+            mySetRealSource(new SumStream(stream, output_columns, groupby_columns, force_null_into_other_columns));
+        }
 
-//       public SumStream(ViewStream stream, string[] output_columns, string[] groupby_columns, bool force_null_into_other_columns)
-//       {
-//          ViewStream rs = null;
-//          groupby_columns = groupby_columns ?? new string[] {};
-//          if (groupby_columns.Length == 0)
-//          {
-//             rs = new AddColumnsStream(stream, "fake_col=0");
-//             rs = new GroupByStream(rs, new MultiColumnSumGroupByEvaluator(output_columns, new string[] {"fake_col"}));
-//             rs = new ProjectStream(rs, "!fake_col");
-//          }
-//          else
-//          {
-//             rs = new GroupByStream(stream, new MultiColumnSumGroupByEvaluator(output_columns, groupby_columns));
-//          }
-//          if (force_null_into_other_columns)
-//          {
-//             List<string> valid_cols = new List<string>(output_columns);
-//             valid_cols.AddRange(groupby_columns);
-//             rs = new MapStream(rs, new NullMapper(valid_cols.ToArray()));
-//          }
-//          mySetRealSource(rs);
-//       }
+        public SumStream(ViewStream stream, string[] output_columns, string[] groupby_columns, bool force_null_into_other_columns) {
+            ViewStream rs = null;
+            groupby_columns = groupby_columns ?? new string[] { };
+            if (groupby_columns.Length == 0) {
+                // rs = new AddColumnsStream(stream, "fake_col=0");
+                // rs = new GroupByStream(rs, new MultiColumnSumGroupByEvaluator(output_columns, new string[] { "fake_col" }));
+                // rs = new ProjectStream(rs, "!fake_col");
+            } else {
+                rs = new GroupByStream(stream, new MultiColumnSumGroupByEvaluator(output_columns, groupby_columns));
+            }
+            if (force_null_into_other_columns) {
+                List<string> valid_cols = new List<string>(output_columns);
+                valid_cols.AddRange(groupby_columns);
+                rs = new MapStream(rs, new NullMapper(valid_cols.ToArray()));
+            }
+            mySetRealSource(rs);
+        }
 
-//    public SumStream (ViewStream stream, string outputCol)
-//    : this (new AddColumnsStream (stream, "fakeCol=0"), outputCol, "fakeCol")
-//    {
-//    }
-// }
+        public SumStream(ViewStream stream, string outputCol)
+        : this(new AddColumnsStream(stream, "fakeCol=0"), outputCol, "fakeCol") {
+        }
+    }
 
-// public class IntSumStream : ViewStream
-// {
-//    public IntSumStream (ViewStream stream, string spaceSeparatedOutputCols, string spaceSeparatedGroupByCols)
-//    {
-//       string[] outputCols = spaceSeparatedOutputCols.Split(' ');
-//       string[] groupByCols = spaceSeparatedGroupByCols.Split(' ');
-//       mySetRealSource(new GroupByStream(stream,
-//                                         outputCols.Length == 1 ? new SumGroupByEvaluator<int>(outputCols[0], groupByCols) as IGroupByEvaluator:
-//                                         new MultiColumnSumGroupByEvaluator<int>(spaceSeparatedOutputCols, spaceSeparatedGroupByCols)));
-//    }
+    public class IntSumStream : ViewStream {
+        public IntSumStream(ViewStream stream, string spaceSeparatedOutputCols, string spaceSeparatedGroupByCols) {
+            string[] outputCols = spaceSeparatedOutputCols.Split(' ');
+            string[] groupByCols = spaceSeparatedGroupByCols.Split(' ');
+            mySetRealSource(new GroupByStream(stream,
+                                              outputCols.Length == 1 ? new SumGroupByEvaluator<int>(outputCols[0], groupByCols) as IGroupByEvaluator :
+                                              new MultiColumnSumGroupByEvaluator<int>(spaceSeparatedOutputCols, spaceSeparatedGroupByCols)));
+        }
 
-// }
+    }
 
 
-//    public class SumStream<T> : ViewStream
-//    {
-//       public SumStream(ViewStream stream, string outputCol, string spaceSeparatedInputCols)
-//       {
-//          string[] inputCols = spaceSeparatedInputCols.Split(' ');
-//          mySetRealSource(new GroupByStream(stream,
-//                                              new SumGroupByEvaluator<T>(outputCol, inputCols)));
-//       }
-//    }
-// }
+    public class SumStream<T> : ViewStream {
+        public SumStream(ViewStream stream, string outputCol, string spaceSeparatedInputCols) {
+            string[] inputCols = spaceSeparatedInputCols.Split(' ');
+            mySetRealSource(new GroupByStream(stream,
+                                                new SumGroupByEvaluator<T>(outputCol, inputCols)));
+        }
+    }
+}

src/DataStreamSuite/DataStream/Primitives/InterRow/RowDiffStream.cs

@@ -0,0 +1,79 @@
+using BCG;
+
+using System.Collections;
+using System.Collections.Generic;
+using System.Linq;
+
+using ZstdSharp.Unsafe;
+
+namespace bts {
+    public class RowDiffStream : ViewStream {
+        private ViewStream _vs;
+        private readonly string _order_col, _col_1, _col_2;
+        private readonly int _diff;
+        private IBlueTableSchema _out_schema;
+        private readonly string _diff_col_name;
+        public RowDiffStream(ViewStream vs, string order_col, string col_1, string col_2, int diff, string diff_col_name) {
+            _vs = vs;
+            _order_col = order_col;
+            _col_1 = col_1;
+            _col_2 = col_2;
+            _diff = diff;
+            _diff_col_name = diff_col_name;
+        }
+
+        private string _diff_col_name_perc {
+            get {
+                return $"{_diff_col_name}_percentage";
+            }
+        }
+
+        protected override void myOnFirstSubscriber() {
+            _vs.onChange += _handle_vs;
+        }
+
+        protected override void myOnLastSubscriber() {
+            _vs.onChange -= _handle_vs;
+        }
+
+        private void _handle_vs(object dp_obj) {
+            var dp = dp_obj as DataPacket;
+            IBlueRowCollection nrows = dp.newRows;
+            _out_schema = BlueDataUtility.add_column(nrows.schema, _diff_col_name, typeof(double));
+            _out_schema = BlueDataUtility.add_column(_out_schema, _diff_col_name_perc, typeof(double));
+            List<IBlueRow> row_list = new List<IBlueRow>();
+            foreach (IBlueRow r in nrows)
+                row_list.Add(r);
+            //OrderBy sorts in ascedning order
+            var sorted_rows = row_list.OrderBy(r => r[_order_col]).ToList();
+            var sorted_row_array = sorted_rows.ToArray();
+            // reverse it to make it descending    
+            sorted_rows.Reverse();
+            //sort rows so we can do the diff
+            OrderedBlueTable rv = new OrderedBlueTable(_out_schema);
+            int i = 0;
+            foreach (IBlueRow r in sorted_rows) {
+                MutableBlueRow nr = new MutableBlueRow(_out_schema, r);
+                //So, in the expression "x - y," where y is subtracted from x, x is the minuend, and y is the subtrahend.
+                // we assume x - y, and name variables like that
+                double x = r.get<double>(_col_1);
+                int y_row_idx = i + _diff;
+                double diff = System.Double.NaN;
+                double perc = System.Double.NaN;
+                if (y_row_idx < sorted_rows.Count()) {
+                    var y = sorted_rows[y_row_idx].get<double>(_col_2);
+                    diff = x - y;
+                    perc = 100.0 * diff / y;
+                }
+                nr.setValue(_diff_col_name, diff);
+                nr.setValue(_diff_col_name_perc, perc);
+                rv.addRow(nr);
+                i++;
+            }
+
+            var dp_out = BlueDataUtility.build_dp_inc(new_rows: rv, myLastValue as DataPacket, true);
+            myFireEvent(dp_out);
+
+        }
+    }
+}

src/DataStreamSuite/DataStream/Primitives/MongoDB/MongoCollectionStream.cs

@@ -0,0 +1,203 @@
+namespace Bts.DataStream;
+
+using BCG;
+
+using bts;
+
+using Bts.DataStreamCore;
+
+using mbts;
+
+using MongoDB.Bson;
+using MongoDB.Driver;
+
+using Spark.Core.Disposables;
+
+using System;
+using System.Collections.Generic;
+using System.Diagnostics.CodeAnalysis;
+using System.Reactive.Subjects;
+using System.Threading;
+using System.Threading.Tasks;
+
+public class MongoCollectionStream : ViewStream, IObservable<DataPacket> {
+   readonly string collectionName;
+   readonly string databaseName;
+   readonly string url;
+   readonly object syncRoot = new();
+   readonly CancellationTokenSource cts = new();
+   readonly Subject<DataPacket> subject = new();
+
+   bool hasStarted = false;
+   IBlueRowFactory<BsonDocument>? rowFactory;
+   MongoClient? clientBacking;
+   IMongoDatabase? databaseBacking;
+   IMongoCollection<BsonDocument>? collectionBacking = null;
+   BlueTableImmutable? table = null;
+
+   public MongoCollectionStream()
+      : this(
+         Environment.GetEnvironmentVariable("MONGO_DB_COLL") ?? string.Empty,
+         Environment.GetEnvironmentVariable("MONGO_DB_NAME") ?? string.Empty,
+         Environment.GetEnvironmentVariable("MONGO_DB_URL") ?? string.Empty) {
+   }
+
+   public MongoCollectionStream(string collectionName)
+      : this(
+         collectionName,
+         Environment.GetEnvironmentVariable("MONGO_DB_NAME") ?? string.Empty,
+         Environment.GetEnvironmentVariable("MONGO_DB_URL") ?? string.Empty) {
+   }
+
+   public MongoCollectionStream(string collectionName, string databaseName)
+      : this(
+         collectionName,
+         databaseName,
+         Environment.GetEnvironmentVariable("MONGO_DB_URL") ?? string.Empty) {
+   }
+
+   public MongoCollectionStream(string collectionName, string databaseName, string url)
+      : this(collectionName, databaseName, url, null) { }
+
+   internal MongoCollectionStream(
+      string collectionName, string databaseName, string url, IBlueRowFactory<BsonDocument>? rowFactory) {
+      this.collectionName = collectionName;
+      this.databaseName = databaseName;
+      this.url = url;
+      this.rowFactory = rowFactory;
+      mySetRealSource(new ObservableViewStream(this));
+   }
+
+   MongoClient client => clientBacking ??= new(url);
+   IMongoDatabase database => databaseBacking ??= client.GetDatabase(databaseName);
+   IMongoCollection<BsonDocument>? collection {
+      get => Volatile.Read(ref collectionBacking);
+      set => Volatile.Write(ref collectionBacking, value);
+   }
+
+   public IDisposable Subscribe(IObserver<DataPacket> observer) {
+      lock (syncRoot) {
+         if (hasStarted) {
+            if (table is not null) {
+               // The first update has already been fired. Seed observer w/ current state.
+               observer.OnNext(new DataPacket(table, table, new BlueTable(table.schema)));
+            }
+            return subject.Subscribe(observer);
+         }
+         hasStarted = true;
+      }
+      var subscription = subject.Subscribe(observer);
+      // Retrieve collection snapshot to bootstrap sequence
+      initTable().ignore();
+      // Watch changes
+      Task.Run(watchChanges);
+      return Disposable.Create(Dispose);
+
+      void Dispose() {
+         Log.Gen.info("Disposing MongoDbChangeObservable subscription");
+         subscription.Dispose();
+         cts.Cancel();
+         cts.Dispose();
+      }
+   }
+
+   protected virtual IBlueRowFactory<BsonDocument> getRowFactory(IEnumerable<BsonDocument> docs) =>
+      MongoSchemaAdaptor.InferSchema(docs);
+
+   async Task watchChanges() {
+      Log.Gen.info(
+         $"MongoCollectionStreamSource. Watching changes. thread: {Environment.CurrentManagedThreadId}");
+      while (collection is null) {
+         await Task.Delay(100);
+      }
+      Log.notnull(collection);
+      using var cursor = collection.Watch(new ChangeStreamOptions {
+         FullDocument = ChangeStreamFullDocumentOption.UpdateLookup
+      });
+      try {
+         while (await cursor.MoveNextAsync(cts.Token)) {
+            // Ignore while waiting for init
+            lock (syncRoot) {
+               if (table is null) {
+                  continue;
+               }
+            }
+            // schemaAdaptor is initialized after table; redundantly assert both
+            // are non-null to be safe, and clarify expectations.
+            Log.notnull(rowFactory);
+            Log.notnull(table);
+
+            var bd = table.ToDataPacketBuilder();
+            foreach (var ch in cursor.Current) {
+               var opDescJson = ch.OperationDescription?.ToJson() ?? string.Empty;
+               var updDescJson = ch.UpdateDescription?.ToJson() ?? string.Empty;
+               Log.Gen.info(
+                  $"MongoCollectionStream.watchChanges. Doc changed. type: {ch.OperationType} " +
+                  $"changes:\n\t\t{updDescJson}");
+               switch (ch.OperationType) {
+                  case ChangeStreamOperationType.Insert: {
+                        bd.AddOrUpdate(Row());
+                        break;
+                     }
+                  case ChangeStreamOperationType.Update: {
+                        bd.Remove(OldRow().Id());
+                        bd.AddOrUpdate(Row());
+                        break;
+                     }
+                  case ChangeStreamOperationType.Replace: {
+                        bd.Remove(OldRow().Id());
+                        bd.AddOrUpdate(Row());
+                        break;
+                     }
+                  case ChangeStreamOperationType.Delete: {
+                        bd.Remove(OldRow().Id());
+                        break;
+                     }
+                  default:
+                     Log.assert(false, $"Unsupported op type '{ch.OperationType}'");
+                     break;
+               }
+
+               IBlueRow Row() =>
+                  rowFactory.GetBlueRow(ch.FullDocumentBeforeChange);
+
+               IBlueRow OldRow() =>
+                  rowFactory.GetBlueRow(ch.FullDocumentBeforeChange);
+            }
+            var dp = bd.ToDataPacket();
+            if (!dp.HasChanges()) {
+               Log.Gen.warn(
+                  $"MongoDbChangeObservable Ignoring update because it didn't change any rows.");
+               continue;
+            }
+            lock (syncRoot) {
+               table = (BlueTableImmutable)dp.newRows;
+               subject.OnNext(dp);
+            }
+         }
+      } catch (TaskCanceledException ex) {
+         Log.Gen.warn(
+            $"MongoDbChangeObservable 'Watch' task cancelled (this is " +
+            $"expected if subscription was disposed). {ex}");
+      }
+   }
+
+
+   [MemberNotNull(nameof(collection))]
+   async Task initTable() {
+      collection = database.GetCollection<BsonDocument>(collectionName);
+      var cursor = collection.Find(Builders<BsonDocument>.Filter.Empty).ToCursor();
+      var docList = await cursor.ToListAsync();
+      rowFactory ??= MongoSchemaAdaptor.InferSchema(docList);
+      var firstTable = new BlueTableImmutable(rowFactory.Schema).ToDataPacketBuilder();
+      foreach (var doc in docList) {
+         var row = rowFactory.GetBlueRow(doc);
+         firstTable.AddOrUpdate(row);
+      }
+      var dp = firstTable.ToDataPacket();
+      lock (syncRoot) {
+         table = (BlueTableImmutable)dp.newRows;
+         subject.OnNext(dp);
+      }
+   }
+}

src/DataStreamSuite/DataStream/Primitives/MongoDB/MongoDBStream.cs

@@ -1,212 +1,210 @@
-using System;
-using System.Collections.Generic;
-using MongoDB.Driver;
-using MongoDB.Bson;
-using BCG;
+namespace BCG;
+
 using BCG.Utilities.General;
-using ZstdSharp.Unsafe;
-using System.Runtime.CompilerServices;
+
 using mbts.spark;
 
+using MongoDB.Bson;
+using MongoDB.Driver;
 
-namespace BCG {
-    [DataStreamUserRespresentation("mongodb_sql")]
-    public class MongoDBSQLStream : ViewStream, IWorker {
-        private ViewStreamDatabase _vsd;
-        private readonly string _connection_string, _db_name, _collection_name;
-        private MongoClient _client;
-        private BlueTable _table;
+using System;
+using System.Collections.Generic;
 
-        // Access the database
-        private IMongoDatabase _database;
-        private IDictionary<string, int> _id_map = new Dictionary<string, int>();
-        // mongodb://192.168.1.80:27017
+[DataStreamUserRespresentation("mongodb_sql")]
+public class MongoDBSQLStream : ViewStream, IWorker {
+   private ViewStreamDatabase _vsd;
+   private readonly string _connection_string, _db_name, _collection_name;
+   private MongoClient _client;
+   private BlueTable _table;
 
-        public MongoDBSQLStream(string connection_string, string db_name, string collection_name)
-        : this(connection_string, db_name, collection_name, ViewStreamDatabase.instance) {
+   // Access the database
+   private IMongoDatabase _database;
+   private IDictionary<string, int> _id_map = new Dictionary<string, int>();
+   // mongodb://192.168.1.80:27017
 
-        }
-        public MongoDBSQLStream(string connection_string, string db_name, string collection_name, ViewStreamDatabase vsd) {
-            bts.Log.Gen.info($"MongoDBStream constructed connection_string: {connection_string} " +
-                                    $"db_name: {db_name}");
-            _connection_string = connection_string;
-            _db_name = db_name;
-            _collection_name = collection_name;
-            _vsd = vsd;
-            _client = new MongoClient(_connection_string);
-            _database = _client.GetDatabase(_db_name);
-        }
+   public MongoDBSQLStream(string connection_string, string db_name, string collection_name)
+   : this(connection_string, db_name, collection_name, ViewStreamDatabase.instance) {
 
-        protected override void myOnFirstSubscriber() {
-            MasterWorker.getInstance(false).addWorker(this, TimeSpan.FromSeconds(1));
-        }
+   }
+   public MongoDBSQLStream(string connection_string, string db_name, string collection_name, ViewStreamDatabase vsd) {
+      bts.Log.Gen.info($"MongoDBStream constructed connection_string: {connection_string} " +
+                              $"db_name: {db_name}");
+      _connection_string = connection_string;
+      _db_name = db_name;
+      _collection_name = collection_name;
+      _vsd = vsd;
+      _client = new MongoClient(_connection_string);
+      _database = _client.GetDatabase(_db_name);
+   }
 
-        protected override void myOnLastSubscriber() {
-            MasterWorker.getInstance(false).removeWorker(this);
-        }
+   protected override void myOnFirstSubscriber() {
+      MasterWorker.getInstance(false).addWorker(this, TimeSpan.FromSeconds(1));
+   }
 
-        public void doWork() {
-            _read_mogodb();
-        }
+   protected override void myOnLastSubscriber() {
+      MasterWorker.getInstance(false).removeWorker(this);
+   }
 
-        private void _read_mogodb() {
-            HighPerformanceTimer timer = new HighPerformanceTimer();
-            timer.start();
-            // Access the collection
-            IMongoCollection<BsonDocument> collection = _database.GetCollection<BsonDocument>(name: _collection_name);
+   public void doWork() {
+      _read_mogodb();
+   }
 
-            // Fetch all documents from the collection
-            var documents = collection.Find(Builders<BsonDocument>.Filter.Empty).ToCursor();
-            _table = _table == null ? null : new BlueTable(_table.schema);
-            // Iterate through the documents and print values
-            foreach (var document in documents.ToEnumerable()) {
-                bts.Log.Gen.info($"document: {document} json: {document.ToJson()}");
-                if (_table == null) {
-                    _init_table(document);
-                }
-                _add_row(document);
-            }
-            timer.stop();
-            bts.Log.Gen.info($"MongoDBStream time to read {_collection_name} {timer.lastIntervalTime} msecs");
-            timer.start();
-            var dp = BlueDataUtility.build_dp_inc(_table, myLastValue == null ? null : myLastValue as DataPacket, true);
-            timer.stop();
-            bts.Log.Gen.info($"MongoDBStream time to compare tables :{timer.lastIntervalTime} msecs");
-            if (dp.addedRows.Count == 0 && dp.deletedRows.Count == 0)
-                return;
-            myFireEvent(dp);
-        }
+   private void _read_mogodb() {
+      HighPerformanceTimer timer = new HighPerformanceTimer();
+      timer.start();
+      // Access the collection
+      IMongoCollection<BsonDocument> collection = _database.GetCollection<BsonDocument>(name: _collection_name);
 
-        private void _init_table(BsonDocument doc_row) {
-            bts.Log.assert(_table == null);
-            List<ColumnInfo> cis = new List<ColumnInfo>();
-            cis.Add((new ColumnInfo("id", 0, typeof(Int32))));
-            foreach (var n in doc_row.Names) {
-                cis.Add(new ColumnInfo(n, cis.Count, typeof(string)));
-            }
-            IBlueTableSchema schema = new BlueTableSchema(cis.ToArray(), _collection_name);
-            _table = new BlueTable(schema);
-        }
+      // Fetch all documents from the collection
+      var documents = collection.Find(Builders<BsonDocument>.Filter.Empty).ToCursor();
+      _table = _table == null ? null : new BlueTable(_table.schema);
+      // Iterate through the documents and print values
+      foreach (var document in documents.ToEnumerable()) {
+         bts.Log.Gen.info($"document: {document} json: {document.ToJson()}");
+         if (_table == null) {
+            _init_table(document);
+         }
+         _add_row(document);
+      }
+      timer.stop();
+      bts.Log.Gen.info($"MongoDBStream time to read {_collection_name} {timer.lastIntervalTime} msecs");
+      timer.start();
+      var dp = BlueDataUtility.build_dp_inc(_table, myLastValue == null ? null : myLastValue as DataPacket, true);
+      timer.stop();
+      bts.Log.Gen.info($"MongoDBStream time to compare tables :{timer.lastIntervalTime} msecs");
+      if (dp.addedRows.Count == 0 && dp.deletedRows.Count == 0)
+         return;
+      myFireEvent(dp);
+   }
 
-        private void _add_row(BsonDocument doc) {
-            var key_val = doc["_id"];
-            var key = key_val.ToString();
-            int id;
-            if (!_id_map.TryGetValue(key, out id)) {
-                id = _id_map.Count;
-                _id_map.Add(key, id);
-            }
-            MutableBlueRow nr = new MutableBlueRow(_table.schema);
-            nr.setValue("id", id);
-            foreach (var n in doc.Names) {
-                var val = doc[n].ToString();
-                nr.setValue(n, val);
-            }
-            _table.addRow(nr);
-        }
-    }
+   private void _init_table(BsonDocument doc_row) {
+      bts.Log.assert(_table == null);
+      List<ColumnInfo> cis = new List<ColumnInfo>();
+      cis.Add((new ColumnInfo("id", 0, typeof(Int32))));
+      foreach (var n in doc_row.Names) {
+         cis.Add(new ColumnInfo(n, cis.Count, typeof(string)));
+      }
+      IBlueTableSchema schema = new BlueTableSchema(cis.ToArray(), _collection_name);
+      _table = new BlueTable(schema);
+   }
 
-    [DataStreamUserRespresentation("mongodb")]
-    public class MongoDBStream : ViewStream, IWorker {
-        private ViewStreamDatabase _vsd;
-        private readonly string _connection_string, _db_name, _collection_name;
-        private MongoClient _client;
-        private BlueTable _table;
+   private void _add_row(BsonDocument doc) {
+      var key_val = doc["_id"];
+      var key = key_val.ToString();
+      int id;
+      if (!_id_map.TryGetValue(key, out id)) {
+         id = _id_map.Count;
+         _id_map.Add(key, id);
+      }
+      MutableBlueRow nr = new MutableBlueRow(_table.schema);
+      nr.setValue("id", id);
+      foreach (var n in doc.Names) {
+         var val = doc[n].ToString();
+         nr.setValue(n, val);
+      }
+      _table.addRow(nr);
+   }
+}
 
-        // Access the database
-        private IMongoDatabase _database;
-        private IDictionary<string, int> _id_map = new Dictionary<string, int>();
-        // mongodb://192.168.1.80:27017
+[DataStreamUserRespresentation("mongodb")]
+public class MongoDBStream : ViewStream, IWorker {
+   private ViewStreamDatabase _vsd;
+   private readonly string _connection_string, _db_name, _collection_name;
+   private MongoClient _client;
+   private BlueTable _table;
 
-        public MongoDBStream(string connection_string, string db_name, string collection_name)
-        : this(connection_string, db_name, collection_name, ViewStreamDatabase.instance) {
+   // Access the database
+   private IMongoDatabase _database;
+   private IDictionary<string, int> _id_map = new Dictionary<string, int>();
+   // mongodb://192.168.1.80:27017
 
-        }
-        public MongoDBStream(string connection_string, string db_name, string collection_name, ViewStreamDatabase vsd) {
-            bts.Log.Gen.info($"MongoDBStream constructed connection_string: {connection_string} " +
-                                    $"db_name: {db_name}");
-            _connection_string = connection_string;
-            _db_name = db_name;
-            _collection_name = collection_name;
-            _vsd = vsd;
-            _client = new MongoClient(_connection_string);
-            _database = _client.GetDatabase(_db_name);
-        }
+   public MongoDBStream(string connection_string, string db_name, string collection_name)
+   : this(connection_string, db_name, collection_name, ViewStreamDatabase.instance) {
 
-        protected override void myOnFirstSubscriber() {
-            MasterWorker.getInstance(false).addWorker(this, TimeSpan.FromSeconds(1));
-        }
+   }
+   public MongoDBStream(string connection_string, string db_name, string collection_name, ViewStreamDatabase vsd) {
+      bts.Log.Gen.info($"MongoDBStream constructed connection_string: {connection_string} " +
+                              $"db_name: {db_name}");
+      _connection_string = connection_string;
+      _db_name = db_name;
+      _collection_name = collection_name;
+      _vsd = vsd;
+      _client = new MongoClient(_connection_string);
+      _database = _client.GetDatabase(_db_name);
+   }
 
-        protected override void myOnLastSubscriber() {
-            MasterWorker.getInstance(false).removeWorker(this);
-        }
+   protected override void myOnFirstSubscriber() {
+      MasterWorker.getInstance(false).addWorker(this, TimeSpan.FromSeconds(1));
+   }
 
-        public void doWork() {
-            _read_mogodb();
-        }
+   protected override void myOnLastSubscriber() {
+      MasterWorker.getInstance(false).removeWorker(this);
+   }
 
-        private void _read_mogodb() {
-            HighPerformanceTimer timer = new HighPerformanceTimer();
-            timer.start();
-            // Access the collection
-            IMongoCollection<BsonDocument> collection = _database.GetCollection<BsonDocument>(name: _collection_name);
+   public void doWork() {
+      _read_mogodb();
+   }
 
-            // Fetch all documents from the collection
-            var documents = collection.Find(Builders<BsonDocument>.Filter.Empty).ToCursor();
-            _table = _table == null ? null : new BlueTable(_table.schema);
-            // Iterate through the documents and print values
-            foreach (var document in documents.ToEnumerable()) {
-                bts.Log.Gen.info($"document: {document} json: {document.ToJson()}");
-                if (_table == null) {
-                    _init_table(document);
-                }
-                _add_doc(document);
-            }
-            timer.stop();
-            bts.Log.Gen.info($"MongoDBStream time to read {_collection_name} {timer.lastIntervalTime} msecs");
-            timer.start();
-            var dp = BlueDataUtility.build_dp_inc(_table, myLastValue == null ? null : myLastValue as DataPacket, true);
-            timer.stop();
-            bts.Log.Gen.info($"MongoDBStream time to compare tables :{timer.lastIntervalTime} msecs");
-            if (dp.addedRows.Count == 0 && dp.deletedRows.Count == 0)
-                return;
-            myFireEvent(dp);
-        }
+   private void _read_mogodb() {
+      HighPerformanceTimer timer = new HighPerformanceTimer();
+      timer.start();
+      // Access the collection
+      IMongoCollection<BsonDocument> collection = _database.GetCollection<BsonDocument>(name: _collection_name);
 
-        private void _init_table(BsonDocument doc_row) {
-            bts.Log.assert(_table == null);
-            var schema = BlueDataUtility.form_schema($"{_database}_{_table}",
-            Tuple.Create("id", TypeCode.Int32),
-            Tuple.Create("doc_id", TypeCode.String),
-            Tuple.Create("key", TypeCode.String),
-            Tuple.Create("value", TypeCode.String));
-            _table = new BlueTable(schema);
-        }
+      // Fetch all documents from the collection
+      var documents = collection.Find(Builders<BsonDocument>.Filter.Empty).ToCursor();
+      _table = _table == null ? null : new BlueTable(_table.schema);
+      // Iterate through the documents and print values
+      foreach (var document in documents.ToEnumerable()) {
+         bts.Log.Gen.info($"document: {document} json: {document.ToJson()}");
+         if (_table == null) {
+            _init_table(document);
+         }
+         _add_doc(document);
+      }
+      timer.stop();
+      bts.Log.Gen.info($"MongoDBStream time to read {_collection_name} {timer.lastIntervalTime} msecs");
+      timer.start();
+      var dp = BlueDataUtility.build_dp_inc(_table, myLastValue == null ? null : myLastValue as DataPacket, true);
+      timer.stop();
+      bts.Log.Gen.info($"MongoDBStream time to compare tables :{timer.lastIntervalTime} msecs");
+      if (dp.addedRows.Count == 0 && dp.deletedRows.Count == 0)
+         return;
+      myFireEvent(dp);
+   }
 
-        private void _add_doc(BsonDocument doc) {
-            var doc_id_bson = doc["_id"];
-            var doc_id = doc_id_bson.ToString();
-            Func<string, string, int> get_id = (string doc_id, string key) => {
-                string hash_key = $"{doc_id}_{key}";
-                int id = 0;
-                if (!_id_map.TryGetValue(hash_key, out id)) {
-                    id = _id_map.Count;
-                    _id_map.Add(hash_key, id);
-                }
-                return id;
-            };
-            foreach (var n in doc.Names) {
-                string key = n;
-                var id = get_id(doc_id, key);
-                MutableBlueRow nr = new MutableBlueRow(_table.schema);
-                var val = doc[n].ToString();
-                nr.setValue("id", id);
-                nr.setValue("doc_id", doc_id);
-                nr.setValue("key", key);
-                nr.setValue("value", val);
-                _table.addRow(nr);
-            }
-        }
-    }
+   private void _init_table(BsonDocument doc_row) {
+      bts.Log.assert(_table == null);
+      var schema = BlueDataUtility.form_schema($"{_database}_{_table}",
+      Tuple.Create("id", TypeCode.Int32),
+      Tuple.Create("doc_id", TypeCode.String),
+      Tuple.Create("key", TypeCode.String),
+      Tuple.Create("value", TypeCode.String));
+      _table = new BlueTable(schema);
+   }
 
-}
+   private void _add_doc(BsonDocument doc) {
+      var doc_id_bson = doc["_id"];
+      var doc_id = doc_id_bson.ToString();
+      Func<string, string, int> get_id = (string doc_id, string key) => {
+         string hash_key = $"{doc_id}_{key}";
+         int id = 0;
+         if (!_id_map.TryGetValue(hash_key, out id)) {
+            id = _id_map.Count;
+            _id_map.Add(hash_key, id);
+         }
+         return id;
+      };
+      foreach (var n in doc.Names) {
+         string key = n;
+         var id = get_id(doc_id, key);
+         MutableBlueRow nr = new MutableBlueRow(_table.schema);
+         var val = doc[n].ToString();
+         nr.setValue("id", id);
+         nr.setValue("doc_id", doc_id);
+         nr.setValue("key", key);
+         nr.setValue("value", val);
+         _table.addRow(nr);
+      }
+   }
+}

src/DataStreamSuite/DataStream/Primitives/MongoDB/MongoDbExtensions.cs

@@ -0,0 +1,63 @@
+namespace Bts.DataStream;
+
+using MongoDB.Bson;
+
+using System;
+
+public static class MongoDbExtensions {
+   public static BsonArray GetArray(this BsonDocument doc, string elementName) =>
+      doc[elementName].AsBsonArray;
+
+   public static BsonArray? GetArrayOrDefault(this BsonDocument doc, string elementName) =>
+      doc.TryGetValue(elementName, out var val) && val.IsBsonArray ?
+         val.AsBsonArray : null;
+
+   public static string GetString(this BsonDocument doc, string elementName) =>
+      doc[elementName].AsString;
+
+   public static string? GetStringOrDefault(this BsonDocument doc, string elementName) =>
+      doc.TryGetValue(elementName, out var val) && val.IsString ?
+         val.AsString : null;
+
+   public static int GetInt32(this BsonDocument doc, string elementName) =>
+      doc[elementName].AsInt32;
+
+   public static int? GetInt32OrDefault(this BsonDocument doc, string elementName) =>
+      doc.TryGetValue(elementName, out var val) && val.IsInt32 ?
+         val.AsInt32 : null;
+
+   public static long GetInt64(this BsonDocument doc, string elementName) =>
+      doc[elementName].AsInt64;
+
+   public static long? GetInt64OrDefault(this BsonDocument doc, string elementName) =>
+      doc.TryGetValue(elementName, out var val) && val.IsInt64 ?
+         val.AsInt64 : null;
+
+   public static double GetDouble(this BsonDocument doc, string elementName) =>
+      doc[elementName].AsDouble;
+
+   public static double? GetDoubleOrDefault(this BsonDocument doc, string elementName) =>
+      doc.TryGetValue(elementName, out var val) && val.IsDouble ?
+         val.AsDouble : null;
+
+   public static DateTime GetDateTime(this BsonDocument doc, string elementName) =>
+      doc[elementName].ToUniversalTime();
+
+   public static DateTime? GetDateTimeOrDefault(this BsonDocument doc, string elementName) =>
+      doc.TryGetValue(elementName, out var val) && val.IsValidDateTime ?
+         val.ToUniversalTime() : null;
+
+   public static DateTime GetTimestamp(this BsonDocument doc, string elementName) =>
+      doc[elementName].ToUniversalTime();
+
+   public static DateTime? GetTimestampOrDefault(this BsonDocument doc, string elementName) =>
+      doc.TryGetValue(elementName, out var val) && val.IsValidDateTime ?
+         val.ToUniversalTime() : null;
+
+   public static ObjectId GetObjectId(this BsonDocument doc, string elementName) =>
+      doc[elementName].AsObjectId;
+
+   public static ObjectId? GetObjectIdOrDefault(this BsonDocument doc, string elementName) =>
+      doc.TryGetValue(elementName, out var val) && val.IsObjectId ?
+         val.AsObjectId : null;
+}

src/DataStreamSuite/DataStream/Primitives/MongoDB/MongoSchemaAdaptor.cs

@@ -0,0 +1,57 @@
+namespace Bts.DataStream;
+
+using BCG;
+
+using bts;
+
+using MongoDB.Bson;
+
+using System;
+using System.Collections.Generic;
+
+public static class MongoSchemaAdaptor {
+   public static BlueTableSchemaAdaptor<BsonDocument> InferSchema(IEnumerable<BsonDocument> docs) {
+      var schemaAdaptor = new BlueTableSchemaAdaptor<BsonDocument>("MongoCollectionStream");
+      foreach (var doc in docs) {
+         foreach (var el in doc) {
+            if (schemaAdaptor.Contains(el.Name)) {
+               continue;
+            }
+            switch (el.Value.BsonType) {
+               case BsonType.Array:
+                  schemaAdaptor.AddColumn(el.Name, d => d.GetArrayOrDefault(el.Name)?.ToString() ?? string.Empty);
+                  break;
+               case BsonType.String:
+                  schemaAdaptor.AddColumn(el.Name, d => d.GetStringOrDefault(el.Name) ?? string.Empty);
+                  break;
+               case BsonType.Int32:
+                  schemaAdaptor.AddColumn(el.Name, d => d.GetInt32OrDefault(el.Name));
+                  break;
+               case BsonType.Int64:
+                  schemaAdaptor.AddColumn(el.Name, d => d.GetInt64OrDefault(el.Name));
+                  break;
+               case BsonType.Double:
+                  schemaAdaptor.AddColumn(el.Name, d => d.GetDoubleOrDefault(el.Name));
+                  break;
+               case BsonType.DateTime:
+                  schemaAdaptor.AddColumn(el.Name, d => d.GetDateTimeOrDefault(el.Name));
+                  break;
+               case BsonType.Timestamp:
+                  schemaAdaptor.AddColumn(el.Name, d => d.GetTimestampOrDefault(el.Name));
+                  break;
+               case BsonType.ObjectId:
+                  schemaAdaptor.AddColumn(el.Name, d => d.GetObjectIdOrDefault(el.Name)?.ToString());
+                  break;
+               case BsonType.Null:
+                  Log.Gen.warn(
+                     $"Ignoring column '{el.Name}' because at least one element is null");
+                  break;
+               default:
+                  throw new InvalidOperationException(
+                     $"Unsupported BsonType '{el.Value.BsonType}'");
+            }
+         }
+      }
+      return schemaAdaptor;
+   }
+}

src/DataStreamSuite/DataStream/Primitives/SideEffectStreams/SchemaGuesserStream.cs

@@ -0,0 +1,34 @@
+using BCG;
+
+namespace bts {
+    public class SchemaGuesserStream : ViewStream {
+
+        private IBlueRowCollection _rows = null;
+        private ViewStream _vs;
+        public SchemaGuesserStream(ViewStream vs) {
+            _vs = vs;
+        }
+
+        protected override void myOnFirstSubscriber() {
+            _vs.onChange += _on_data;
+        }
+        protected override void myOnLastSubscriber() {
+            _vs.onChange -= _on_data;
+        }
+
+        private void _on_data(object obj) {
+            DataPacket dp = obj as DataPacket;
+            if (_rows == null) {
+                _rows = BlueDataUtility.get_typed_table(dp.newRows);
+            }
+            var dp_out = BlueDataUtility.build_dp_inc(_rows, myLastValue as DataPacket);
+            myFireEvent(dp_out);
+
+
+        }
+
+
+
+
+    }
+}

src/DataStreamSuite/DataStream/Primitives/Sort/SortStream.cs

@@ -1,99 +1,99 @@
-using Bts.ColumnSortExpressions;
-
-using System;
-using System.Collections;
-using System.Collections.Generic;
-using System.Diagnostics.CodeAnalysis;
-
-namespace BCG;
-
-/// <summary>
-/// Defines a Sort stream, with performance of O(n) for all the increments. It uses simple merge sort
-/// to merge the additions to the accumulates table
-/// </summary>
-public class SortStream : ViewStream {
-   ViewStream stream;
-   IList<ColumnSortExpression> sortCols;
-   IComparer? comparer;
-   DataPacket? lastFiredPacket;
-   DataPacket? lastReceivedPacket;
-
-   public SortStream(ViewStream stream, string sortExpression) {
-      this.stream = stream;
-      SortExpression = sortExpression;
-   }
-
-   public SortStream(ViewStream vs, string colName, bool isAsc) {
-      stream = vs;
-      SortExpression = isAsc ? "+" + colName : "-" + colName;
-   }
-
-   public SortStream(ViewStream stream, IComparer comparer) {
-      this.stream = stream;
-      this.comparer = comparer;
-      sortCols = Array.Empty<ColumnSortExpression>();
-   }
-
-   [MemberNotNull(nameof(sortCols))]
-   public string SortExpression {
-      set {
-         comparer = null;
-         lastFiredPacket = null;
-         sortCols = ColumnSortExpressionInterpreter.Evaluate(value);
-         if (lastReceivedPacket is not null) {
-            myChangeHandler(lastReceivedPacket);
-         }
-      }
-   }
-
-   public IComparer Comparer {
-      set {
-         comparer = value;
-         lastFiredPacket = null;
-         if (lastReceivedPacket is not null) {
-            myChangeHandler(lastReceivedPacket);
-         }
-      }
-   }
-
-   public void myChangeHandlerTest(object o) =>
-      myChangeHandler(o);
-
-   [MemberNotNull(nameof(lastFiredPacket))]
-   internal void myChangeHandler(object o) {
-      lastReceivedPacket = (DataPacket)o;
-      // First callback, so we need to sort the whole table, this is the
-      // only time we will do that for all the other sorting operations would
-      // be sorting the increments and inserting them into the whole list
-      if (lastFiredPacket is null) {
-         lastFiredPacket = new() {
-            newRows = comparer is null ?
-               new SortedTable(lastReceivedPacket.newRows, sortCols) :
-               new SortedTable(lastReceivedPacket.newRows, comparer)
-         };
-         lastFiredPacket.addedRows = new OrderedBlueTable(lastFiredPacket.newRows);
-         lastFiredPacket.deletedRows = new BlueTable(lastFiredPacket.newRows.schema);
-         myFireEvent(lastFiredPacket.Clone());
-         return;
-      }
-      // Deletes
-      lastFiredPacket.deletedRows =
-         comparer == null ?
-         new SortedTable(lastReceivedPacket.deletedRows, sortCols) :
-         new SortedTable(lastReceivedPacket.deletedRows, comparer);
-      ((SortedTable)lastFiredPacket.newRows).deleteRows(lastFiredPacket.deletedRows);
-      // Adds
-      lastFiredPacket.addedRows = new BlueTable(lastReceivedPacket.addedRows);
-      ((SortedTable)lastFiredPacket.newRows).addRows(lastFiredPacket.addedRows);
-      myFireEvent(lastFiredPacket.Clone());
-   }
-
-   protected override void myOnFirstSubscriber() =>
-      stream.onChange += myChangeHandler;
-
-   protected override void myOnLastSubscriber() {
-      lastFiredPacket = null;
-      lastReceivedPacket = null;
-      stream.onChange -= myChangeHandler;
-   }
-}
+using Bts.ColumnSortExpressions;
+
+using System;
+using System.Collections;
+using System.Collections.Generic;
+using System.Diagnostics.CodeAnalysis;
+
+namespace BCG;
+
+/// <summary>
+/// Defines a Sort stream, with performance of O(n) for all the increments. It uses simple merge sort
+/// to merge the additions to the accumulates table
+/// </summary>
+public class SortStream : ViewStream {
+   ViewStream stream;
+   IList<ColumnSortExpression> sortCols;
+   IComparer? comparer;
+   DataPacket? lastFiredPacket;
+   DataPacket? lastReceivedPacket;
+
+   public SortStream(ViewStream stream, string sortExpression) {
+      this.stream = stream;
+      SortExpression = sortExpression;
+   }
+
+   public SortStream(ViewStream vs, string colName, bool isAsc) {
+      stream = vs;
+      SortExpression = isAsc ? "+" + colName : "-" + colName;
+   }
+
+   public SortStream(ViewStream stream, IComparer comparer) {
+      this.stream = stream;
+      this.comparer = comparer;
+      sortCols = Array.Empty<ColumnSortExpression>();
+   }
+
+   [MemberNotNull(nameof(sortCols))]
+   public string SortExpression {
+      set {
+         comparer = null;
+         lastFiredPacket = null;
+         sortCols = ColumnSortExpressionInterpreter.Evaluate(value);
+         if (lastReceivedPacket is not null) {
+            myChangeHandler(lastReceivedPacket);
+         }
+      }
+   }
+
+   public IComparer Comparer {
+      set {
+         comparer = value;
+         lastFiredPacket = null;
+         if (lastReceivedPacket is not null) {
+            myChangeHandler(lastReceivedPacket);
+         }
+      }
+   }
+
+   public void myChangeHandlerTest(object o) =>
+      myChangeHandler(o);
+
+   [MemberNotNull(nameof(lastFiredPacket))]
+   internal void myChangeHandler(object o) {
+      lastReceivedPacket = (DataPacket)o;
+      // First callback, so we need to sort the whole table, this is the
+      // only time we will do that for all the other sorting operations would
+      // be sorting the increments and inserting them into the whole list
+      if (lastFiredPacket is null) {
+         lastFiredPacket = new() {
+            newRows = comparer is null ?
+               new SortedTable(lastReceivedPacket.newRows, sortCols) :
+               new SortedTable(lastReceivedPacket.newRows, comparer)
+         };
+         lastFiredPacket.addedRows = new OrderedBlueTable(lastFiredPacket.newRows);
+         lastFiredPacket.deletedRows = new BlueTable(lastFiredPacket.newRows.schema);
+         myFireEvent(lastFiredPacket.Clone());
+         return;
+      }
+      // Deletes
+      lastFiredPacket.deletedRows =
+         comparer == null ?
+         new SortedTable(lastReceivedPacket.deletedRows, sortCols) :
+         new SortedTable(lastReceivedPacket.deletedRows, comparer);
+      ((SortedTable)lastFiredPacket.newRows).deleteRows(lastFiredPacket.deletedRows);
+      // Adds
+      lastFiredPacket.addedRows = new BlueTable(lastReceivedPacket.addedRows);
+      ((SortedTable)lastFiredPacket.newRows).addRows(lastFiredPacket.addedRows);
+      myFireEvent(lastFiredPacket.Clone());
+   }
+
+   protected override void myOnFirstSubscriber() =>
+      stream.onChange += myChangeHandler;
+
+   protected override void myOnLastSubscriber() {
+      lastFiredPacket = null;
+      lastReceivedPacket = null;
+      stream.onChange -= myChangeHandler;
+   }
+}

src/DataStreamSuite/DataStream/Primitives/Top/TopStream.cs

@@ -1,49 +1,43 @@
-// using System;
+using System;
 
-// namespace BCG
-// {
-//    public class TopStream : MethodDrivenStream
-//    {
-//       private int myTopCount;
+namespace BCG {
+    public class TopStream : MethodDrivenStream {
+        private int myTopCount;
 
-//       public TopStream (ViewStream stream, int topCount) : base (stream)  {myTopCount = topCount;}
+        public TopStream(ViewStream stream, int topCount) : base(stream) { myTopCount = topCount; }
 
-//       public TopStream (ViewStream stream, 
-//          int topCount, 
-//          string sortExpression) 
-//          :
-//          this (sortExpression == null? 
-//          stream :
-//          new SortStream (stream, sortExpression), topCount)
-//       {}
+        public TopStream(ViewStream stream,
+           int topCount,
+           string sortExpression)
+           :
+           this(sortExpression == null ?
+           stream :
+           new SortStream(stream, sortExpression), topCount) { }
 
-//       protected override void myOnTableClear () 
-//       {
-//          myLastFiredPacket.deletedRows = myLastFiredPacket.newRows;
-//          myLastFiredPacket.newRows = new OrderedBlueTable (myLastFiredPacket.deletedRows.schema);
-//          myLastFiredPacket.addedRows = new OrderedBlueTable (myLastFiredPacket.deletedRows.schema);
-//       }
+        protected override void myOnTableClear() {
+            myLastFiredPacket.deletedRows = myLastFiredPacket.newRows;
+            myLastFiredPacket.newRows = new OrderedBlueTable(myLastFiredPacket.deletedRows.schema);
+            myLastFiredPacket.addedRows = new OrderedBlueTable(myLastFiredPacket.deletedRows.schema);
+        }
 
-//       protected override void myHandleFirstCall () 
-//       {
-//          myLastFiredPacket = new DataPacket ();
-//          myLastFiredPacket.newRows = new OrderedBlueTable (myLastArrivedPacket.newRows, myTopCount);
-//          myLastFiredPacket.addedRows = new OrderedBlueTable(myLastFiredPacket.newRows);
-//          myLastFiredPacket.deletedRows = new OrderedBlueTable (myLastFiredPacket.newRows.schema);
-//       }
+        protected override void myHandleFirstCall() {
+            myLastFiredPacket = new DataPacket();
+            myLastFiredPacket.newRows = new OrderedBlueTable(myLastArrivedPacket.newRows, myTopCount);
+            myLastFiredPacket.addedRows = new OrderedBlueTable(myLastFiredPacket.newRows);
+            myLastFiredPacket.deletedRows = new OrderedBlueTable(myLastFiredPacket.newRows.schema);
+        }
 
-//       protected override void myHandleIncrementalAdd ()     {}
+        protected override void myHandleIncrementalAdd() { }
 
-//       protected override void myHandleIncrementalDelete()   
-//       {
-//          IBlueRowCollection oldRows = myLastFiredPacket.newRows;
-//          myLastFiredPacket.newRows = new OrderedBlueTable (myLastArrivedPacket.newRows, myTopCount);
-//          BlueDataUtility.computeDifferences (myLastFiredPacket.newRows,
-//             oldRows,
-//             myLastFiredPacket.schema.columnNames,
-//             out myLastFiredPacket.deletedRows,
-//             out myLastFiredPacket.addedRows);
-//          myFireSwitch = myLastFiredPacket.addedRows.Count + myLastFiredPacket.deletedRows.Count == 0 ? false : true;
-//       }
-//    }
-// }
+        protected override void myHandleIncrementalDelete() {
+            IBlueRowCollection oldRows = myLastFiredPacket.newRows;
+            myLastFiredPacket.newRows = new OrderedBlueTable(myLastArrivedPacket.newRows, myTopCount);
+            BlueDataUtility.computeDifferences(myLastFiredPacket.newRows,
+               oldRows,
+               myLastFiredPacket.schema.columnNames,
+               out myLastFiredPacket.deletedRows,
+               out myLastFiredPacket.addedRows);
+            myFireSwitch = myLastFiredPacket.addedRows.Count + myLastFiredPacket.deletedRows.Count == 0 ? false : true;
+        }
+    }
+}

src/DataStreamSuite/DataStreamCore/BlueData/BlueDataUtility.cs

@@ -6,6 +6,7 @@ using System.Reflection;
 using System.IO;
 using System.Diagnostics;
 using System.Linq;
+using System.Globalization;
 
 //using BCG.Utilities;
 using BCG.Utilities.General;
@@ -292,21 +293,118 @@ namespace BCG {
          return TypeCode.Empty;
       }
 
-      public static IBlueTableSchema to_typed_schema(IBlueRowCollection rows) {
-         List<ColumnInfo> cis = new List<ColumnInfo>();
-         foreach (var col in rows.schema.columnInfo) {
-            var text_vals = BlueDataUtility.get_colvalues<string>(rows, col.colName);
-            TypeCode type_code = convert_column_type(text_vals);
-            cis.Add(new ColumnInfo(col.colName, cis.Count, _typecode_to_type[type_code]));
+      // public static Type guess_type(string val) {
+      //    //DateTime
+      //    {
+      //       DateTime dt;
+      //       var success = DateTime.TryParseExact(val, "yyyy-MM-dd HH:mm:ss", null, DateTimeStyles.AllowWhiteSpaces, out dt);
+      //       if (success) {
+      //          return typeof(DateTime);
+      //       }
+      //       DateTimeOffset dtoff;
+      //       success = DateTimeOffset.TryParseExact(val, "yyyy-MM-dd HH:mm:sszzz",
+      //                                              null, System.Globalization.DateTimeStyles.None, out dtoff);
+
+      //       if (success) {
+      //          return typeof(DateTime);
+      //       }
+      //    }
+      //    //int
+      //    {
+      //       int i;
+      //       var success = Int32.TryParse(val, out i);
+      //       if (success) {
+      //          return typeof(Int32);
+      //       }
+      //    }
+      //    //double
+      //    {
+      //       double d;
+      //       var success = Double.TryParse(val, out d);
+      //       if (success) {
+      //          return typeof(Double);
+      //       }
+      //    }
+      //    //bool
+      //    {
+      //       bool b;
+      //       var success = Boolean.TryParse(val, out b);
+      //       if (success) {
+      //          return typeof(bool);
+      //       }
+      //    }
+      //    //string
+      //    return typeof(string);
+      // }
+
+      private static Type[] _parsable_types = { typeof(DateTime), typeof(int), typeof(double), typeof(bool) };
+      public static Type guess_type(string val) {
+         foreach (var t in _parsable_types) {
+            var type_val = convert_to_type(t, val);
+            if (type_val != null)
+               return t;
          }
-         return new BlueTableSchema(cis.ToArray(), rows.schema.tableName);
+         return typeof(string);
+
       }
 
-      // public static IBlueRow convert_string_row(IBlueRow r, IBlueTableSchema out_schema) {
-      //    foreach (var ci in out_schema.columnInfo){
+      public static IBlueTableSchema get_typed_schema(IBlueRowCollection rows) {
+         // just use one row to guess
+         List<ColumnInfo> cis = new List<ColumnInfo>();
+         foreach (IBlueRow r in rows) {
+            foreach (var ci in r.schema.columnInfo) {
+               if (ci.colType != typeof(System.String)) {
+                  cis.Add(ci);
+                  continue;
+               }
+               Type type = guess_type(r[ci.colName] as string);
+               cis.Add(new ColumnInfo(ci.colName, ci.colNo, type));
+            }
+            return new BlueTableSchema(cis.ToArray(), rows.schema.tableName);
+         }
+         return null;
+      }
+
+      static object convert_to_type(Type t, string value) {
+         // Attempt to convert the string to the specified type
+         try {
+            return Convert.ChangeType(value, t);
+         } catch (Exception) {
+            if (t == typeof(double)) {
+               // some doubles come with currency and ',' we clean that up here (there might be better ways to do this)
+               string cleaned_val = new string(value.Where(c => c != '$' && c != ',').ToArray());
+               cleaned_val = cleaned_val.Replace('−', '-');
+               double rv_d;
+               if (Double.TryParse(cleaned_val, out rv_d))
+                  return rv_d;
+               else
+                  return null;
+            }
+            return null;
+            // Handle conversion failure gracefully
+            // throw new ArgumentException("Failed to convert the string to the specified type.");
+         }
+      }
+
+      // converts a row where all columns are string types to a schema where the best types are guesed
+      public static IBlueRowCollection get_typed_table(IBlueRowCollection string_rows) {
+         Func<IBlueRow, IBlueTableSchema, IBlueRow> get_typed_row = (IBlueRow string_row, IBlueTableSchema typed_schema) => {
+            MutableBlueRow rv = new MutableBlueRow(typed_schema);
+            foreach (ColumnInfo ci in typed_schema.columnInfo) {
+               var typed_val = convert_to_type(ci.colType, value: string_row[ci.colName].ToString());
+               rv.setValue(ci.colName, typed_val);
+            }
+            return rv;
+         };
+         IBlueTableSchema typed_schema = get_typed_schema(string_rows);
+         var rv = new BlueTable(typed_schema);
+         foreach (IBlueRow r in string_rows) {
+            var typed_row = get_typed_row(r, typed_schema);
+            rv.addRow(typed_row);
+         }
+         return rv;
+      }
 
-      //    }
-      // }
 
 
       /// <summary>
@@ -2112,5 +2210,33 @@ namespace BCG {
       }
 
       #endregion
+
+      public class RowComparer<T> : IComparer<IBlueRow> {
+         private readonly IEnumerable<Tuple<string, bool>> _sort_cols;
+         public RowComparer(IEnumerable<Tuple<string, bool>> sort_cols) {
+            _sort_cols = sort_cols;
+         }
+         public int Compare(IBlueRow? x, IBlueRow? y) {
+            Func<object, object, int> compare_cols = (o1, o2) => {
+               return (o1 as IComparable).CompareTo(o2);
+            };
+            foreach (var c in _sort_cols) {
+               int i = compare_cols(x[c.Item1], y[c.Item1]);
+
+            }
+            return 1;
+         }
+
+      }
+      public static IList<IBlueRow> get_sorted_row(IBlueRowCollection rows, IEnumerable<Tuple<string, bool>> sort_cols) {
+         List<IBlueRow> row_list = new List<IBlueRow>();
+         foreach (IBlueRow r in rows) {
+            row_list.Add(r);
+         }
+         var row_comparer = new RowComparer<IBlueRow>(sort_cols);
+         ParallelSort.QuicksortSequential<IBlueRow>(row_list, row_comparer);
+         return row_list;
+
+      }
    }
 }

src/DataStreamSuite/DataStreamCore/BlueData/BlueRowEqualityComparer.cs

@@ -0,0 +1,24 @@
+namespace Bts.DataStreamCore;
+
+using BCG;
+
+using System;
+using System.Collections.Generic;
+using System.Diagnostics.CodeAnalysis;
+using System.Linq;
+
+public class BlueRowEqualityComparer : IEqualityComparer<IBlueRow> {
+   public static BlueRowEqualityComparer Instance { get; } = new();
+
+   public bool Equals(IBlueRow x, IBlueRow y) =>
+      x.schema.columnInfo.SequenceEqual(y.schema.columnInfo) &&
+      x.Values.Cast<object>().SequenceEqual(y.Values.Cast<object>());
+
+   public int GetHashCode([DisallowNull] IBlueRow obj) {
+      var hash = new HashCode();
+      foreach (var v in obj.Values) {
+         hash.Add(v?.GetHashCode() ?? 0);
+      }
+      return hash.ToHashCode();
+   }
+}

src/DataStreamSuite/DataStreamCore/BlueData/BlueTable.cs

@@ -1,8 +1,15 @@
+using Bts.DataStreamCore;
+
+using Spark.DataStream;
+
 using System;
 using System.Collections;
 using System.Collections.Generic;
 using System.Data;
+using System.Diagnostics.CodeAnalysis;
+
 //using BCG.Utilities;
+
 using System.Linq;
 
 namespace BCG {
@@ -10,7 +17,7 @@ namespace BCG {
    /// Anologue of the dotNet DataTable.
    /// </summary>
    [Serializable]
-   public class BlueTable : IBlueRowCollection {
+   public class BlueTable : IBlueRowCollection, IDictionary<int, IBlueRow> {
       /// <summary>
       /// Hashtable indexed on "id" to speed up removes
       /// </summary>
@@ -86,6 +93,12 @@ namespace BCG {
                throw new BlueDataException("ID " + id.ToString() + " does not exist in table " + mySchema.tableName);
             return (IBlueRow)row;
          }
+         set {
+            if ((int)value["id"] != id) {
+               throw new ArgumentException("id arg not equal to row's actual id", nameof(id));
+            }
+            changeRow(value);
+         }
       }
 
       /// <summary>
@@ -202,6 +215,8 @@ namespace BCG {
 
       #endregion
 
+      public IDictionary<int, IBlueRow> AsDictionary() => this;
+
       #region ICollection members
       public virtual IEnumerator GetEnumerator() //{return myRows.GetEnumerator ();}
       { return myIdIndex.Values.GetEnumerator(); }
@@ -214,5 +229,55 @@ namespace BCG {
 
       public virtual int Count { get { return myIdIndex.Count; } }
       #endregion
+
+      #region IDictionary<int, IBlueRow> members
+      public void Add(int key, IBlueRow value) {
+         throw new NotImplementedException();
+      }
+
+      bool IDictionary<int, IBlueRow>.ContainsKey(int key) =>
+         containsID(key);
+
+      bool IDictionary<int, IBlueRow>.Remove(int key) =>
+         deleteRow(key);
+
+      bool IDictionary<int, IBlueRow>.TryGetValue(
+         int key, [MaybeNullWhen(false)] out IBlueRow value) =>
+         tryGetValue(key, out value);
+
+      void ICollection<KeyValuePair<int, IBlueRow>>.Add(KeyValuePair<int, IBlueRow> item) =>
+         addRow(item.Value);
+
+      void ICollection<KeyValuePair<int, IBlueRow>>.Clear() =>
+         clear();
+
+      bool ICollection<KeyValuePair<int, IBlueRow>>.Contains(KeyValuePair<int, IBlueRow> item) =>
+         tryGetValue(item.Key, out var r)
+         && BlueRowEqualityComparer.Instance.Equals(r, item.Value);
+
+      void ICollection<KeyValuePair<int, IBlueRow>>.CopyTo(
+         KeyValuePair<int, IBlueRow>[] array, int arrayIndex) {
+         foreach (DictionaryEntry de in myIdIndex) {
+            array[arrayIndex++] = new((int)de.Key, (IBlueRow)de.Value);
+         }
+      }
+
+      bool ICollection<KeyValuePair<int, IBlueRow>>.Remove(KeyValuePair<int, IBlueRow> item) =>
+         deleteRow(item.Key);
+
+      IEnumerator<KeyValuePair<int, IBlueRow>> IEnumerable<KeyValuePair<int, IBlueRow>>.GetEnumerator() {
+         foreach (DictionaryEntry de in myIdIndex) {
+            yield return new((int)de.Key, (IBlueRow)de.Value);
+         }
+      }
+
+      ICollection<int> IDictionary<int, IBlueRow>.Keys =>
+         GenericCollectionWrapper.Create<int>(myIdIndex.Keys);
+
+      ICollection<IBlueRow> IDictionary<int, IBlueRow>.Values =>
+         GenericCollectionWrapper.Create<IBlueRow>(myIdIndex.Values);
+
+      bool ICollection<KeyValuePair<int, IBlueRow>>.IsReadOnly => false;
+      #endregion
    }
 }

src/DataStreamSuite/DataStreamCore/BlueData/BlueTableImmutable.cs

@@ -0,0 +1,80 @@
+namespace Bts.DataStreamCore;
+
+using BCG;
+
+using System;
+using System.Collections;
+using System.Collections.Immutable;
+
+// I would have named this ImmutableBlueTable, but that was already taken.
+public class BlueTableImmutable : IBlueRowCollection, IImmutableRowCollection {
+   readonly ImmutableDictionary<int, IBlueRow> rows;
+
+   public BlueTableImmutable(IBlueTableSchema schema)
+      : this(schema, ImmutableDictionary<int, IBlueRow>.Empty) { }
+
+   public BlueTableImmutable(IBlueTableSchema schema, ImmutableDictionary<int, IBlueRow> rows) {
+      this.schema = schema;
+      this.rows = rows;
+   }
+
+   public int Count => rows.Count;
+   public bool IsSynchronized => false;
+   public IBlueTableSchema schema { get; }
+   public object SyncRoot => this;
+
+   public IBlueRow this[int id] => rows[id];
+
+   public bool containsID(int id) =>
+      rows.ContainsKey(id);
+
+   public void CopyTo(Array array, int index) {
+      foreach (var r in rows.Values) {
+         array.SetValue(r, index++);
+      }
+   }
+
+   public IEnumerator GetEnumerator() =>
+      rows.Values.GetEnumerator();
+
+   public DataPacketBuilder ToDataPacketBuilder() =>
+      new(this);
+
+   public class DataPacketBuilder {
+      readonly IBlueTableSchema schema;
+      readonly ImmutableDictionary<int, IBlueRow>.Builder table;
+      readonly ImmutableDictionary<int, IBlueRow>.Builder added =
+         ImmutableDictionary<int, IBlueRow>.Empty.ToBuilder();
+      readonly ImmutableDictionary<int, IBlueRow>.Builder removed =
+         ImmutableDictionary<int, IBlueRow>.Empty.ToBuilder();
+
+      public DataPacketBuilder(BlueTableImmutable table) {
+         schema = table.schema;
+         this.table = table.rows.ToBuilder();
+      }
+
+      public void AddOrUpdate(IBlueRow row) {
+         var id = row.Id();
+         added[id] = row;
+         if (table.TryGetValue(id, out var orig)) {
+            removed.Remove(id);
+         }
+      }
+
+      public void Remove(int id) {
+         added.Remove(id);
+         if (table.TryGetValue(id, out var orig)) {
+            removed[id] = orig;
+         }
+      }
+
+      public DataPacket ToDataPacket() {
+         table.RemoveRange(removed.Keys);
+         table.AddRange(added);
+         var newRows = new BlueTableImmutable(schema, table.ToImmutable());
+         var addedRows = new BlueTableImmutable(schema, added.ToImmutable());
+         var removedRows = new BlueTableImmutable(schema, removed.ToImmutable());
+         return new(newRows, addedRows, removedRows);
+      }
+   }
+}

src/DataStreamSuite/DataStreamCore/BlueData/BlueTableSchemaAdaptor.cs

@@ -0,0 +1,84 @@
+#nullable enable
+namespace Bts.DataStream;
+
+using BCG;
+
+using bts.collections;
+
+using System;
+using System.Collections.Generic;
+using System.Linq;
+
+public class BlueTableSchemaAdaptor<T> {
+   readonly string name;
+   readonly List<ColumnAdaptor> columns = [];
+   BlueTableSchema? schema;
+
+   public BlueTableSchemaAdaptor(string name) =>
+      this.name = name;
+
+   public IBlueTableSchema Schema =>
+      schema ??= new BlueTableSchema([.. columns.WrapSelectorRO(c => c.Info)], name);
+
+   public bool Contains(string name) =>
+      columns.Any(c => c.Info.colName == name);
+
+   public void AddColumn(string name, Func<T, int> getValue) =>
+      AddColumn(name, r => getValue(r), typeof(int));
+
+   public void AddColumn(string name, Func<T, int?> getValue) =>
+      AddColumn(name, r => getValue(r) ?? (object)DBNull.Value, typeof(int));
+
+   public void AddColumn(string name, Func<T, long> getValue) =>
+      AddColumn(name, r => getValue(r), typeof(long));
+
+   public void AddColumn(string name, Func<T, long?> getValue) =>
+      AddColumn(name, r => getValue(r) ?? (object)DBNull.Value, typeof(long));
+
+   public void AddColumn(string name, Func<T, double> getValue) =>
+      AddColumn(name, r => getValue(r), typeof(double));
+
+   public void AddColumn(string name, Func<T, double?> getValue) =>
+      AddColumn(name, r => getValue(r) ?? (object)DBNull.Value, typeof(double));
+
+   public void AddColumn(string name, Func<T, DateTime> getValue) =>
+      AddColumn(name, r => getValue(r), typeof(DateTime));
+
+   public void AddColumn(string name, Func<T, DateTime?> getValue) =>
+      AddColumn(name, r => getValue(r) ?? (object)DBNull.Value, typeof(DateTime));
+
+   public void AddColumn(string name, Func<T, string?> getValue) =>
+      AddColumn(name, r => getValue(r) ?? (object)DBNull.Value, typeof(string));
+
+   public void AddColumn(string name, Func<T, object?> getValue, Type dataType) {
+      if (!ColumnInfo.isAllowedType(dataType)) {
+         throw new ArgumentException(
+            $"'{dataType}' is not a supported column type", nameof(dataType));
+      }
+      // Invalidate cached schema (set it to null)
+      schema = null;
+      columns.Add(new ColumnAdaptor(new(name, columns.Count, dataType), r => getValue(r)));
+   }
+
+   public IBlueRow GetBlueRow(T item) {
+      var mbr = new MutableBlueRow(Schema);
+      foreach (var col in columns) {
+         mbr.setValue(col.Info.colNo, col.GetValue(item));
+      }
+      return mbr;
+   }
+
+   class ColumnAdaptor {
+      readonly Func<T, object?> getValue;
+
+      public ColumnAdaptor(ColumnInfo info, Func<T, object?> getValue) {
+         Info = info;
+         this.getValue = getValue;
+      }
+
+      public ColumnInfo Info { get; }
+
+      public object? GetValue(T row) =>
+         getValue(row);
+   }
+}

src/DataStreamSuite/DataStreamCore/BlueData/ColumnInfo.cs

@@ -74,13 +74,13 @@ namespace BCG
       //if it does, we are in serious trouble
       private short myLookupTableGroup;
 
-      [JsonPropertyName("index")]
+      [JsonPropertyName("colNo")]
       public int colNo{get{return myColNo;}}
 
-      [JsonPropertyName("name")]
+      [JsonPropertyName("colName")]
       public string colName {get{return myColName;}}
 
-      [JsonPropertyName("type")]
+      [JsonPropertyName("colType")]
       public Type colType {get{return myColType;}}
 
 

src/DataStreamSuite/DataStreamCore/BlueRowExtensions.cs

@@ -0,0 +1,11 @@
+namespace Bts.DataStreamCore;
+
+using BCG;
+
+using System;
+
+public static class BlueRowExtensions {
+   public static int Id(this IBlueRow row) =>
+      (int)(row["id"] ?? throw new ArgumentException(
+         "Row doesn't have an 'id' column", nameof(row)));
+}

src/DataStreamSuite/DataStreamCore/BlueRowJsonConverter.cs

@@ -1,63 +1,61 @@
-// #nullable enable
-// using BCG;
+#nullable enable
+using BCG;
 
-// using Spark.Core.Json;
+using System;
+using System.Text.Json;
+using System.Text.Json.Serialization;
 
-// using System;
-// using System.Text.Json;
-// using System.Text.Json.Nodes;
-// using System.Text.Json.Serialization;
+namespace Bts.DataStreamCore;
 
-// namespace Bts.DataStreamCore;
+class BlueRowJsonConverter : JsonConverter<IBlueRow> {
+   IBlueTableSchema? schema = null;
 
-// class BlueRowJsonConverter : JsonConverter<IBlueRow> {
-//    public override IBlueRow Read(
-//       ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options) {
-//       var row = JsonSerializer.Deserialize<JsonObject>(ref reader, options)!;
-//       var schemaj = row["schema"]!.GetValue<JsonObject>() ??
-//          throw new InvalidOperationException("Failed to read schema");
-//       // var columns = schemaj["columns"]
-//       var mbr = new MutableBlueRow(schema);
-//       int colNum = 0;
-//       switch (reader.TokenType) {
-//          case JsonTokenType.StartArray:
-//             while (reader.Read() && reader.TokenType != JsonTokenType.EndArray) {
-//                mbr.setValue(colNum++, readCellValue(ref reader));
-//                colNum++;
-//             }
-//             break;
-//          case JsonTokenType.StartObject:
-//             while (reader.Read() && reader.TokenType != JsonTokenType.EndObject) {
-//                if (reader.TokenType != JsonTokenType.PropertyName) {
-//                   throw new InvalidOperationException(
-//                      "Expected 'PropertyName' immediately after 'StartObject' token in row json");
-//                }
-//                var colName = reader.GetString();
-//                if (!reader.Read()) {
-//                   throw new InvalidOperationException(
-//                      "Failed to read token immediately after 'PropertyName' token in row json");
-//                }
-//                mbr.setValue(colName, readCellValue(ref reader));
-//             }
-//             break;
-//       }
-//       if (mbr.Count != schema.columnInfo.Length) {
-//          throw new InvalidOperationException(
-//             $"Row json array length '{mbr.Count}' does not match the schema's " +
-//             $"column length '{schema.columnInfo.Length}'.");
-//       }
-//       return mbr;
+   public override IBlueRow Read(
+      ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options) {
+      if (schema is null) {
+         throw new InvalidOperationException("Cannot read row from json because schema is null");
+      }
+      var mbr = new MutableBlueRow(schema);
+      int colNum = 0;
+      switch (reader.TokenType) {
+         case JsonTokenType.StartArray:
+            while (reader.Read() && reader.TokenType != JsonTokenType.EndArray) {
+               mbr.setValue(colNum++, readCellValue(ref reader));
+               colNum++;
+            }
+            break;
+         case JsonTokenType.StartObject:
+            while (reader.Read() && reader.TokenType != JsonTokenType.EndObject) {
+               if (reader.TokenType != JsonTokenType.PropertyName) {
+                  throw new InvalidOperationException(
+                     "Expected 'PropertyName' immediately after 'StartObject' token in row json");
+               }
+               var colName = reader.GetString();
+               if (!reader.Read()) {
+                  throw new InvalidOperationException(
+                     "Failed to read token immediately after 'PropertyName' token in row json");
+               }
+               mbr.setValue(colName, readCellValue(ref reader));
+            }
+            break;
+      }
+      if (mbr.Count != schema.columnInfo.Length) {
+         throw new InvalidOperationException(
+            $"Row json array length '{mbr.Count}' does not match the schema's " +
+            $"column length '{schema.columnInfo.Length}'.");
+      }
+      return mbr;
 
-//       static object? readCellValue(ref Utf8JsonReader reader) =>
-//          reader.TokenType switch {
-//             JsonTokenType.String => reader.GetString(),
-//             JsonTokenType.Number => reader.GetDouble(),
-//             JsonTokenType.True or JsonTokenType.False => reader.GetBoolean(),
-//             _ => throw new InvalidOperationException("Failed to read cell value"),
-//          };
-//    }
+      static object? readCellValue(ref Utf8JsonReader reader) =>
+         reader.TokenType switch {
+            JsonTokenType.String => reader.GetString(),
+            JsonTokenType.Number => reader.GetDouble(),
+            JsonTokenType.True or JsonTokenType.False => reader.GetBoolean(),
+            _ => throw new InvalidOperationException("Failed to read cell value"),
+         };
+   }
 
-//    public override void Write(Utf8JsonWriter writer, IBlueRow value, JsonSerializerOptions options) {
-//       throw new NotImplementedException();
-//    }
-// }
+   public override void Write(Utf8JsonWriter writer, IBlueRow value, JsonSerializerOptions options) {
+      throw new NotImplementedException();
+   }
+}

src/DataStreamSuite/DataStreamCore/DataPacketJsonConverter.cs

@@ -1,186 +1,155 @@
-// #nullable enable
-// using BCG;
+#nullable enable
+using BCG;
 
-// using Spark.Core.Json;
+using Spark.Core.Json;
 
-// using System;
-// using System.Collections.Generic;
-// using System.Text.Json;
-// using System.Text.Json.Nodes;
-// using System.Text.Json.Serialization;
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text.Json;
+using System.Text.Json.Serialization;
 
-// namespace Bts.DataStreamCore;
+namespace Bts.DataStreamCore;
 
-// public class DataStreamUpdateJsonConverter : JsonConverter<DataStreamUpdate> {
-//    /// <summary>
-//    /// Whether or not to include a field for current rows (i.e., snapshot of current state). By
-//    /// default, only incremental changes are included ('added' and 'deleted').
-//    /// </summary>
-//    public bool IncludeCurrent { get; set; }
+class DataPacketJsonConverter : JsonConverter<DataPacket> {
+   /// <summary>
+   /// Whether or not to include a field for current rows (i.e., snapshot of current state). By
+   /// default, only incremental changes are included ('added' and 'deleted').
+   /// </summary>
+   public bool IncludeCurrent { get; set; }
 
-//    /// <summary>
-//    /// Whether to treat rows as objects (name/value pairs), or arrays (list of values). Arrays
-//    /// result in smaller payloads, but aren't as easy to read as name/value pairs (for mere humans).
-//    /// </summary>
-//    public bool UseObjectRows { get; set; }
+   /// <summary>
+   /// Whether to treat rows as objects (name/value pairs), or arrays (list of values). Arrays
+   /// result in smaller payloads, but aren't as easy to read as name/value pairs (for mere humans).
+   /// </summary>
+   public bool UseObjectRows { get; set; }
 
-//    public override DataStreamUpdate? Read(ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options) {
-//       var dpj = JsonSerializer.Deserialize<JsonObject>(ref reader);
-//       if (dpj is null) {
-//          return null;
-//       }
-//       var tableName = dpj["tableName"]!.AsString();
-//       var columns = dpj["columns"]!.AsArray()
-//          .Select((c, i) => {
-//             var dataType = c!["type"]!.AsEnum<DataStreamDataType>();
-//             return (info: new ColumnInfo(c["name"]!.AsString(), i, dataType.ToType()), dataType);
-//          })
-//          .ToArray();
-//       var schema = new BlueTableSchema(columns.Cut1().ToArray(), tableName);
-//       return new() {
-//          TableName = tableName,
-//          AddedRows = ToRows(dpj.GetArrayOrEmpty("added")).ToList(),
-//          DeletedRows = ToRows(dpj.GetArrayOrEmpty("deleted")).ToList(),
-//          Columns = [.. schema.columnInfo]
-//       };
+   public override DataPacket? Read(ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options) {
+      if (!reader.ReadObject()) {
+         return null;
+      }
+      if (!reader.ReadPropertyName(out var prop)) {
+         return null;
+      }
+      if (!reader.Read()) {
+         return null;
+      }
+      if (prop == "added") {
+         var added = new List<MutableBlueRow>();
+         if (reader.TokenType != JsonTokenType.StartArray) {
+            return null;
+         }
+         while (true) {
+            if (!reader.Read()) {
+               break;
+            }
+            if (reader.TokenType != JsonTokenType.StartArray || !reader.Read()) {
+               return null;
+            }
+            switch (reader.TokenType) {
+               case JsonTokenType.String:
+                  // DAMNIT! Need the schema.
+                  break;
+            }
+         }
+      }
+      return null;
+   }
 
-//       IEnumerable<IBlueRow> ToRows(JsonArray rows) {
-//          foreach (var row in rows) {
-//             var mbr = new MutableBlueRow(schema);
-//             switch (row!.GetValueKind()) {
-//                case JsonValueKind.Object:
-//                   var rowObj = row.AsObject();
-//                   for (int i = 0; i < columns.Length; i++) {
-//                      SetCellValue(mbr, i, rowObj[columns[i].info.colName]!, columns[i].dataType);
-//                   }
-//                   break;
-//                case JsonValueKind.Array:
-//                   var rowArr = row.AsArray();
-//                   for (int i = 0; i < columns.Length; i++) {
-//                      SetCellValue(mbr, i, rowArr[i]!, columns[i].dataType);
-//                   }
-//                   break;
-//             }
-//             yield return mbr;
-//          }
-//       }
+   public override void Write(Utf8JsonWriter writer, DataPacket value, JsonSerializerOptions options) {
+      using var _ = writer.WriteObject();
+      using (writer.WriteArray("schema")) {
+         foreach (var col in value.schema.columnInfo) {
+            using (writer.WriteObject()) {
+               writer.WriteString("name", col.colName);
+               writer.WriteNumber("index", col.colNo);
+               // TODO: use language agnostic type names (e.g., 'string', 'integer', 'float', 'date' etc.)
+               writer.WriteString("type", col.colType.ToString());
+            }
+         }
+      }
 
-//       static void SetCellValue(MutableBlueRow mbr, int i, JsonNode cell, DataStreamDataType dataType) {
-//          switch (dataType) {
-//             case DataStreamDataType.String:
-//                mbr.setValue(i, cell.GetValue<string>());
-//                break;
-//             case DataStreamDataType.Int32:
-//                mbr.setValue(i, cell.GetValue<int>());
-//                break;
-//             case DataStreamDataType.Int64:
-//                mbr.setValue(i, cell.GetValue<long>());
-//                break;
-//             case DataStreamDataType.Single:
-//                mbr.setValue(i, cell.GetValue<float>());
-//                break;
-//             case DataStreamDataType.Double:
-//                mbr.setValue(i, cell.GetValue<double>());
-//                break;
-//          }
-//       }
-//    }
+      var cellWriters = value.schema.columnInfo.Select(getCellWriter).ToList();
+      using (writer.WriteArray("added")) {
+         foreach (IBlueRow row in value.addedRows) {
+            using (writer.WriteArray()) {
+               for (int i = 0; i < cellWriters.Count; i++) {
+                  cellWriters[i](row[i]);
+               }
+            }
+         }
+      }
+      using (writer.WriteArray("deleted")) {
+         foreach (IBlueRow row in value.deletedRows) {
+            using (writer.WriteArray()) {
+               for (int i = 0; i < cellWriters.Count; i++) {
+                  cellWriters[i](row[i]);
+               }
+            }
+         }
+      }
+      if (IncludeCurrent) {
+         using (writer.WriteArray("current")) {
+            foreach (IBlueRow row in value.newRows) {
+               using (writer.WriteArray()) {
+                  for (int i = 0; i < cellWriters.Count; i++) {
+                     cellWriters[i](row[i]);
+                  }
+               }
+            }
+         }
+      }
 
-//    public override void Write(Utf8JsonWriter writer, DataStreamUpdate value, JsonSerializerOptions options) {
-//       using var _ = writer.WriteObject();
-//       writer.WriteString("tableName", value.TableName);
-//       using (writer.WriteArray("columns")) {
-//          foreach (var col in value.Columns.empty_if_null()) {
-//             using (writer.WriteObject()) {
-//                writer.WriteString("name", col.colName);
-//                writer.WriteNumber("index", col.colNo);
-//                // TODO: use language agnostic type names (e.g., 'string', 'integer', 'float', 'date' etc.)
-//                writer.WriteString("type", col.colType.ToString());
-//             }
-//          }
-//       }
+      Action<object?> getCellWriter(ColumnInfo colInfo) {
+         var writeVal = getCellValueWriter(colInfo);
+         if (!UseObjectRows) {
+            return writeVal;
+         }
+         return v => {
+            writer.WritePropertyName(colInfo.colName);
+            writeVal(v);
+         };
+      }
 
-//       var cellWriters = value.Columns!.Select(getCellWriter).ToList();
-//       using (writer.WriteArray("added")) {
-//          foreach (IBlueRow row in value.AddedRows!) {
-//             using (writer.WriteArray()) {
-//                for (int i = 0; i < cellWriters.Count; i++) {
-//                   cellWriters[i](row[i]);
-//                }
-//             }
-//          }
-//       }
-//       using (writer.WriteArray("deleted")) {
-//          foreach (IBlueRow row in value.DeletedRows!) {
-//             using (writer.WriteArray()) {
-//                for (int i = 0; i < cellWriters.Count; i++) {
-//                   cellWriters[i](row[i]);
-//                }
-//             }
-//          }
-//       }
-//       if (IncludeCurrent) {
-//          using (writer.WriteArray("current")) {
-//             foreach (IBlueRow row in value.newRows) {
-//                using (writer.WriteArray()) {
-//                   for (int i = 0; i < cellWriters.Count; i++) {
-//                      cellWriters[i](row[i]);
-//                   }
-//                }
-//             }
-//          }
-//       }
+      Action<object?> getCellValueWriter(ColumnInfo colInfo) {
+         var t = colInfo.colType;
+         if (t == typeof(int)) {
+            return nullCheckWrapper(v => writer.WriteNumberValue((int)v));
+         }
+         if (t == typeof(uint)) {
+            return nullCheckWrapper(v => writer.WriteNumberValue((uint)v));
+         }
+         if (t == typeof(long)) {
+            return nullCheckWrapper(v => writer.WriteNumberValue((long)v));
+         }
+         if (t == typeof(ulong)) {
+            return nullCheckWrapper(v => writer.WriteNumberValue((ulong)v));
+         }
+         if (t == typeof(short)) {
+            return nullCheckWrapper(v => writer.WriteNumberValue((short)v));
+         }
+         if (t == typeof(ushort)) {
+            return nullCheckWrapper(v => writer.WriteNumberValue((ushort)v));
+         }
+         if (t == typeof(double)) {
+            return nullCheckWrapper(v => writer.WriteNumberValue((double)v));
+         }
+         if (t == typeof(float)) {
+            return nullCheckWrapper(v => writer.WriteNumberValue((float)v));
+         }
+         if (t == typeof(bool)) {
+            return nullCheckWrapper(v => writer.WriteBooleanValue((bool)v));
+         }
+         return nullCheckWrapper(v => writer.WriteStringValue(v?.ToString()));
 
-//       Action<object?> getCellWriter(ColumnInfo colInfo) {
-//          var writeVal = getCellValueWriter(colInfo);
-//          if (!UseObjectRows) {
-//             return writeVal;
-//          }
-//          return v => {
-//             writer.WritePropertyName(colInfo.colName);
-//             writeVal(v);
-//          };
-//       }
-
-//       Action<object?> getCellValueWriter(ColumnInfo colInfo) {
-//          var t = colInfo.colType;
-//          if (t == typeof(int)) {
-//             return nullCheckWrapper(v => writer.WriteNumberValue((int)v));
-//          }
-//          if (t == typeof(uint)) {
-//             return nullCheckWrapper(v => writer.WriteNumberValue((uint)v));
-//          }
-//          if (t == typeof(long)) {
-//             return nullCheckWrapper(v => writer.WriteNumberValue((long)v));
-//          }
-//          if (t == typeof(ulong)) {
-//             return nullCheckWrapper(v => writer.WriteNumberValue((ulong)v));
-//          }
-//          if (t == typeof(short)) {
-//             return nullCheckWrapper(v => writer.WriteNumberValue((short)v));
-//          }
-//          if (t == typeof(ushort)) {
-//             return nullCheckWrapper(v => writer.WriteNumberValue((ushort)v));
-//          }
-//          if (t == typeof(double)) {
-//             return nullCheckWrapper(v => writer.WriteNumberValue((double)v));
-//          }
-//          if (t == typeof(float)) {
-//             return nullCheckWrapper(v => writer.WriteNumberValue((float)v));
-//          }
-//          if (t == typeof(bool)) {
-//             return nullCheckWrapper(v => writer.WriteBooleanValue((bool)v));
-//          }
-//          return nullCheckWrapper(v => writer.WriteStringValue(v?.ToString()));
-
-//          Action<object?> nullCheckWrapper(Action<object> writeNonNull) =>
-//             v => {
-//                if (v is null) {
-//                   writer.WriteNullValue();
-//                } else {
-//                   writeNonNull(v);
-//                }
-//             };
-//       }
-//    }
-// }
+         Action<object?> nullCheckWrapper(Action<object> writeNonNull) =>
+            v => {
+               if (v is null) {
+                  writer.WriteNullValue();
+               } else {
+                  writeNonNull(v);
+               }
+            };
+      }
+   }
+}

src/DataStreamSuite/DataStreamCore/DataStreamCore.csproj

@@ -4,7 +4,6 @@
     <AllowUnsafeBlocks>true</AllowUnsafeBlocks>
     <TargetFramework>net8.0</TargetFramework>
     <Nullable>disable</Nullable>
-    <ImplicitUsings>enable</ImplicitUsings>
   </PropertyGroup>
 
   <ItemGroup>

src/DataStreamSuite/DataStreamCore/DataStreamDataType.cs

@@ -1,11 +0,0 @@
-namespace Bts.DataStreamCore;
-
-public enum DataStreamDataType {
-   Any,
-   Bytes,
-   String,
-   Int32,
-   Int64,
-   Single,
-   Double
-}

src/DataStreamSuite/DataStreamCore/DataStreamExtensions.cs

@@ -1,42 +0,0 @@
-#nullable enable
-
-using Bts.DataStreamCore;
-
-public static class DataStreamExtensions {
-   public static Type ToType(this DataStreamDataType dataType) =>
-      dataType switch {
-         DataStreamDataType.Any => typeof(object),
-         DataStreamDataType.String => typeof(string),
-         DataStreamDataType.Int32 => typeof(int),
-         DataStreamDataType.Int64 => typeof(long),
-         DataStreamDataType.Single => typeof(float),
-         DataStreamDataType.Double => typeof(double),
-         _ => throw new ArgumentException($"DataType '{dataType}' not supported.", nameof(dataType)),
-      };
-
-   public static DataStreamDataType FromType(Type type) {
-      if (type == typeof(object)) {
-         return DataStreamDataType.Any;
-      }
-      if (type == typeof(string)) {
-         return DataStreamDataType.String;
-      }
-      if (type == typeof(int)) {
-         return DataStreamDataType.Int32;
-      }
-      if (type == typeof(long)) {
-         return DataStreamDataType.Int64;
-      }
-      if (type == typeof(float)) {
-         return DataStreamDataType.Single;
-      }
-      if (type == typeof(double)) {
-         return DataStreamDataType.Double;
-      }
-      if (type == typeof(byte[])) {
-         return DataStreamDataType.Bytes;
-      }
-      throw new ArgumentException(
-         $"Unable to convert type '{type}' to {typeof(DataStreamDataType)}");
-   }
-}

src/DataStreamSuite/DataStreamCore/Usings.cs

@@ -1 +0,0 @@
-using System.Linq;

src/DataStreamSuite/DataStreamPublisher/Program.cs

@@ -1,4 +1,5 @@
-using BCG;
+using BCG;
+using BCG.Utilities.General;
 
 using bts;
 using bts.api.core;
@@ -13,6 +14,8 @@ using mbts.config;
 
 using System;
 using System.Collections.Generic;
+using System.IO;
+using System.Linq;
 using System.Threading.Tasks;
 
 await Parser.Default.ParseArguments<DataStreamPublisherConfig>(args)
@@ -69,6 +72,8 @@ static void start_echo_client() {
 void _publish_streams(IEnumerable<KeyValuePair<string, string>> prepublish_streams) {
    PubTableBroker pb_broker = new PubTableBroker();
    CloudStreamManager csm = new CloudStreamManager();
+   string csv_folder = @"C:\Temp\csvs_topublish";
+   CSVPublisher csv_publisher = new CSVPublisher(csv_folder, pb_broker);
    foreach (var kv in prepublish_streams) {
       var name = kv.Key;
       var exp = kv.Value;
@@ -89,3 +94,36 @@ void _publish_streams(IEnumerable<KeyValuePair<string, string>> prepublish_strea
 void on_change(object obj) {
    bts.Log.Gen.info($"DSPublisher on_change dp: {obj}");
 }
+
+public class CSVPublisher : IWorker {
+   private IDictionary<string, ViewPublisher> _pubs = new Dictionary<string, ViewPublisher>();
+   // read from config
+   private readonly string _csv_directory = @"C:\Temp\csvs_topublish\";
+
+   private PubTableBroker _pub_table_broker;
+   public CSVPublisher(string csv_directory, PubTableBroker pub_table_broker) {
+      _csv_directory = csv_directory;
+      _pub_table_broker = pub_table_broker;
+      MasterWorker.getInstance(false).addWorker(this, TimeSpan.FromSeconds(1));
+   }
+   public void doWork() {
+      var file_names = Directory.GetFiles(_csv_directory);
+      foreach (var fn in file_names) {
+         if (!_pubs.ContainsKey(fn)) {
+            _handle_new_file(fn);
+         }
+      }
+   }
+
+   private void _handle_new_file(string file_name) {
+      if (file_name.EndsWith(".csv")) {
+         ViewStream vs = new CSVReaderStream(file_name);
+         vs = new SchemaGuesserStream(vs);
+         var toks = file_name.Split('\\');
+         string table_name = toks.Last();
+         table_name = table_name.Remove(table_name.Length - ".csv".Length);
+         ViewPublisher vp = new ViewPublisher(table_name, _pub_table_broker, vs);
+         _pubs.Add(file_name, vp);
+      }
+   }
+}

src/DataStreamSuite/DataStreamSuite.sln

@@ -48,7 +48,7 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Bts.DataStream.Websocket",
 EndProject
 Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Bts.DataStream.WebsocketServer", "Bts.DataStream.WebsocketServer\Bts.DataStream.WebsocketServer.csproj", "{35823BE0-C883-476A-8B30-D6097CBA1A8D}"
 EndProject
-Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "spark_websocket", "..\client\spark_websocket\spark_websocket.csproj", "{EF492D73-C40F-4EEF-B917-4ADA7B30D728}"
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "spark_websocket", "..\client\spark_websocket\spark_websocket.csproj", "{09B0FF33-D34F-4BB4-BE96-B7D35960BB3B}"
 EndProject
 Global
 	GlobalSection(SolutionConfigurationPlatforms) = preSolution
@@ -151,9 +151,9 @@ Global
 		{35823BE0-C883-476A-8B30-D6097CBA1A8D}.Debug|Any CPU.Build.0 = Debug|Any CPU
 		{35823BE0-C883-476A-8B30-D6097CBA1A8D}.Release|Any CPU.ActiveCfg = Release|Any CPU
 		{35823BE0-C883-476A-8B30-D6097CBA1A8D}.Release|Any CPU.Build.0 = Release|Any CPU
-		{EF492D73-C40F-4EEF-B917-4ADA7B30D728}.Debug|Any CPU.ActiveCfg = Debug|x64
-		{EF492D73-C40F-4EEF-B917-4ADA7B30D728}.Debug|Any CPU.Build.0 = Debug|x64
-		{EF492D73-C40F-4EEF-B917-4ADA7B30D728}.Release|Any CPU.ActiveCfg = Release|x64
-		{EF492D73-C40F-4EEF-B917-4ADA7B30D728}.Release|Any CPU.Build.0 = Release|x64
+		{09B0FF33-D34F-4BB4-BE96-B7D35960BB3B}.Debug|Any CPU.ActiveCfg = Debug|x64
+		{09B0FF33-D34F-4BB4-BE96-B7D35960BB3B}.Debug|Any CPU.Build.0 = Debug|x64
+		{09B0FF33-D34F-4BB4-BE96-B7D35960BB3B}.Release|Any CPU.ActiveCfg = Release|x64
+		{09B0FF33-D34F-4BB4-BE96-B7D35960BB3B}.Release|Any CPU.Build.0 = Release|x64
 	EndGlobalSection
 EndGlobal

src/DataStreamSuite/ExpressionTester/ExpressionTester.csproj

@@ -1,14 +0,0 @@
-<Project Sdk="Microsoft.NET.Sdk">
-
-  <ItemGroup>
-    <ProjectReference Include="..\ExpressionTests\ExpressionTests.csproj" />
-  </ItemGroup>
-
-  <PropertyGroup>
-    <OutputType>Exe</OutputType>
-    <TargetFramework>net8.0</TargetFramework>
-    <ImplicitUsings>enable</ImplicitUsings>
-    <Nullable>enable</Nullable>
-  </PropertyGroup>
-
-</Project>

src/DataStreamSuite/ExpressionTester/Program.cs

@@ -1,9 +0,0 @@
-// See https://aka.ms/new-console-template for more information
-using ExpressionTests;
-
-Console.WriteLine("Hello, World!");
-
-var tester = new DataStreamExpressionTests();
-tester.Setup();
-tester.Test();
-tester.TestLineComments();

src/DataStreamSuite/ExpressionTests/ArithmeticExpressionTest.cs

@@ -1,23 +0,0 @@
-namespace ExpressionTests;
-
-using Bts.ArithmeticExpressions;
-
-public class ArithmeticExpressionTests {
-   [SetUp]
-   public void Setup() {
-   }
-
-   [Test]
-   public void Test() {
-      const string SIMPLE_SUM = "rv = 3 + 4";
-      var interpreter = ArithmeticProgramParser.Parse(SIMPLE_SUM);
-      Assert.NotNull(interpreter);
-      var env = new Dictionary<string, object?>();
-      var result = interpreter.Evaluate(env);
-      Assert.IsTrue(result.TryGetValue("rv", out var res) && Equals(res, 7.0));
-   }
-   
-   public void SimpleOperatorTest() {
-      // Assert.IsTrue(BooleanExpression);
-   }
-}

src/DataStreamSuite/ExpressionTests/DataStreamExpressionTests.cs

@@ -1,84 +0,0 @@
-namespace ExpressionTests;
-
-using BCG;
-
-using Bts.DataStreamExpressions;
-using Bts.ExpressionTests;
-
-using Spark.Core;
-
-public class DataStreamExpressionTests {
-   static string simpleExpression => "literal(\"n\", \"2.0\")";
-
-   [SetUp]
-   public void Setup() { }
-
-   void validateExpression(string expressionStr, bool expectSuccess = true) {
-      var env = new Dictionary<string, StatementExpression>();
-      if (expectSuccess) {
-         var expression = DataStreamExpressionsInterpreter.Evaluate(expressionStr);
-         Assert.NotNull(expression.getView(env));
-      } else {
-         try {
-            // var expression = DataStreamExpressionsInterpreter.TryEvaluate(expressionStr);
-            // Assert.IsNull(expression);
-         } catch {
-         }
-      }
-   }
-
-   void validateExpression(IEnumerable<string> lines, bool expectSuccess = true) {
-      validateExpression("\n".join(lines), expectSuccess);
-   }
-
-   [Test]
-   public void TestLineComments() {
-      var commentBodies = new[] { "comment", "comment w/ odd symbols $@!&*.;: (i.e., those!)" };
-      // Create 2 layers of permutations.
-      //    1. with and without white space between '//' and comment body
-      //    2. with and without leading white space
-      //    3. with and without trailing white space
-      var comments = commentBodies
-         .SelectMany(c => new[] { c, $" {c}", $" {c} ", $"{c} " })
-         .SelectMany(c => new[] { $"//{c}", $"// {c}" })
-         .SelectMany(c => new[] { c, $" {c}", $"{c} ", $" {c} " });
-      foreach (var c in comments) {
-         impl(c);
-      }
-      void impl(string comment) {
-         // Test single line comment (evaluation should fail)
-         var expLines = new List<string>() { comment };
-         validateExpression(expLines, false);
-
-         // Add valid expression after comment (evaluation should succeed)
-         expLines.Add(simpleExpression);
-         validateExpression(expLines);
-
-         // Add comment after expression 
-         expLines.Add(comment);
-         validateExpression(expLines);
-         expLines.Add(string.Empty);
-         validateExpression(expLines);
-      }
-   }
-
-   [Test]
-   public void Test() {
-      var expressionLines = new List<string>() {
-         "// fake comment w/ odd symbols $@!&*.;: (i.e., those!)",
-         "literal(\"n\", \"2.0\")",
-         "// com"
-      };
-      var expressionStr = "\n".join(expressionLines);
-      var expression = DataStreamExpressionsInterpreter.Evaluate(expressionStr + "\n");
-      var env = new Dictionary<string, StatementExpression>();
-      foreach (var s in expression.Statements) {
-         Console.WriteLine("Statement: " + s.ToString());
-      }
-      Assert.NotNull(expression.getView(env));
-   }
-
-   public void SimpleOperatorTest() {
-      // Assert.IsTrue(BooleanExpression);
-   }
-}

src/DataStreamSuite/ExpressionTests/EvaluatesRecursively.cs

@@ -1,7 +0,0 @@
-namespace Bts.ExpressionTests;
-
-class EvaluatesRecursively {
-   public void SetEvaluator(ExpressionEvaluator evaluator) {
-      
-   }
-}

src/DataStreamSuite/ExpressionTests/EvaluatesToFoo.cs

@@ -1,24 +0,0 @@
-namespace Bts.ExpressionTests;
-
-class EvaluatesToFoo : IPropertyEvaluator {
-   public Value? GetValue(string prop) {
-      if (!prop.StartsWith("Foo")) {
-         return null;
-      }
-      if (prop == "Foo.number") {
-         return new Value(42.0);
-      }
-      if (prop == "Foo.list") {
-         var values = new List<Value> {
-               new("foo"),
-               new("bar"),
-               new("baz")
-            };
-         return new(values);
-      }
-      if (prop == "Foo.bool") {
-         return new Value(true);
-      }
-      return new Value("Foo");
-   }
-}

src/DataStreamSuite/ExpressionTests/ExpressionEvaluator.cs

@@ -1,13 +0,0 @@
-namespace Bts.ExpressionTests;
-
-class ExpressionEvaluator {
-   readonly IPropertyEvaluator propEval;
-
-   public ExpressionEvaluator(IPropertyEvaluator propEval) {
-      this.propEval = propEval;
-   }
-   
-   public bool Eval(string expression) {
-      throw new NotImplementedException();
-   }
-}

src/DataStreamSuite/ExpressionTests/ExpressionTests.csproj

@@ -1,27 +0,0 @@
-<Project Sdk="Microsoft.NET.Sdk">
-
-  <PropertyGroup>
-    <TargetFramework>net8.0</TargetFramework>
-    <ImplicitUsings>enable</ImplicitUsings>
-    <Nullable>enable</Nullable>
-
-    <IsPackable>false</IsPackable>
-    <IsTestProject>true</IsTestProject>
-  </PropertyGroup>
-
-  <ItemGroup>
-    <PackageReference Include="Microsoft.NET.Test.Sdk" Version="17.5.0" />
-    <PackageReference Include="NUnit" Version="3.13.3" />
-    <PackageReference Include="NUnit3TestAdapter" Version="4.4.2" />
-    <PackageReference Include="NUnit.Analyzers" Version="3.6.1" />
-    <PackageReference Include="coverlet.collector" Version="3.2.0" />
-    <PackageReference Include="OneOf" Version="3.0.263" />
-  </ItemGroup>
-
-  <ItemGroup>
-    <ProjectReference Include="..\ArithmeticExpressions\ArithmeticExpressions.csproj" />
-    <ProjectReference Include="..\BooleanExpressions\BooleanExpressions.csproj" />
-    <ProjectReference Include="..\DataStreamExpressions\DataStreamExpressions.csproj" />
-  </ItemGroup>
-
-</Project>

src/DataStreamSuite/ExpressionTests/IPropertyEvaluator.cs

@@ -1,5 +0,0 @@
-namespace Bts.ExpressionTests;
-
-interface IPropertyEvaluator {
-   Value? GetValue(string property);
-}

src/DataStreamSuite/ExpressionTests/Usings.cs

@@ -1 +0,0 @@
-global using NUnit.Framework;

src/DataStreamSuite/ExpressionTests/Value.cs

@@ -1,20 +0,0 @@
-namespace Bts.ExpressionTests;
-
-using OneOf;
-
-enum ValueType {
-   Boolean,
-   Integer,
-   Real,
-   String,
-   List,
-   Any
-}
-
-class Value {
-   public Value(OneOf<bool, int, double, string, List<Value>>? value = null) {
-      this.value = value ?? new();
-   }
-
-   public OneOf<bool, int, double, string, List<Value>> value { get; }
-}

src/DataStreamSuite/ExpressionTests/expression_parser_tests.cpp

@@ -1,179 +0,0 @@
-#include "core/bts_core/utest/utest.h"
-
-#include "../IPropertyEvaluator.h"
-#include "../ExpressionEvaluator.h"
-#include "../ParserExceptions.h"
-
-#include "core/bts_core/utils/BTS_SharedPtr.h"
-#include <utility>
-
-using namespace bts::expression;
-
-namespace bts {
-
-   // Simple property evaluator for testing
-   class EvaluatesToFoo : public IPropertyEvaluator
-   {
-      public:
-      EvaluatesToFoo()
-      { }
-      ~EvaluatesToFoo()
-      { }
-      BTS_NO_COPY(EvaluatesToFoo);
-
-      bool get_value(const std::string& prop, Value* out_value) OVERRIDE
-      {
-         bool valid = prop.substr(0,3) == "Foo";
-         if (valid && out_value != nullptr)
-         {
-            if (prop == "Foo.number")
-            {
-               *out_value = Value(42.0);
-            }
-            else if (prop == "Foo.list")
-            {
-               std::vector<Value> values;
-               values.push_back(Value(std::string("foo")));
-               values.push_back(Value(std::string("bar")));
-               values.push_back(Value(std::string("baz")));
-               *out_value = Value(values);
-            }
-            else if (prop == "Foo.bool")
-            {
-               *out_value = Value(true);
-            }
-            else
-            {
-               // Foo.*
-               *out_value = Value(std::string("Foo"));
-            }
-         }
-
-         return valid;
-      }
-   };
-
-   class EvaluatesRecursively : public IPropertyEvaluator
-   {
-   public:
-      EvaluatesRecursively()
-         : _e(nullptr)
-      { }
-      ~EvaluatesRecursively()
-      { }
-      BTS_NO_COPY(EvaluatesRecursively);
-
-      void set_evaluator(ExpressionEvaluator* e)
-      {
-         _e = e;
-      }
-
-      bool get_value(const std::string& prop, Value* out_value) OVERRIDE
-      {
-         std::string recur = "42 == " + prop;
-         if (out_value)
-         {
-            *out_value = Value(_e->eval(recur));
-         }
-         return true;
-      }
-
-      ExpressionEvaluator* _e;
-   };
-
-   struct expression_tests : public TestSuite
-   {
-      expression_tests() { }
-      ~expression_tests() { }
-      BTS_NO_COPY(expression_tests);
-
-      void setup()
-      {
-      }
-
-      void teardown()
-      {
-      }
-
-      void simple_operator_test()
-      {
-         ExpressionEvaluator e(BTS_SharedPtr<IPropertyEvaluator> (new EvaluatesToFoo()));
-         test_assert(e.eval("!!true"));
-         test_assert(e.eval("3 == 3"));
-         test_assert(e.eval("4 > 3"));
-         test_assert(e.eval("3 < 6"));
-         test_assert(e.eval("3 <= 3.5"));
-         test_assert(e.eval("3 in [1,2,3]"));
-         test_assert(!e.eval("3 in [1,2,4]"));
-         test_assert(e.eval("'oo' in Foo.foo"));
-         test_assert(e.eval("'oo' == 'oo'"));
-         test_assert(e.eval("'afoobar' ~= 'foo.*'"));
-         test_assert(!e.eval("'afoobar' ~= '^foo.*'"));
-      }
-
-      void test_bool_conversion()
-      {
-         ExpressionEvaluator e(BTS_SharedPtr<IPropertyEvaluator> (new EvaluatesToFoo()));
-         test_assert(e.eval("true"));
-         test_assert(e.eval("1"));
-         test_assert(e.eval("1.5"));
-         test_assert(e.eval("'true string'"));
-         //test_assert(e.eval("[1,2,3]")); // list not valid here - only after "in"
-         test_assert(e.eval("Foo.x"));
-         test_assert(e.eval("Foo.list"));
-         test_assert(!e.eval("''"));
-      }
-
-      void property_validator_test()
-      {
-         ExpressionEvaluator e(BTS_SharedPtr<IPropertyEvaluator> (new EvaluatesToFoo()));
-         test_assert(e.check_valid("Foo.bar == 'Foo' or 'x' in [Foo.x, Foo.y, 'z']", nullptr));
-         std::string bad_property;
-         test_assert(!e.check_valid("Foo.bar == 'Foo' or 'x' in [Fuu.x, Foo.y, 'z']", &bad_property));
-         test_assert(bad_property == "Fuu.x");
-      }
-
-      void more_complex_test()
-      {
-         ExpressionEvaluator e(BTS_SharedPtr<IPropertyEvaluator> (new EvaluatesToFoo()));
-
-         test_assert(e.eval("Foo.bar == 'Foo' and 'b' in ['a', 'b', 'c', Foo.bar]"));
-         test_assert(e.eval("Foo.number > 2 or (3.5 <= 7 and Foo.number == 3) or 'x' in ['a', 'b', 'c']"));
-         test_assert(!e.eval("(3.5 <= 7 and Foo.number == 3) or 'x' in ['a', 'b', 'c']"));
-         test_assert(e.eval("'foo' in Foo.list and Foo.number == 42.0"));
-         test_assert(e.eval("!!('oo' in Foo.foo)"));
-         test_assert(e.eval("!('ooo' in Foo.foo)"));
-         test_assert(e.eval("!false in [false, false, false, false, true, Foo.bool]"));
-      }
-
-      void recursive_test()
-      {
-         BTS_SharedPtr<EvaluatesRecursively> badprops (new EvaluatesRecursively());
-         ExpressionEvaluator e(badprops);
-         badprops->set_evaluator(&e);
-
-         // Try to get the 'fuzzy'property recursively
-         bool success = false;
-         try
-         {
-            test_assert(e.eval("fuzzy == 42"));
-         }
-         catch(RecursivePropertyGetException)
-         {
-            success = true;
-         }
-         test_assert(success);
-      }
-
-   };
-
-
-   BEGIN_TEST_SUITE(expression_tests)
-      ADD_TEST(simple_operator_test);
-      ADD_TEST(test_bool_conversion);
-      ADD_TEST(property_validator_test);
-      ADD_TEST(more_complex_test);
-      ADD_TEST(recursive_test);
-   END_TEST_SUITE
-
-}

src/DataStreamSuite/PubSub/Testing/SimpleTCPServer.cs

@@ -2,16 +2,20 @@ using System;
 using System.Collections.Generic;
 using bts;
 using System.Threading;
+using System.Threading.Tasks;
 
 namespace BCG {
 
    public class SimpleTCPServer : ITCPConnectionHandler, ITCPMessageHandler {
+      readonly TimeSpan? echoDelay;
+
       int _n_msgs_received = 0;
       bool _isDisposed = false;
       AsyncTCPConnectionAcceptor_APM _acceptor;
       IDictionary<RemoteConnectionInfo, ITCPConnection> _conn_map = new Dictionary<RemoteConnectionInfo, ITCPConnection>();
 
-      public SimpleTCPServer(int port) {
+      public SimpleTCPServer(int port, TimeSpan? echoDelay = null) {
+         this.echoDelay = echoDelay;
          _acceptor = new AsyncTCPConnectionAcceptor_APM(port, this);
       }
 
@@ -34,22 +38,27 @@ namespace BCG {
       public void onData(RemoteMessage msg) {
          Console.WriteLine($"SimpleTCPServer.onData msg: {msg}");
          RemoteConnectionInfo rci = msg.remoteConnectionInfo;
-         ITCPConnection this_conn;
-         bool found = _conn_map.TryGetValue(rci, out this_conn);
-         if (!found) {
+         if (!_conn_map.TryGetValue(rci, out var this_conn)) {
             Console.WriteLine("Could not find connection !!!!!!");
             return;
          }
          _n_msgs_received++;
          Console.WriteLine($"\n\nSimpleTCPServerAsync.onData msgs_received: {_n_msgs_received} msg: {msg} ci: {rci} NOW echoing it back\n\n");
          if (msg.messageType == RemoteMessageType.Data) {
-            this_conn.send(msg.message);
+            Echo();
          } else {
             if (msg.messageType == RemoteMessageType.Disconnection) {
                var ci = msg.remoteConnectionInfo;
                _conn_map.Remove(ci);
             }
          }
+
+         async void Echo() {
+            if (echoDelay.HasValue) {
+               await Task.Delay(echoDelay.Value);
+            }
+            this_conn.send(msg.message);
+         }
       }
 
       public bool isDisposed => _isDisposed;

src/DataStreamSuite/PubSubServer/Configurations/SimpleEchoServer.json

@@ -0,0 +1,4 @@
+{
+  "simple_echo_server": true,
+  "echo_delay": "00:00:02"
+}

src/DataStreamSuite/PubSubServer/Program.cs

@@ -17,9 +17,10 @@ Parser.Default.ParseArguments<PubSubServerConfig>(args)
 
 int Run(PubSubServerConfig cfg) {
    try {
+      cfg.MergeWithConfigFile();
       Configuration.initialize();
       return cfg.SimpleEchoServer ?? false ?
-         RunEchoServer(cfg.Port ?? 12345) :
+         RunEchoServer(cfg.Port ?? 12345, cfg.EchoDelay) :
          RunPubSubServer(cfg);
    } catch (Exception ex) {
       // Write the error to console first, in case the logging system is compromised.
@@ -52,11 +53,11 @@ int RunPubSubServer(PubSubServerConfig cfg) {
    return 0;
 }
 
-int RunEchoServer(short port) {
+int RunEchoServer(short port, TimeSpan? echoDelay) {
    Console.WriteLine("Running as TCP Echo server, Hit enter to continue");
    Console.ReadLine();
    Console.WriteLine($"ready to listen to port: {port}");
-   var sts = new SimpleTCPServer(port);
+   var sts = new SimpleTCPServer(port, echoDelay);
    sts.init();
    Console.CancelKeyPress += OnCancelKeyPress;
    while (true) {

src/DataStreamSuite/PubSubServer/PubSubServerConfig.cs

@@ -38,9 +38,13 @@ public class PubSubServerConfig {
    public string? Sparkd { get; set; }
 
    [JsonPropertyName("simple_echo_server")]
-   [Option("simple-echo-server", HelpText = "Run a simple echo server (won'connect to sparkd)")]
+   [Option("simple-echo-server", HelpText = "Run a simple echo server (won't connect to sparkd)")]
    public bool? SimpleEchoServer { get; set; }
 
+   [JsonPropertyName("echo_delay")]
+   [Option("echo-delay", HelpText = "The amount of time to delay echo replies (must run with --simple-echo-server)")]
+   public TimeSpan? EchoDelay { get; set; }
+
    [JsonIgnore]
    [Option(
       "config",

src/DataStreamSuite/SharedStreams/Business/MarketData/Charting/TradeBarsStream.cs

@@ -0,0 +1,161 @@
+using BCG;
+using BCG.Utilities.General;
+
+using System;
+
+namespace bts {
+    [DataStreamUserRespresentation("tradebars")]
+    public class TradeBarsStream : ViewStream, IWorker {
+
+        public class TradeBar {
+            static double[] calculate_log_returns(double[] prices) {
+                // Calculate logarithmic returns
+                double[] returns = new double[prices.Length - 1];
+                for (int i = 1; i < prices.Length; i++) {
+                    returns[i - 1] = Math.Log(prices[i] / prices[i - 1]);
+                }
+                return returns;
+            }
+
+            static double calculate_volatility(double[] returns) {
+                // Calculate standard deviation of logarithmic returns
+                double mean = returns.Average();
+                double sum_of_squares = returns.Sum(x => Math.Pow(x - mean, 2));
+                double variance = sum_of_squares / (returns.Length - 1);
+                double volatility = Math.Sqrt(variance);
+
+                return volatility;
+            }
+            public void add_trade(IBlueRow r) {
+                double price = r.get<double>("price");
+                //prices.Add(price);
+                double qty = r.get<double>(col_name: "qty");
+                if (n_entries == 0) {
+                    open = price;
+                    low = price < low ? price : low;
+                    high = price > high ? price : high;
+                }
+                n_entries++;
+                volume += qty;
+                vwap_sum += price * qty;
+                close = price;
+                if ("Buy".Equals(r.get<string>("aggressor"))) {
+                    n_buys_aggs++;
+                } else {
+                    n_sell_aggs++;
+                }
+            }
+            private List<double> prices = new List<double>();
+            public readonly string security;
+            public double low = Double.MaxValue, high, open, close, median, std_dev, vwap_sum, volume;
+            public uint n_buys_aggs, n_sell_aggs, n_entries;
+            public DateTime bar_start = DateTime.Now;
+            public TradeBar(string security_) {
+                security = security_;
+            }
+            public double volatility {
+                get {
+                    //if (prices.Count == 1)
+                    return 0.0;
+                    var log_returns = calculate_log_returns(prices.ToArray());
+                    return calculate_volatility(log_returns);
+                }
+            }
+        }
+
+        private readonly int _bar_period_seconds;
+        private readonly ViewStreamDatabase _vsd;
+        private ViewStream _timesales_stream;
+        private ViewStream _bbo_stream;
+        private IDictionary<uint, TradeBar> _sec_bar_map = new Dictionary<uint, TradeBar>();
+        private bool _dirty = false;
+        private TimeSpan _fire_period;
+        private DateTime _last_fire_time = DateTime.Now;
+        private BlueTable _table = new BlueTable(BlueDataUtility.form_schema("TradeBars",
+        Tuple.Create("id", TypeCode.Int32),
+        Tuple.Create("bar_start", TypeCode.DateTime),
+        Tuple.Create("bar_end", TypeCode.DateTime),
+        Tuple.Create("day_of_week", TypeCode.String),
+        Tuple.Create("n_trades", TypeCode.UInt32),
+        Tuple.Create("security", TypeCode.String),
+        Tuple.Create("sec_key", TypeCode.UInt32),
+        Tuple.Create("low", TypeCode.Double),
+        Tuple.Create("high", TypeCode.Double),
+        Tuple.Create("open", TypeCode.Double),
+        Tuple.Create("close", TypeCode.Double),
+        Tuple.Create("vwap_sum", TypeCode.Double),
+        Tuple.Create("volume", TypeCode.Double),
+        Tuple.Create("n_buy_aggs", TypeCode.UInt32),
+        Tuple.Create("n_sell_aggs", TypeCode.UInt32),
+        Tuple.Create("volatility", TypeCode.Double)));
+
+
+        public TradeBarsStream(int bar_period_seconds)
+        : this(bar_period_seconds, ViewStreamDatabase.instance) { }
+        public TradeBarsStream(int bar_period_seconds, ViewStreamDatabase vsd) {
+            _vsd = vsd;
+            _bar_period_seconds = bar_period_seconds;
+            _timesales_stream = new TimeSalesStream(10000, vsd);
+            _fire_period = TimeSpan.FromSeconds(bar_period_seconds);
+            _bbo_stream = new BBOStream(_vsd);
+        }
+
+        protected override void myOnFirstSubscriber() {
+            _timesales_stream.onChange += _on_timesales;
+            _bbo_stream.onChange += _on_bbo;
+            _vsd.addWorker(this);
+        }
+
+        protected override void myOnLastSubscriber() {
+            _vsd.removeWorker(this);
+            _timesales_stream.onChange -= _on_timesales;
+            _bbo_stream.onChange -= _on_bbo;
+        }
+
+        public void doWork() {
+            if (DateTime.Now - _last_fire_time > _fire_period) {
+                if (_dirty) {
+                    _dirty = false;
+                    _last_fire_time = DateTime.Now;
+                    BlueTable added_rows = new BlueTable(_table.schema);
+                    foreach (var kv in _sec_bar_map) {
+                        var sec_key = kv.Key;
+                        var tbar = kv.Value;
+                        IBlueRow new_row = new BlueRow(_table.schema, _table.Count, tbar.bar_start, DateTime.Now,
+                        tbar.bar_start.DayOfWeek.ToString(),
+                                                        tbar.n_entries, tbar.security, sec_key, tbar.low, tbar.high, tbar.open,
+                                                        tbar.close, tbar.vwap_sum, tbar.volume, tbar.n_sell_aggs, tbar.n_sell_aggs,
+                                                        tbar.volatility);
+                        added_rows.addRow(new_row);
+                        _table.addRow(new_row);
+                    }
+                    _sec_bar_map.Clear();
+                    DataPacket dp_out = new DataPacket(_table, added_rows, new BlueTable(_table.schema));
+                    myFireEvent(dp_out);
+                }
+            }
+        }
+
+        private void _on_timesales(object obj) {
+            DataPacket dp = obj as DataPacket;
+            foreach (IBlueRow r in dp.addedRows) {
+                uint sec_key = r.get<UInt32>("sec_key");
+                TradeBar tbar = null;
+                if (!_sec_bar_map.TryGetValue(sec_key, out tbar)) {
+                    tbar = new TradeBar(r.get<string>("sec"));
+                    _sec_bar_map.Add(sec_key, tbar);
+                }
+                _dirty = true;
+                tbar.add_trade(r);
+            }
+
+
+        }
+
+        private void _on_bbo(object obj) {
+
+        }
+    }
+
+
+}

src/DataStreamSuite/SharedStreams/Business/MarketData/PriceStatsStream.cs

@@ -1,127 +1,130 @@
-namespace BCG {
-   using BCG.Utilities.General;
-
-   using bts.api;
-   using bts.api.domain;
-   using bts.api.synthetic;
-   using bts.md;
-   using bts.spark;
-   using bts.utils;
-
-   using mbts;
-   using mbts.co2;
-   using mbts.config;
-   using mbts.spark;
-   using mbts.ui;
-
-   using Spark.Core.Data;
-   using Spark.Core.Disposables;
-   using Spark.Core.Reactive;
-
-   using System;
-   using System.Collections.Concurrent;
-   using System.Collections.Generic;
-   using System.Collections.ObjectModel;
-   using System.Linq;
-   using System.Reactive.Linq;
-   using bts.api.core;
-   using Spark.Api.MarketData;
-
-   using SecurityKey = System.UInt32;
-   using System.Data;
-   using System.Reflection.Metadata;
-
-   [DataStreamUserRespresentation("price_stats")]
-   public class PriceStatsStream : ViewStream, IWorker {
-      private readonly TimeSpan _poll_period = TimeSpan.FromMilliseconds(500);
-      private DateTime _last_poll_time = DateTime.MinValue;
-      private ViewStreamDatabase _vsd;
-      private int _id = 0;
-      IReadOnlyDictionary<SecurityKey, PriceStats> _cached_state = null;
-      MarketDataClient _md_client;
-      private bool _dirty = false;
-      private readonly IBlueTableSchema _pstats_scheam = BlueDataUtility.form_schema("bbo",
-                                                Tuple.Create("id", TypeCode.Int32),
-                                                Tuple.Create("sec_key", TypeCode.Int32),
-                                                Tuple.Create("sec", TypeCode.String),
-                                                Tuple.Create("open", TypeCode.UInt32),
-                                                Tuple.Create("close", TypeCode.Double),
-                                                Tuple.Create("settlement", TypeCode.Single));
-      IDisposable? subscription;
-
-      // TODO: add cons for symbol, exchange etc 
-      public PriceStatsStream()
-      : this(ViewStreamDatabase.instance) {
-      }
-
-      public PriceStatsStream(ViewStreamDatabase vsd) {
-         bts.Log.Gen.info("PriceStatsStream.cons");
-         _vsd = vsd;
-         _md_client = new MarketDataClient(new MDSource(TimeSpan.FromSeconds(0.5)));
-      }
-
-      protected override void myOnFirstSubscriber() {
-         _vsd.addWorker(this);
-         subscription = _md_client.price_stats().Subscribe(_on_price_stats);
-      }
-
-      private void _log_updates(TableUpdate<PriceStats, uint> update) {
-         int i = 0;
-         // foreach (BookTuple bt in update.added) {
-         //    bts.Log.Gen.info($"PriceStatsStream added: idx: {i++} bid: {bt.book.bids[0].price} / {bt.book.bids[0].size} ask: {bt.book.asks[0].price} / {bt.book.asks[0].size}");
-         // }
-         // foreach (Change<BookTuple> c in update.changes) {
-         //    var bt = c.New;
-         //    bts.Log.Gen.info($"PriceStatsStream changes: idx: {i} bid: {bt.book.bids[0].price} / {bt.book.bids[0].size} ask: {bt.book.asks[0].price} / {bt.book.asks[0].size}");
-         // }
-      }
-
-      private void _on_price_stats(TableUpdate<PriceStats, uint> update) {
-         _log_updates(update);
-         // just cacching in here, will populate the rows in _poll_scan and fire the datapacket
-         bts.Log.Gen.info($"PriceStatsStream._on_price_stats state: {update.state.Count()} changes: {update.changes.Count()} added: {update.added.Count()}");
-         _cached_state = update.state;
-         _dirty = true;
-      }
-
-      protected override void myOnLastSubscriber() {
-         _vsd.removeWorker(this);
-         Disposable.dispose(ref subscription);
-      }
-
-      private void _scan() {
-         // this shouldn be made incremental using the TableUpdate
-         bts.Log.Gen.info("PriceStatsStream._scan_bbo");
-         if (_cached_state == null)
-            return;
-         BlueTable new_state = new BlueTable(_pstats_scheam);
-         foreach (var kv in _cached_state) {
-            SecurityKey skey = kv.Key;
-            int id = Convert.ToInt32(skey);
-            PriceStats pstats = kv.Value;
-            IBlueRow r = new BlueRow(_pstats_scheam, id, skey, pstats.security.ToString(), pstats.open, pstats.close, pstats.settlement);
-            new_state.addRow(r);
-         }
-         DataPacket dp = null;
-         if (myLastValue == null) {
-            dp = new DataPacket(new_state, new_state, new BlueTable(_pstats_scheam));
-         } else {
-            DataPacket old_dp = myLastValue as DataPacket;
-            IBlueRowCollection added_rows, del_rows;
-            BlueDataUtility.compute_differences_fast(new_state, old_dp.newRows, out del_rows, out added_rows);
-            dp = new DataPacket(new_state, added_rows, del_rows);
-         }
-         if (dp.addedRows.Count == 0 && dp.deletedRows.Count == 0)
-            return;
-         myFireEvent(dp);
-      }
-
-      public void doWork() {
-         if (_dirty && DateTime.Now - _last_poll_time > _poll_period) {
-            _last_poll_time = DateTime.Now;
-            _dirty = false;
-            _scan();
-         }
-      }
-   }
-}
+namespace BCG {
+   using BCG.Utilities.General;
+
+   using bts.api;
+   using bts.api.domain;
+   using bts.api.synthetic;
+   using bts.md;
+   using bts.spark;
+   using bts.utils;
+
+   using mbts;
+   using mbts.co2;
+   using mbts.config;
+   using mbts.spark;
+   using mbts.ui;
+
+   using Spark.Core.Data;
+   using Spark.Core.Disposables;
+   using Spark.Core.Reactive;
+
+   using System;
+   using System.Collections.Concurrent;
+   using System.Collections.Generic;
+   using System.Collections.ObjectModel;
+   using System.Linq;
+   using System.Reactive.Linq;
+   using bts.api.core;
+   using Spark.Api.MarketData;
+
+   using SecurityKey = System.UInt32;
+   using System.Data;
+   using System.Reflection.Metadata;
+
+   [DataStreamUserRespresentation("price_stats")]
+   public class PriceStatsStream : ViewStream, IWorker {
+      private readonly TimeSpan _poll_period = TimeSpan.FromMilliseconds(500);
+      private DateTime _last_poll_time = DateTime.MinValue;
+      private ViewStreamDatabase _vsd;
+      private int _id = 0;
+      IReadOnlyDictionary<SecurityKey, PriceStats> _cached_state = null;
+      MarketDataClient _md_client;
+      private bool _dirty = false;
+      private IDisposable _sub = null;
+      private readonly IBlueTableSchema _pstats_scheam = BlueDataUtility.form_schema("bbo",
+                                                Tuple.Create("id", TypeCode.Int32),
+                                                Tuple.Create("sec_key", TypeCode.UInt32),
+                                                Tuple.Create("sec", TypeCode.String),
+                                                Tuple.Create("date", TypeCode.DateTime),
+                                                Tuple.Create("open", TypeCode.Double),
+                                                Tuple.Create("close", TypeCode.Double),
+                                                Tuple.Create("settlement", TypeCode.Double));
+      IDisposable? subscription;
+
+      // TODO: add cons for symbol, exchange etc 
+      public PriceStatsStream()
+      : this(ViewStreamDatabase.instance) {
+      }
+
+      public PriceStatsStream(ViewStreamDatabase vsd) {
+         bts.Log.Gen.info("PriceStatsStream.cons");
+         _vsd = vsd;
+         _md_client = new MarketDataClient(new MDSource(TimeSpan.FromSeconds(0.5)));
+      }
+
+      protected override void myOnFirstSubscriber() {
+         _vsd.addWorker(this);
+         subscription = _md_client.price_stats().Subscribe(_on_price_stats);
+      }
+
+      private void _log_updates(TableUpdate<PriceStats, uint> update) {
+         int i = 0;
+         // foreach (BookTuple bt in update.added) {
+         //    bts.Log.Gen.info($"PriceStatsStream added: idx: {i++} bid: {bt.book.bids[0].price} / {bt.book.bids[0].size} ask: {bt.book.asks[0].price} / {bt.book.asks[0].size}");
+         // }
+         // foreach (Change<BookTuple> c in update.changes) {
+         //    var bt = c.New;
+         //    bts.Log.Gen.info($"PriceStatsStream changes: idx: {i} bid: {bt.book.bids[0].price} / {bt.book.bids[0].size} ask: {bt.book.asks[0].price} / {bt.book.asks[0].size}");
+         // }
+      }
+
+      private void _on_price_stats(TableUpdate<PriceStats, uint> update) {
+         _log_updates(update);
+         // just cacching in here, will populate the rows in _poll_scan and fire the datapacket
+         bts.Log.Gen.info($"PriceStatsStream._on_price_stats state: {update.state.Count()} changes: {update.changes.Count()} added: {update.added.Count()}");
+         _cached_state = update.state;
+         _dirty = true;
+      }
+
+      protected override void myOnLastSubscriber() {
+         _vsd.removeWorker(this);
+         Disposable.dispose(ref _sub);
+      }
+
+      private void _scan() {
+         // this shouldn be made incremental using the TableUpdate
+         bts.Log.Gen.info("PriceStatsStream._scan_bbo");
+         if (_cached_state == null)
+            return;
+         BlueTable new_state = new BlueTable(_pstats_scheam);
+         foreach (var kv in _cached_state) {
+            SecurityKey skey = kv.Key;
+            int id = Convert.ToInt32(skey);
+            PriceStats pstats = kv.Value;
+            var today = DateTime.Now.Date;
+            IBlueRow r = new BlueRow(_pstats_scheam, id, skey, pstats.security.ToString(), today, pstats.open, pstats.close, pstats.settlement);
+            new_state.addRow(r);
+         }
+         DataPacket dp = null;
+         if (myLastValue == null) {
+            dp = new DataPacket(new_state, new_state, new BlueTable(_pstats_scheam));
+         } else {
+            DataPacket old_dp = myLastValue as DataPacket;
+            IBlueRowCollection added_rows, del_rows;
+            BlueDataUtility.compute_differences_fast(new_state, old_dp.newRows, out del_rows, out added_rows);
+            dp = new DataPacket(new_state, added_rows, del_rows);
+         }
+         if (dp.addedRows.Count == 0 && dp.deletedRows.Count == 0)
+            return;
+         myFireEvent(dp);
+      }
+
+      public void doWork() {
+         if (_dirty && DateTime.Now - _last_poll_time > _poll_period) {
+            _last_poll_time = DateTime.Now;
+            _dirty = false;
+            _scan();
+         }
+      }
+   }
+}

src/DataStreamSuite/SharedStreams/SharedStreams.csproj

@@ -12,6 +12,7 @@
     <ProjectReference Include="..\DataStreamCore\DataStreamCore.csproj" />
     <ProjectReference Include="..\DataStream\DataStream.csproj" />
     <ProjectReference Include="..\PubSub\PubSub.csproj" />
+    <ProjectReference Include="..\..\client\spark_core\spark_core.csproj" />
   </ItemGroup>
 
   <ItemGroup>