check point (bc i don't trust googledrive sync yet)

* IsSocketStateOpen
* IsClosedProperly

.gitmodules

@@ -1,6 +1,6 @@
 [submodule "externals"]
 	path = externals
-	url = git@bitbucket.org:bluetradesys/externals.git
+	url = https://x-token-auth:ATCTT3xFfGN0h_ENnUSfEAIzbfVzkQPLG26cIli50MtcTwZ7VEc8lin5tJYqibjlBJ9PM9R642Ra5GjiL52TBfbO8KeUBT0UoMT7gLQHL2Y1TkW9yZr6NkvsoDkSl49DeD4Ijiu4_0u4qEm4oMVDDg2_fB059-uOWOo7ZL94tGZET8CSVpFIGvs=B0E03660@bitbucket.org/bluetradesys/externals.git
 [submodule "vcpkg"]
 	path = vcpkg
 	url = https://github.com/microsoft/vcpkg.git

.gitpod.yml

@@ -0,0 +1,49 @@
+# This configuration file was automatically generated by Gitpod.
+# Please adjust to your needs (see https://www.gitpod.io/docs/introduction/learn-gitpod/gitpod-yaml)
+# and commit this file to your remote git repository to share the goodness with others.
+
+# Learn more from ready-to-use templates: https://www.gitpod.io/docs/introduction/getting-started/quickstart
+
+image: gitpod/workspace-dotnet-8
+tasks:
+  - name: Install Deps
+    init: |
+      (
+        set -e
+        sudo install-packages rapidjson-dev
+        sudo install-packages libssh2-1-dev
+        sudo install-packages openssh-server
+        sudo install-packages liblttng-ust-dev
+        sudo install-packages libpcap0.8-dev
+        sudo install-packages ccache
+        sudo ln -s $(which ccache) /usr/local/bin/clang
+        sudo ln -s $(which ccache) /usr/local/bin/clang++
+        sudo ln -s $(which ccache) /usr/local/bin/clang-17
+        sudo ln -s $(which ccache) /usr/local/bin/clang++-17
+        sudo ln -s $(which ccache) /usr/local/bin/gcc
+        sudo ln -s $(which ccache) /usr/local/bin/g++
+        sudo ln -s $(which ccache) /usr/local/bin/gcc-12
+        sudo ln -s $(which ccache) /usr/local/bin/g++-12
+        if [ -e externals/src/emumba/install_emumba-deribit-connector.sh ]; then
+          externals/src/emumba/install_emumba-deribit-connector.sh
+        fi
+      )
+  - name: Setup Python venv
+    init: |
+      python -m pip install --upgrade pip
+      python -m venv .venv
+      . .venv/bin/activate
+      python -m pip install --upgrade pip
+      pip install -r requirements.txt
+
+ports:
+  - name: sparkd config
+    port: 8000
+    onOpen: notify
+  - name: sparkd bus
+    port: 8001
+
+gitConfig:
+  alias.st: "status"
+  status.short: "true"
+  core.pager: "less -FXRS"

CMakePresets.json

@@ -143,7 +143,7 @@
       "cacheVariables": {
         "CMAKE_C_COMPILER": "gcc",
         "CMAKE_CXX_COMPILER": "g++",
-        "CMAKE_CXX_FLAGS_RELEASE": "-fPIC -Wall -pedantic -Wextra -Werror -Wno-missing-braces -Wno-ignored-qualifiers -Wno-stringop-overflow --std=c++20 -fdiagnostics-color -Wno-stringop-overread -Wno-maybe-uninitialized -fno-omit-frame-pointer -DNDEBUG -O3",
+        "CMAKE_CXX_FLAGS_RELEASE": "-fPIC -Wall -pedantic -Wextra -Werror -Wno-missing-braces -Wno-ignored-qualifiers --std=c++20 -fdiagnostics-color -Wno-stringop-overread -Wno-maybe-uninitialized -fno-omit-frame-pointer -DNDEBUG -O3",
         "CMAKE_INSTALL_PREFIX": "${sourceDir}/install/gcc/release/sparkd"
       }
     },

docker/sparkd/dev/Dockerfile

@@ -2,55 +2,40 @@ FROM ubuntu:22.04
 
 ARG DEBIAN_FRONTEND=noninteractive
 
-# Note installations that take a while (more than 10s)
+# Foundational Deps
+RUN apt-get update && apt-get install -y \
+    build-essential \
+    ca-certificates \
+    curl \
+    gnupg \
+    liblttng-ust-dev \
+    libpcap0.8-dev \
+    libssh2-1-dev \
+    locales \
+    openssh-server \
+    software-properties-common \
+    sudo \
+    tzdata
 
-RUN apt-get update
-RUN apt-get install -y ca-certificates
-RUN apt-get install -y ccache
-RUN apt-get install -y curl
-RUN apt-get install -y fish
-RUN apt-get install -y gdb
-# git-all: ~250s
-RUN apt-get install -y git-all
-RUN apt-get install -y gnupg
-# liblttng-ust-dev: ~30s
-RUN apt-get install -y liblttng-ust-dev
-RUN apt-get install -y libssh2-1-dev
-RUN apt-get install -y locales
-RUN apt-get install -y ninja-build
-# openssh-server: ~40s
-RUN apt-get install -y openssh-server
-RUN apt-get install -y pkg-config
-RUN apt-get install -y software-properties-common
-RUN apt-get install -y sudo
-RUN apt-get install -y tzdata
-RUN apt-get install -y vim
-RUN apt-get install -y zip
-
-# Python: ~30s
-RUN apt-get install -y \
-    python3-pip \
-    python3-venv \
-    && python3 -m pip install --upgrade pip
-
-# GCC
-ARG gcc_major=13
-RUN add-apt-repository ppa:ubuntu-toolchain-r/test \
-    && apt-get update \
-    && apt-get install -y g++-$gcc_major
-
-# Clang
-ARG clang_major=17
+# Clang 17
 RUN curl -fsSL https://apt.llvm.org/llvm-snapshot.gpg.key 2> /dev/null \
     | gpg --dearmor - \
     | tee /usr/share/keyrings/llvm-snapshot.gpg > /dev/null \
-    && echo "deb [signed-by=/usr/share/keyrings/llvm-snapshot.gpg] http://apt.llvm.org/$(lsb_release -cs)/ llvm-toolchain-$(lsb_release -cs)-$clang_major main" \
+    && echo "deb [signed-by=/usr/share/keyrings/llvm-snapshot.gpg] http://apt.llvm.org/jammy/ llvm-toolchain-jammy-17 main" \
     | tee -a "/etc/apt/sources.list.d/llvm.list" \
     && apt-get update \
     && apt-get install -y \
-    clang-$clang_major \
-    clang-format-$clang_major \
-    clangd-$clang_major
+    clang-17 \
+    clang-format-17 \
+    clangd-17
+
+# G++
+RUN apt-get update && apt-get install -y \
+    g++
+
+# Ninja
+RUN apt-get update && apt-get install -y \
+    ninja-build
 
 # CMake
 RUN curl -fsSL https://apt.kitware.com/keys/kitware-archive-latest.asc 2> /dev/null \
@@ -68,6 +53,12 @@ RUN curl -fsSLO "https://packages.microsoft.com/config/ubuntu/$(lsb_release -rs)
     && apt-get update \
     && apt-get install -y dotnet-sdk-8.0
 
+# Python
+RUN apt-get update && apt-get install -y \
+    python3-pip \
+    python3-venv \
+    && python3 -m pip install --upgrade pip
+
 # Node
 ARG node_major=18
 RUN mkdir -p /etc/apt/keyrings \
@@ -78,35 +69,41 @@ RUN mkdir -p /etc/apt/keyrings \
     && apt-get update \
     && apt-get install -y nodejs
 
-# Emumba deps
+# Sqlite3
 RUN apt-get update && apt-get install -y \
-    build-essential \
-    doxygen \
-    graphviz \
-    libgtest-dev \
-    libssl-dev \
-    pkg-config \
-    python3-sphinx \
-    rapidjson-dev \
-    zlib1g-dev
+    sqlite3
+
+# Dev Utils (install last so they can be added/removed with minimal rebuild)
+RUN apt-get update && apt-get install -y \ 
+    ccache \
+    fish \
+    gdb \
+    git-all \
+    vim
 
 # Set up default executables (update-alternatives)
-RUN update-alternatives --install /usr/bin/clang clang /usr/bin/clang-$clang_major 100; \
-    update-alternatives --install /usr/bin/clang++ clang++ /usr/bin/clang++-$clang_major 100; \
-    update-alternatives --install /usr/bin/clang-format clang-format /usr/bin/clang-format-$clang_major 100; \
-    update-alternatives --install /usr/bin/clangd clangd /usr/bin/clangd-$clang_major 100; \
-    update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-$gcc_major 100; \
-    update-alternatives --install /usr/bin/g++ g++ /usr/bin/g++-$gcc_major 100
+RUN update-alternatives --install /usr/bin/clang++ clang++ /usr/bin/clang++-17 100; \
+    update-alternatives --install /usr/bin/clang clang /usr/bin/clang-17 100; \
+    update-alternatives --install /usr/bin/clang-format clang-format /usr/bin/clang-format-17 100; \
+    update-alternatives --install /usr/bin/clangd clangd /usr/bin/clangd-17 100; \
+    update-alternatives --install /usr/bin/g++ g++ /usr/bin/g++-12 100; \
+    update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-12 100
 
 # Add compiler symlinks for ccache 
 RUN ln -s /usr/bin/ccache /usr/local/bin/clang; \
     ln -s /usr/bin/ccache /usr/local/bin/clang++; \
-    ln -s /usr/bin/ccache /usr/local/bin/clang-$clang_major; \
-    ln -s /usr/bin/ccache /usr/local/bin/clang++-$clang_major; \
+    ln -s /usr/bin/ccache /usr/local/bin/clang-13; \
+    ln -s /usr/bin/ccache /usr/local/bin/clang++-13; \
+    ln -s /usr/bin/ccache /usr/local/bin/clang-15; \
+    ln -s /usr/bin/ccache /usr/local/bin/clang++-15; \
+    ln -s /usr/bin/ccache /usr/local/bin/clang-17; \
+    ln -s /usr/bin/ccache /usr/local/bin/clang++-17; \
     ln -s /usr/bin/ccache /usr/local/bin/gcc; \
     ln -s /usr/bin/ccache /usr/local/bin/g++; \
-    ln -s /usr/bin/ccache /usr/local/bin/gcc-$gcc_major; \
-    ln -s /usr/bin/ccache /usr/local/bin/g++-$gcc_major
+    ln -s /usr/bin/ccache /usr/local/bin/gcc-11; \
+    ln -s /usr/bin/ccache /usr/local/bin/g++-11; \
+    ln -s /usr/bin/ccache /usr/local/bin/gcc-12; \
+    ln -s /usr/bin/ccache /usr/local/bin/g++-12
 
 RUN locale-gen en_US.UTF-8
 

global.json

@@ -0,0 +1,6 @@
+{
+  "sdk": {
+    "version": "6.0.200",
+    "rollForward": "latestMajor"
+  }
+}

src/DataStreamSuite/BooleanExpressions/BooleanExpressionTokens.lex

@@ -42,6 +42,7 @@ OpenParenthesis          \(
 CloseParenthesis         \)
 Comma                    ,
 Negate                   '!'
+MapsTo                   ':'
 IsOfType                 '/'
 
 %%
@@ -97,6 +98,10 @@ IsOfType                 '/'
    return (int)Token.NEGATE;
 }
 
+{MapsTo} {
+   return (int)Token.MAPS_TO;
+}
+
 {Equal} {
    return (int)Token.EQUAL;
 }

src/DataStreamSuite/Bts.DataStream.Websocket/DataSync.cs

@@ -5,9 +5,9 @@ using BCG;
 using bts;
 using bts.api.core;
 using bts.collections;
-using bts.websocket_core.json_rpc;
 
 using Bts.DataStreamExpressions;
+using Bts.Websocket;
 
 using mbts.utils;
 

src/DataStreamSuite/Bts.DataStream.Websocket/JsonParseResult.cs

@@ -0,0 +1,24 @@
+namespace Bts.DataStream.Websocket;
+
+using Newtonsoft.Json.Linq;
+
+class JsonParseResult {
+   public string? MessageId { get; init; }
+   public RequestType Type { get; init; }
+   public JObject? Parameters { get; init; }
+   public string? Error { get; init; }
+   public bool Successful => Error is not null;
+
+   public static JsonParseResult CreateError(string error) =>
+      new() {
+         Error = error,
+         MessageId = "missing"
+      };
+
+   public static JsonParseResult Create(string messageId, RequestType type, JObject parameters) =>
+      new() {
+         MessageId = messageId,
+         Parameters = parameters,
+         Type = type
+      };
+}

src/DataStreamSuite/Bts.DataStream.Websocket/JsonRpcErrorCode.cs

@@ -1,4 +1,4 @@
-namespace bts.websocket_core.json_rpc;
+namespace Bts.DataStream.Websocket;
 
 public enum JsonRpcErrorCode {
    None = 0,

src/DataStreamSuite/Bts.DataStream.Websocket/Session.cs

@@ -2,6 +2,8 @@ namespace Bts.DataStream.Websocket;
 
 using bts;
 
+using Bts.Websocket;
+
 using Google.Protobuf;
 
 using System.Text.Json;
@@ -13,7 +15,7 @@ public static class Session {
       WebsocketServerConfig config, JsonSerializerOptions? opts) {
       opts ??= JsonSerializerOptions.Default;
       var sessionId = Interlocked.Increment(ref sessionIdCounter);
-      var gateway = new WebsocketClient(config);
+      var gateway = new WebsocketGatewayObservable(config);
       var sync = new DataSync(opts);
       using (gateway.Subscribe(sync))
       using (sync.Subscribe(gateway)) {

src/DataStreamSuite/Bts.DataStream.Websocket/SubscriptionParams.cs

@@ -1,30 +1,30 @@
 namespace Bts.DataStream.Websocket;
 
-using bts.utils;
-
 using Spark.Core;
-using Spark.Core.Json;
 
 using System;
 using System.Diagnostics.CodeAnalysis;
 using System.Text.Json;
 using System.Text.Json.Serialization;
 
-public class SubscriptionParams(string? query = null) : IEquatable<SubscriptionParams> {
+public class SubscriptionParams : IEquatable<SubscriptionParams> {
    static class PropNames {
       public const string QUERY = "query";
    }
 
+   public SubscriptionParams(string? query) {
+      Query = query;
+   }
+
    // Serializer supports plain string, or an array of lines for convenience
    [MaybeNull]
    [JsonPropertyName(PropNames.QUERY)]
    [JsonConverter(typeof(MultilineStringJsonConverter))]
-   public string? Query { get; set; } = query;
+   public string? Query { get; set; }
 
    public bool Equals(SubscriptionParams? other) =>
       other is not null &&
-      // Not the most elegant equals impl, but it'll work
-      Query.safe_equals(other.Query, (a, b) => a?.ToString() == b?.ToString());
+      Query == other.Query;
 
    class SubscriptionParamsConverter : JsonConverter<SubscriptionParams> {
       public override bool CanConvert(Type typeToConvert) =>
@@ -38,19 +38,19 @@ public class SubscriptionParams(string? query = null) : IEquatable<SubscriptionP
          return read(ref reader);
       }
 
-      static SubscriptionParams read(ref Utf8JsonReader reader) {
+      SubscriptionParams read(ref Utf8JsonReader reader) {
          return reader.TokenType switch {
-            JsonTokenType.String => StringImpl(ref reader),
-            JsonTokenType.StartArray => ArrayImpl(ref reader),
-            JsonTokenType.StartObject => ObjectImpl(ref reader),
+            JsonTokenType.String => stringImpl(ref reader),
+            JsonTokenType.StartArray => arrayImpl(ref reader),
+            JsonTokenType.StartObject => objectImpl(ref reader),
             _ => throw new ArgumentException(
                $"Unable to parse {typeof(SubscriptionParams)} from {reader.TokenType}"),
          };
 
-         static SubscriptionParams StringImpl(ref Utf8JsonReader reader) =>
+         static SubscriptionParams stringImpl(ref Utf8JsonReader reader) =>
             new(reader.GetString()!);
 
-         static SubscriptionParams ArrayImpl(ref Utf8JsonReader reader) {
+         static SubscriptionParams arrayImpl(ref Utf8JsonReader reader) {
             var lines = new List<string>(2);
             while (reader.Read()) {
                switch (reader.TokenType) {
@@ -65,7 +65,7 @@ public class SubscriptionParams(string? query = null) : IEquatable<SubscriptionP
                $"Failed to parse {typeof(SubscriptionParams)} (did not reach end of array)");
          }
 
-         static SubscriptionParams ObjectImpl(ref Utf8JsonReader reader) {
+         static SubscriptionParams objectImpl(ref Utf8JsonReader reader) {
             var lines = new List<string>(2);
             while (reader.Read()) {
                switch (reader.TokenType) {
@@ -74,9 +74,9 @@ public class SubscriptionParams(string? query = null) : IEquatable<SubscriptionP
                      if (propName == PropNames.QUERY) {
                         switch (reader.TokenType) {
                            case JsonTokenType.StartArray:
-                              return ArrayImpl(ref reader);
+                              return arrayImpl(ref reader);
                            case JsonTokenType.String:
-                              return StringImpl(ref reader);
+                              return stringImpl(ref reader);
                         }
                      }
                      break;
@@ -100,13 +100,11 @@ public class SubscriptionParams(string? query = null) : IEquatable<SubscriptionP
             writer.WriteString(PropNames.QUERY, value.Query);
             return;
          }
-         writer.WriteArray(PropNames.QUERY, lines);
+         writer.WriteStartArray(PropNames.QUERY);
+         foreach (var l in lines) {
+            writer.WriteStringValue(l);
+         }
+         writer.WriteEndArray();
       }
    }
-
-   public override bool Equals(object? obj) =>
-      Equals(obj as SubscriptionParams);
-
-   public override int GetHashCode() =>
-      Query?.Length ?? typeof(SubscriptionParams).GetHashCode();
 }

src/DataStreamSuite/Bts.DataStream.WebsocketClient/Bts.DataStream.WebsocketClient.csproj

@@ -0,0 +1,10 @@
+<Project Sdk="Microsoft.NET.Sdk.Web">
+
+  <PropertyGroup>
+    <TargetFramework>net7.0</TargetFramework>
+    <Nullable>enable</Nullable>
+    <ImplicitUsings>enable</ImplicitUsings>
+    <EnableDefaultContentItems>True</EnableDefaultContentItems>
+  </PropertyGroup>
+
+</Project>

src/DataStreamSuite/Bts.DataStream.WebsocketClient/Pages/Error.cshtml

@@ -0,0 +1,26 @@
+@page
+@model ErrorModel
+@{
+    ViewData["Title"] = "Error";
+}
+
+<h1 class="text-danger">Error.</h1>
+<h2 class="text-danger">An error occurred while processing your request.</h2>
+
+@if (Model.ShowRequestId)
+{
+    <p>
+        <strong>Request ID:</strong> <code>@Model.RequestId</code>
+    </p>
+}
+
+<h3>Development Mode</h3>
+<p>
+    Swapping to the <strong>Development</strong> environment displays detailed information about the error that occurred.
+</p>
+<p>
+    <strong>The Development environment shouldn't be enabled for deployed applications.</strong>
+    It can result in displaying sensitive information from exceptions to end users.
+    For local debugging, enable the <strong>Development</strong> environment by setting the <strong>ASPNETCORE_ENVIRONMENT</strong> environment variable to <strong>Development</strong>
+    and restarting the app.
+</p>

src/DataStreamSuite/Bts.DataStream.WebsocketClient/Pages/Error.cshtml.cs

@@ -0,0 +1,27 @@
+using System.Diagnostics;
+using Microsoft.AspNetCore.Mvc;
+using Microsoft.AspNetCore.Mvc.RazorPages;
+
+namespace Bts.DataStream.WebsocketClient.Pages;
+
+[ResponseCache(Duration = 0, Location = ResponseCacheLocation.None, NoStore = true)]
+[IgnoreAntiforgeryToken]
+public class ErrorModel : PageModel
+{
+    public string? RequestId { get; set; }
+
+    public bool ShowRequestId => !string.IsNullOrEmpty(RequestId);
+
+    private readonly ILogger<ErrorModel> _logger;
+
+    public ErrorModel(ILogger<ErrorModel> logger)
+    {
+        _logger = logger;
+    }
+
+    public void OnGet()
+    {
+        RequestId = Activity.Current?.Id ?? HttpContext.TraceIdentifier;
+    }
+}
+

src/DataStreamSuite/Bts.DataStream.WebsocketClient/Pages/Index.cshtml

@@ -0,0 +1,15 @@
+@page
+@model IndexModel
+@{
+  ViewData["Title"] = "Home page";
+}
+
+<div class="text-center">
+  <h1 class="display-4">Welcome</h1>
+  <p>Learn about <a href="https://docs.microsoft.com/aspnet/core">building Web apps with ASP.NET Core</a>.</p>
+  <button id="connectButton">Connect to Websocket</button>
+  <div id="output"></div>
+  <input type="text" id="methodInput" placeholder="Enter method name" />
+  <input type="text" id="paramsInput" placeholder="Enter method parameters (if any)" />
+  <button id="sendMessageButton">Send JSON-RPC Request</button>
+</div>

src/DataStreamSuite/Bts.DataStream.WebsocketClient/Pages/Index.cshtml.cs

@@ -0,0 +1,16 @@
+using Microsoft.AspNetCore.Mvc;
+using Microsoft.AspNetCore.Mvc.RazorPages;
+
+namespace Bts.DataStream.WebsocketClient.Pages;
+
+public class IndexModel : PageModel {
+   private readonly ILogger<IndexModel> _logger;
+
+   public IndexModel(ILogger<IndexModel> logger) {
+      _logger = logger;
+   }
+
+   public void OnGet() {
+
+   }
+}

src/DataStreamSuite/Bts.DataStream.WebsocketClient/Pages/Privacy.cshtml

@@ -0,0 +1,8 @@
+@page
+@model PrivacyModel
+@{
+    ViewData["Title"] = "Privacy Policy";
+}
+<h1>@ViewData["Title"]</h1>
+
+<p>Use this page to detail your site's privacy policy.</p>

src/DataStreamSuite/Bts.DataStream.WebsocketClient/Pages/Privacy.cshtml.cs

@@ -0,0 +1,19 @@
+using Microsoft.AspNetCore.Mvc;
+using Microsoft.AspNetCore.Mvc.RazorPages;
+
+namespace Bts.DataStream.WebsocketClient.Pages;
+
+public class PrivacyModel : PageModel
+{
+    private readonly ILogger<PrivacyModel> _logger;
+
+    public PrivacyModel(ILogger<PrivacyModel> logger)
+    {
+        _logger = logger;
+    }
+
+    public void OnGet()
+    {
+    }
+}
+

src/DataStreamSuite/Bts.DataStream.WebsocketClient/Pages/Shared/_Layout.cshtml

@@ -0,0 +1,51 @@
+<!DOCTYPE html>
+<html lang="en">
+<head>
+    <meta charset="utf-8" />
+    <meta name="viewport" content="width=device-width, initial-scale=1.0" />
+    <title>@ViewData["Title"] - Bts.DataStream.WebsocketClient</title>
+    <link rel="stylesheet" href="~/lib/bootstrap/dist/css/bootstrap.min.css" />
+    <link rel="stylesheet" href="~/css/site.css" asp-append-version="true" />
+    <link rel="stylesheet" href="~/Bts.DataStream.WebsocketClient.styles.css" asp-append-version="true" />
+</head>
+<body>
+    <header>
+        <nav class="navbar navbar-expand-sm navbar-toggleable-sm navbar-light bg-white border-bottom box-shadow mb-3">
+            <div class="container">
+                <a class="navbar-brand" asp-area="" asp-page="/Index">Bts.DataStream.WebsocketClient</a>
+                <button class="navbar-toggler" type="button" data-bs-toggle="collapse" data-bs-target=".navbar-collapse" aria-controls="navbarSupportedContent"
+                        aria-expanded="false" aria-label="Toggle navigation">
+                    <span class="navbar-toggler-icon"></span>
+                </button>
+                <div class="navbar-collapse collapse d-sm-inline-flex justify-content-between">
+                    <ul class="navbar-nav flex-grow-1">
+                        <li class="nav-item">
+                            <a class="nav-link text-dark" asp-area="" asp-page="/Index">Home</a>
+                        </li>
+                        <li class="nav-item">
+                            <a class="nav-link text-dark" asp-area="" asp-page="/Privacy">Privacy</a>
+                        </li>
+                    </ul>
+                </div>
+            </div>
+        </nav>
+    </header>
+    <div class="container">
+        <main role="main" class="pb-3">
+            @RenderBody()
+        </main>
+    </div>
+
+    <footer class="border-top footer text-muted">
+        <div class="container">
+            &copy; 2023 - Bts.DataStream.WebsocketClient - <a asp-area="" asp-page="/Privacy">Privacy</a>
+        </div>
+    </footer>
+
+    <script src="~/lib/jquery/dist/jquery.min.js"></script>
+    <script src="~/lib/bootstrap/dist/js/bootstrap.bundle.min.js"></script>
+    <script src="~/js/site.js" asp-append-version="true"></script>
+
+    @await RenderSectionAsync("Scripts", required: false)
+</body>
+</html>

src/DataStreamSuite/Bts.DataStream.WebsocketClient/Pages/Shared/_Layout.cshtml.css

@@ -0,0 +1,48 @@
+/* Please see documentation at https://docs.microsoft.com/aspnet/core/client-side/bundling-and-minification
+for details on configuring this project to bundle and minify static web assets. */
+
+a.navbar-brand {
+  white-space: normal;
+  text-align: center;
+  word-break: break-all;
+}
+
+a {
+  color: #0077cc;
+}
+
+.btn-primary {
+  color: #fff;
+  background-color: #1b6ec2;
+  border-color: #1861ac;
+}
+
+.nav-pills .nav-link.active, .nav-pills .show > .nav-link {
+  color: #fff;
+  background-color: #1b6ec2;
+  border-color: #1861ac;
+}
+
+.border-top {
+  border-top: 1px solid #e5e5e5;
+}
+.border-bottom {
+  border-bottom: 1px solid #e5e5e5;
+}
+
+.box-shadow {
+  box-shadow: 0 .25rem .75rem rgba(0, 0, 0, .05);
+}
+
+button.accept-policy {
+  font-size: 1rem;
+  line-height: inherit;
+}
+
+.footer {
+  position: absolute;
+  bottom: 0;
+  width: 100%;
+  white-space: nowrap;
+  line-height: 60px;
+}

src/DataStreamSuite/Bts.DataStream.WebsocketClient/Pages/Shared/_ValidationScriptsPartial.cshtml

@@ -0,0 +1,2 @@
+<script src="~/lib/jquery-validation/dist/jquery.validate.min.js"></script>
+<script src="~/lib/jquery-validation-unobtrusive/jquery.validate.unobtrusive.min.js"></script>

src/DataStreamSuite/Bts.DataStream.WebsocketClient/Pages/_ViewImports.cshtml

@@ -0,0 +1,3 @@
+@using Bts.DataStream.WebsocketClient
+@namespace Bts.DataStream.WebsocketClient.Pages
+@addTagHelper *, Microsoft.AspNetCore.Mvc.TagHelpers

src/DataStreamSuite/Bts.DataStream.WebsocketClient/Pages/_ViewStart.cshtml

@@ -0,0 +1,3 @@
+@{
+    Layout = "_Layout";
+}

src/DataStreamSuite/Bts.DataStream.WebsocketClient/Program.cs

@@ -0,0 +1,26 @@
+using Microsoft.AspNetCore.Mvc;
+
+var builder = WebApplication.CreateBuilder(args);
+
+// Add services to the container.
+builder.Services.AddRazorPages();
+
+var app = builder.Build();
+
+// Configure the HTTP request pipeline.
+if (!app.Environment.IsDevelopment()) {
+   app.UseExceptionHandler("/Error");
+   // The default HSTS value is 30 days. You may want to change this for production scenarios, see https://aka.ms/aspnetcore-hsts.
+   app.UseHsts();
+}
+
+app.UseHttpsRedirection();
+app.UseStaticFiles();
+
+app.UseRouting();
+
+app.UseAuthorization();
+
+app.MapRazorPages();
+
+app.Run();

src/DataStreamSuite/Bts.DataStream.WebsocketClient/Properties/launchSettings.json

@@ -0,0 +1,37 @@
+{
+  "iisSettings": {
+    "windowsAuthentication": false,
+    "anonymousAuthentication": true,
+    "iisExpress": {
+      "applicationUrl": "http://localhost:29312",
+      "sslPort": 44390
+    }
+  },
+  "profiles": {
+    "http": {
+      "commandName": "Project",
+      "dotnetRunMessages": true,
+      "launchBrowser": true,
+      "applicationUrl": "http://localhost:5214",
+      "environmentVariables": {
+        "ASPNETCORE_ENVIRONMENT": "Development"
+      }
+    },
+    "https": {
+      "commandName": "Project",
+      "dotnetRunMessages": true,
+      "launchBrowser": true,
+      "applicationUrl": "https://localhost:7176;http://localhost:5214",
+      "environmentVariables": {
+        "ASPNETCORE_ENVIRONMENT": "Development"
+      }
+    },
+    "IIS Express": {
+      "commandName": "IISExpress",
+      "launchBrowser": true,
+      "environmentVariables": {
+        "ASPNETCORE_ENVIRONMENT": "Development"
+      }
+    }
+  }
+}

src/DataStreamSuite/Bts.DataStream.WebsocketClient/appsettings.Development.json

@@ -0,0 +1,9 @@
+{
+  "DetailedErrors": true,
+  "Logging": {
+    "LogLevel": {
+      "Default": "Information",
+      "Microsoft.AspNetCore": "Warning"
+    }
+  }
+}

src/DataStreamSuite/Bts.DataStream.WebsocketClient/appsettings.json

@@ -0,0 +1,9 @@
+{
+  "Logging": {
+    "LogLevel": {
+      "Default": "Information",
+      "Microsoft.AspNetCore": "Warning"
+    }
+  },
+  "AllowedHosts": "*"
+}

src/DataStreamSuite/Bts.DataStream.WebsocketClient/wwwroot/css/site.css

@@ -0,0 +1,22 @@
+html {
+  font-size: 14px;
+}
+
+@media (min-width: 768px) {
+  html {
+    font-size: 16px;
+  }
+}
+
+.btn:focus, .btn:active:focus, .btn-link.nav-link:focus, .form-control:focus, .form-check-input:focus {
+  box-shadow: 0 0 0 0.1rem white, 0 0 0 0.25rem #258cfb;
+}
+
+html {
+  position: relative;
+  min-height: 100%;
+}
+
+body {
+  margin-bottom: 60px;
+}

src/DataStreamSuite/Bts.DataStream.WebsocketClient/wwwroot/js/site.js

@@ -0,0 +1,67 @@
+// Please see documentation at https://docs.microsoft.com/aspnet/core/client-side/bundling-and-minification
+// for details on configuring this project to bundle and minify static web assets.
+
+// Write your JavaScript code.
+const connectButton = document.getElementById('connectButton');
+const output = document.getElementById('output');
+const methodInput = document.getElementById('methodInput');
+const paramsInput = document.getElementById('paramsInput');
+const sendMessageButton = document.getElementById('sendMessageButton');
+
+let socket;
+
+connectButton.addEventListener('click', () => {
+    if (socket && socket.readyState === WebSocket.OPEN) {
+        return;
+    }
+
+    socket = new WebSocket('ws://localhost:8080'); // Replace with your WebSocket server URL
+
+    socket.onopen = (event) => {
+        output.innerHTML += 'Connected to WebSocket server<br>';
+    };
+
+    socket.onmessage = (event) => {
+        const message = JSON.parse(event.data);
+        output.innerHTML += `Received JSON-RPC response: ${JSON.stringify(message)}<br>`;
+    };
+
+    socket.onclose = (event) => {
+        if (event.wasClean) {
+            output.innerHTML += `Connection closed cleanly, code=${event.code}, reason=${event.reason}<br>`;
+        } else {
+            output.innerHTML += 'Connection abruptly closed<br>';
+        }
+    };
+
+    socket.onerror = (error) => {
+        output.innerHTML += `WebSocket Error: ${error.message}<br>`;
+    };
+});
+
+sendMessageButton.addEventListener('click', () => {
+    // const method = methodInput.value;
+    // const params = paramsInput.value;
+    const method = 'subscribe';
+    const params = `{
+"expression": "literal("n", "2.0")"
+}'
+`
+    if (socket && socket.readyState === WebSocket.OPEN && method) {
+        // const reqParams = {
+        //   'expression': message
+        // }
+        const request = {
+            jsonrpc: '2.0',
+            method: method,
+            params: {
+              query: `literal("n", "2.0")`
+            },
+            id: 1, // You can set a unique ID for each request
+        };
+
+        socket.send(JSON.stringify(request));
+        methodInput.value = ''; // Clear the input fields
+        paramsInput.value = '';
+    }
+});

src/DataStreamSuite/Bts.DataStream.WebsocketServer/Bts.DataStream.WebsocketServer.csproj

@@ -2,7 +2,7 @@
 
   <PropertyGroup>
     <OutputType>Exe</OutputType>
-    <TargetFramework>net8.0</TargetFramework>
+    <TargetFramework>net7.0</TargetFramework>
     <ImplicitUsings>enable</ImplicitUsings>
     <Nullable>enable</Nullable>
   </PropertyGroup>

src/DataStreamSuite/Bts.DataStream.WebsocketServer/Usings.cs

@@ -1,5 +1,6 @@
+global using Bts.Websocket;
+
 global using bts.websocket_core;
-global using bts.websocket_core.json_rpc;
 
 global using System.CommandLine;
 

src/DataStreamSuite/Bts.DataStreamWebApp.Server/Controllers/DataStreamSubscriptionController.cs

@@ -3,9 +3,9 @@ namespace Bts.DataStreamWebApp.Server.Controllers;
 using BCG;
 
 using bts.api.core;
-using bts.websocket_core.json_rpc;
 
 using Bts.DataStreamExpressions;
+using Bts.Websocket;
 
 using Microsoft.AspNetCore.Mvc;
 
@@ -17,13 +17,13 @@ using System.Text.Json;
 
 [Route("/ws")]
 public class DataStreamSubscriptionController(ILogger<DataStreamSubscriptionController> logger) : ControllerBase {
-   readonly ILogger<DataStreamSubscriptionController> logger = logger;
+   readonly ILogger<DataStreamSubscriptionController> _logger = logger;
    IDisposable? streamSubscription;
 
    [Route("/ws")]
    public async Task Get() {
       if (!HttpContext.WebSockets.IsWebSocketRequest) {
-         logger.LogError("Expected websocket request on path '{}'", HttpContext.Request.Path);
+         _logger.LogError("Expected websocket request on path '{}'", HttpContext.Request.Path);
          return;
       }
       Console.WriteLine("Establishing websocket connection...");
@@ -67,17 +67,16 @@ public class DataStreamSubscriptionController(ILogger<DataStreamSubscriptionCont
                 new ArraySegment<byte>(buffer), CancellationToken.None);
             var msg = Encoding.UTF8.GetString(buffer, 0, receiveResult.Count);
             var rpcReq = JsonSerializer.Deserialize<JsonRpcRequest<DataStreamSubscription>>(msg);
-            var query = rpcReq?.Params?.Query ?? throw new InvalidOperationException(
-               "Failed to parse JsonRpcRequest.Params");
-            var exp = DataStreamExpressionsInterpreter.Evaluate(query);
+            var queryArgs = rpcReq!.Params;
+            var exp = DataStreamExpressionsInterpreter.Evaluate(queryArgs.Query!);
             if (exp is null) {
-               Console.WriteLine($"Invalid query expression: {query}.");
+               Console.WriteLine($"Invalid query expression: {queryArgs.Query}.");
                continue;
             }
             var vs = exp.getView(new Dictionary<string, StatementExpression>(0));
             Disposable.dispose(ref streamSubscription);
-            vs.onChange += OnStreamChanged;
-            streamSubscription = Disposable.Create(() => vs.onChange -= OnStreamChanged);
+            vs.onChange += onStreamChanged;
+            streamSubscription = Disposable.Create(() => vs.onChange -= onStreamChanged);
          }
       } catch (Exception ex) {
          Console.WriteLine($"Exception on WebsocketServer thread: {ex}");
@@ -86,7 +85,7 @@ public class DataStreamSubscriptionController(ILogger<DataStreamSubscriptionCont
 
       return 0;
 
-      async void OnStreamChanged(object newValue) {
+      async void onStreamChanged(object newValue) {
          if (newValue is not DataPacket dp) {
             Console.WriteLine(
                $"OnStreamChanged delegate expected DataPacket (actually received {newValue?.GetType()}.");

src/DataStreamSuite/Bts.DataStreamWebApp.Server/Program.cs

@@ -3,7 +3,6 @@ using BCG;
 using bts;
 using bts.api.core;
 using bts.utils;
-using bts.websocket_core.json_rpc;
 
 using Bts.DataStreamExpressions;
 using Bts.DataStreamWebApp.Server;
@@ -112,7 +111,7 @@ async Task<int> runWebSocketServer(string? sparkd, string user, string password,
          receiveResult = await webSocket.ReceiveAsync(
              new ArraySegment<byte>(buffer), CancellationToken.None);
          var msg = Encoding.UTF8.GetString(buffer, 0, receiveResult.Count);
-         var rpcReq = JsonSerializer.Deserialize<JsonRpcRequest<DataStreamSubscription>>(msg);
+         var rpcReq = JsonSerializer.Deserialize<Bts.Websocket.JsonRpcRequest<DataStreamSubscription>>(msg);
          var queryArgs = rpcReq!.Params!;
          // var queryArgs = new DataStreamSubscription { Query = msg };
          var exp = DataStreamExpressionsInterpreter.Evaluate(queryArgs.Query!);

src/DataStreamSuite/DataStream.Console/ColumnMapTests.cs

@@ -0,0 +1,34 @@
+// namespace ColumnMapTests;
+
+// using Bts.ExpressionCore;
+// using Bts.ColumnMapExpressions;
+// using Bts.DataStreamExpressions;
+
+// using DataStream.Console;
+
+// public class TempColumnMapTests {
+//    public static void Run(ExpressionTest test) {
+//       Run(test.Expression, test.ExpectedResults);
+//    }
+
+//    public static void Run(string expression, IDictionary<string, string> expectedResults) {
+//       var interpreter = ColumnMapExpression.Parse(expression);
+//       var env = new Dictionary<string, object?>();
+//       var result = interpreter.Evaluate(env);
+//       foreach (var (key, expected) in expectedResults) {
+//          if (!result.TryGetValue(key, out var actual)) {
+//             throw new Exception($"ExpectedResults did not contain key '{key}'");
+//          }
+//          if (!expected.Equals(actual?.ToString() ?? string.Empty)) {
+//             throw new Exception($"Expected '{key}' to be {expected} (actually {actual})");
+//          }
+//          ConsoleHelper.WriteLine($"PASS: {key}={actual}={expected}", ConsoleColor.Green);
+//       }
+//    }
+
+//    static void assert(bool condition) {
+//       if (!condition) {
+//          throw new Exception();
+//       }
+//    }
+// }

src/DataStreamSuite/DataStream.Console/ConsoleHelper.cs

@@ -0,0 +1,29 @@
+namespace DataStream.Console;
+
+using Spark.Core.Disposables;
+
+using System;
+
+static class ConsoleHelper {
+   public static IDisposable PushBg(ConsoleColor value) {
+      var orig = Console.BackgroundColor;
+      Console.BackgroundColor = value;
+      return Disposable.Create(pop);
+      void pop() =>
+         Console.BackgroundColor = orig;
+   }
+
+   public static IDisposable PushFg(ConsoleColor value) {
+      var orig = Console.ForegroundColor;
+      Console.ForegroundColor = value;
+      return Disposable.Create(pop);
+      void pop() =>
+         Console.ForegroundColor = orig;
+   }
+   
+   public static void WriteLine(string text, ConsoleColor? fg = null, ConsoleColor? bg = null) {
+      using IDisposable? fgPush = fg is null ? null : PushFg(fg.Value);
+      using IDisposable? bgPush = bg is null ? null : PushBg(bg.Value);
+      Console.WriteLine(text);
+   }
+}

src/DataStreamSuite/DataStream.Console/DataStream.Console.csproj

@@ -0,0 +1,15 @@
+<Project Sdk="Microsoft.NET.Sdk">
+
+  <ItemGroup>
+    <ProjectReference Include="..\ArithmeticExpressions\ArithmeticExpressions.csproj" />
+    <ProjectReference Include="..\DataStreamExpressions\DataStreamExpressions.csproj" />
+  </ItemGroup>
+
+  <PropertyGroup>
+    <OutputType>Exe</OutputType>
+    <TargetFramework>net8.0</TargetFramework>
+    <ImplicitUsings>enable</ImplicitUsings>
+    <Nullable>enable</Nullable>
+  </PropertyGroup>
+
+</Project>

src/DataStreamSuite/DataStream.Console/ExpressionTest.cs

@@ -0,0 +1,15 @@
+namespace DataStream.Console;
+
+using Spark.Core;
+
+using System.Text;
+
+public class ExpressionTest {
+   public ExpressionTest(string expression, IDictionary<string, string> expectedResults) {
+      Expression = expression;
+      ExpectedResults = expectedResults;
+   }
+
+   public string Expression { get; }
+   public IDictionary<string, string> ExpectedResults { get; }
+}

src/DataStreamSuite/DataStream.Console/ExpressionTestParser.cs

@@ -0,0 +1,25 @@
+namespace DataStream.Console;
+
+using Spark.Core;
+
+using System.Text;
+
+public class ExpressionTestParser {
+   public static ExpressionTest Parse(IEnumerable<string> lines) {
+      var expressionLines = lines.SkipWhile(l => l != "BEGIN_TEST").Skip(1);
+      var expressionBd = new StringBuilder();
+      Dictionary<string, string>? expectedResults = null;
+      foreach (var l in expressionLines) {
+         const string END_SENTINEL = "END_TEST";
+         if (l.StartsWith(END_SENTINEL)) {
+            expectedResults = l[END_SENTINEL.Length..].Trim()
+               .Split(" ")
+               .Select(kv => kv.Split(":"))
+               .ToDictionary(kv => kv[0], kv => kv[1]);
+            break;
+         }
+         expressionBd.AppendLine(l);
+      }
+      return new(expressionBd.ToString(), expectedResults ?? new(0));
+   }
+}

src/DataStreamSuite/DataStream.Console/Program.cs

@@ -0,0 +1,32 @@
+// See https://aka.ms/new-console-template for more information
+using DataStream.Console;
+
+using ExpressionTests;
+
+using Spark.Core.Collections;
+
+using System.Reactive.Disposables;
+
+
+try {
+   foreach (var arg in args) {
+      if (File.Exists(arg)) {
+         using var fs = File.OpenRead(arg) ??
+            throw new Exception("File.OpenRead returned null. LIES!");
+         var expressionLines = new StringStreamEnumerable(new(fs));
+         var test = ExpressionTestParser.Parse(expressionLines);
+         TempExpressionTests.Run(test);
+      }
+   }
+   if (args.Length == 0) {
+      TempExpressionTests.Run("rv = 3 + 4", new Dictionary<string, string> { { "rv", "7" } });
+   }
+} catch (Exception ex) {
+   using (ConsoleHelper.PushBg(ConsoleColor.Red)) {
+      Console.WriteLine($"Failed! {ex}");
+   }
+   return;
+}
+
+using var _ = ConsoleHelper.PushFg(ConsoleColor.Green);
+Console.WriteLine($"Passed!");

src/DataStreamSuite/DataStream.Console/TempExpressionTests.cs

@@ -0,0 +1,32 @@
+namespace ExpressionTests;
+
+using Bts.ArithmeticExpressions;
+
+using DataStream.Console;
+
+public class TempExpressionTests {
+   public static void Run(ExpressionTest test) {
+      Run(test.Expression, test.ExpectedResults);
+   }
+
+   public static void Run(string expression, IDictionary<string, string> expectedResults) {
+      var prg = ArithmeticProgramParser.Parse(expression);
+      var env = new Dictionary<string, object?>();
+      var result = prg.Evaluate(env);
+      foreach (var (key, expected) in expectedResults) {
+         if (!result.TryGetValue(key, out var actual)) {
+            throw new Exception($"ExpectedResults did not contain key '{key}'");
+         }
+         if (!expected.Equals(actual?.ToString() ?? string.Empty)) {
+            throw new Exception($"Expected '{key}' to be {expected} (actually {actual})");
+         }
+         ConsoleHelper.WriteLine($"PASS: {key}={actual}={expected}", ConsoleColor.Green);
+      }
+   }
+
+   static void assert(bool condition) {
+      if (!condition) {
+         throw new Exception();
+      }
+   }
+}

src/DataStreamSuite/DataStream.Console/TestExpressions.txt

@@ -0,0 +1,6 @@
+BEGIN_TEST
+x=3
+y=4
+xySum= x+y
+constSum = 3 + 4
+END_TEST constSum:7 xySum:7

src/DataStreamSuite/DataStream/DataStreamExtensions.cs

@@ -1,26 +0,0 @@
-namespace Bts.DataStream;
-
-using BCG;
-
-using Spark.Core.Disposables;
-
-using System;
-using System.Reactive.Linq;
-
-public static class DataStreamExtensions {
-   public static ViewStream AsViewStream(this IObservable<DataPacket> source) =>
-      source as ViewStream ?? new ObservableViewStream(source);
-
-   public static IObservable<DataPacket> AsObservable(this ViewStream source) {
-      return source as IObservable<DataPacket> ?? Observable.Create<DataPacket>(Subscribe);
-
-      IDisposable Subscribe(IObserver<DataPacket> observer) {
-         source.onChange += OnChange;
-         return Disposable.Create(() => source.onChange -= OnChange);
-
-         void OnChange(object? newValue) =>
-            observer.OnNext(newValue as DataPacket ?? throw new ArgumentException(
-               $"Expected DataPacket, got {newValue?.GetType()}", nameof(newValue)));
-      }
-   }
-}

src/DataStreamSuite/DataStream/Misc/PandasStream.cs

@@ -1,62 +0,0 @@
-// using System;
-// using System.Diagnostics;
-
-// using BCG;
-// using BCG.Utilities.General;
-
-
-// namespace bts {
-
-//     public class PandasStream : ViewStream, IWorker {
-//         //C:\Users\Owner\AppData\Local\Microsoft\WindowsApps\python.exe
-
-//         public PandasStream(ViewStream vs, ViewStreamDatabase vsd, string pandas_code) {
-
-//         }
-
-//         public PandasStream(ViewStream vs, string pandas_code) :
-//         this(vs, ViewStreamDatabase.instance, pandas_code) {
-//         }
-
-
-//         protected override void myOnFirstSubscriber() {
-
-//             // string python_code = @"
-//             //    import winsound
-//             //    frequency = 2500  # Set frequency (in Hz)
-//             //    duration = 1000   # Set duration (in milliseconds)
-//             //    winsound.Beep(frequency, duration)
-//             // ";
-//             string python_code = @"
-//             file_path = \"C:\\Temp\\test2.txt\\"  # r prefix for raw string to avoid escape characters
-//             with open(file_path, \"w\") as f:
-//                 f.write(\"Hello, world!\n\")
-//             ";
-
-//             execute_python_code(python_code);
-//         }
-
-//         static void execute_python_code(string python_code) {
-//             ProcessStartInfo start = new ProcessStartInfo();
-//             start.FileName = "python"; // Python executable
-//             start.Arguments = "-c \"" + python_code.Replace("\"", "\\\"") + "\""; // Pass code as argument
-//             start.UseShellExecute = false;
-//             start.RedirectStandardOutput = true;
-
-//             using (Process process = Process.Start(start)) {
-//                 using (System.IO.StreamReader reader = process.StandardOutput) {
-//                     string result = reader.ReadToEnd();
-//                     Console.Write(result);
-//                 }
-//             }
-//         }
-
-//         protected override void myOnLastSubscriber() {
-//         }
-
-//         public void doWork() {
-
-//         }
-
-//     }
-// }

src/DataStreamSuite/DataStream/Misc/test.py

@@ -1,3 +0,0 @@
-file_path = r"C:\Temp\test2.txt"  # r prefix for raw string to avoid escape characters
-with open(file_path, "w") as f:
-    f.write("Hello, world!\n")

src/DataStreamSuite/DataStream/ObservableViewStream.cs

@@ -1,34 +0,0 @@
-namespace Bts.DataStream;
-
-using BCG;
-
-using bts;
-
-using System;
-
-public class ObservableViewStream : ViewStream, IObservable<DataPacket>, IDisposable {
-   readonly IObservable<DataPacket> source;
-   IDisposable? subscription;
-
-   public ObservableViewStream(IObservable<DataPacket> source) =>
-      this.source = source;
-
-   public void Dispose() {
-      subscription?.Dispose();
-      subscription = null;
-   }
-
-   protected override void myOnFirstSubscriber() {
-      Log.isnull(subscription);
-      subscription = source.Subscribe(myFireEvent);
-   }
-
-   protected override void myOnLastSubscriber() {
-      Log.notnull(subscription);
-      subscription.Dispose();
-      subscription = null;
-   }
-
-   IDisposable IObservable<DataPacket>.Subscribe(IObserver<DataPacket> observer) =>
-      source.Subscribe(observer);
-}

src/DataStreamSuite/DataStream/Primitives/AddColumns/AddConstantColumnRow.cs

@@ -1,12 +0,0 @@
-namespace bts {
-    using BCG;
-
-    using System;
-    using System.Collections;
-
-    public class AddConstantColumnRow {
-        AddConstantColumnRow(IBlueRow r, Tuple<string, Type, object>[] constant_cols) {
-
-        }
-    }
-}

src/DataStreamSuite/DataStream/Primitives/FileStreams/PersistorStream.cs

@@ -15,11 +15,6 @@ namespace BCG {
    // data:
    // epochtime, size, serialized data
    [DataStreamUserRespresentation("pers")]
-   // Saves each udpate of a stream into a file. The token is the the unique logical name of the stream,
-   // which maps the name of the directory where the streams are saved. The directory (3rd arg) is the 
-   // parent directory where the token folder is stored. Each file goes up to rolloevr_size after which a new
-   // file is created. The file names are token_epoch_datetime
-   // the first entry in a file is the complete state
    public class PersistorStream : ViewStream {
       private ViewStream _source_stream;
       private string _token;

src/DataStreamSuite/DataStream/Primitives/FileStreams/SnapshotStream.cs

@@ -11,27 +11,13 @@ using System.Reflection.Metadata;
 
 namespace BCG {
    public class SnapshotStream : ViewStream {
-      public enum SpecificSnapshot {
-         FirstOfDay,
-         LastOfDay
-      };
+
       private BlueTable _current_table;
       private IClientDeserializer _deserilializer = new ClientUniversalDeserializer();
       public DateTime _snapshot_time;
       public string _snapshot_file;
       public IEnumerable<PlaybackPacket> _packets;
-      public SnapshotStream(string snapshot_date, string specific_snapshot, string directory_name, string token) {
-         DateTime dt;
-         var parsed = DateTime.TryParse(snapshot_date, out dt);
-         dt = dt.Date;
-         bts.Log.assert(parsed);
-         SpecificSnapshot specific_snapshot_enum;
-         parsed = Enum.TryParse<SpecificSnapshot>(specific_snapshot, out specific_snapshot_enum);
-         bts.Log.assert(parsed);
-         string snapshot_dt = StreamPersistorUtils.find_snapshot_file($"{directory_name}\\{token}", dt, specific_snapshot_enum);
-         ViewStream rs = new SnapshotStream(snapshot_dt, directory_name, token);
-         mySetRealSource(rs);
-      }
+
 
       public SnapshotStream(string snapshot_dt, string directory_name, string token) {
          _snapshot_time = PersistorStream.parse_datetime(snapshot_dt);

src/DataStreamSuite/DataStream/Primitives/FileStreams/StreamPersistorUtils.cs

@@ -37,20 +37,6 @@ namespace BCG {
       public IEnumerable<PlaybackPacket> packets = new List<PlaybackPacket>();
    }
    public static class StreamPersistorUtils {
-      public static string find_snapshot_file(string directory_name, DateTime snapshot_date, SnapshotStream.SpecificSnapshot specific_snapshot) {
-         DateTime max_dt = DateTime.MinValue;
-         string rv = String.Empty;
-         string[] file_names = Directory.GetFiles(directory_name);
-         for (int i = 0; i < file_names.Count() - 1; i++) {
-            string fn = file_names[i];
-            var dt = parse_datetime_from_filename(fn);
-            if (dt > max_dt) {
-               max_dt = dt;
-               rv = fn;
-            }
-         }
-         return rv;
-      }
       public static string find_snapshot_file(string directory_name, DateTime snapshot_time) {
          string[] file_names = Directory.GetFiles(directory_name);
          for (int i = 0; i < file_names.Count() - 1; i++) {

src/DataStreamSuite/DataStream/Primitives/GroupBy/MaxGroupByEvaluator.cs

@@ -1,42 +1,47 @@
-using System;
-using System.Collections;
+// using System;
+// using System.Collections;
+// using BCG.Utilities;
 
-using BCG.Utilities;
+// namespace BCG
+// {
+// 	/// <summary>
+// 	/// Computes the max for a given column
+// 	/// </summary>
+// 	public class MaxGroupByEvaluator : GroupByEvaluator
+// 	{
 
-namespace BCG {
-    /// <summary>
-    /// Computes the max for a given column
-    /// </summary>
-    public class MaxGroupByEvaluator : GroupByEvaluator {
+//       public MaxGroupByEvaluator (string outputCol, string[] groupByCols) : base (outputCol, groupByCols)
+//       {}
 
-        public MaxGroupByEvaluator(string outputCol, string[] groupByCols) : base(outputCol, groupByCols) { }
+//       public MaxGroupByEvaluator (string colMapEpression, string outputCol, string[] groupByCols)
+//          : base (colMapEpression, outputCol, groupByCols)   {}
 
-        public MaxGroupByEvaluator(string colMapEpression, string outputCol, string[] groupByCols)
-           : base(colMapEpression, outputCol, groupByCols) { }
-
-        public override IBlueRow evaluate(DataPacket dp) {
-            myOutputTable = myOutputTable == null ? new GroupByOutputTable(dp.schema) :
-               myOutputTable;
-            string hashString = BlueDataUtility.rowToString(GroupByStream.getRepRow(dp), myGroupByCols);
-            IBlueRow currMaxRow = myOutputTable.getRow(hashString);
-            IBlueRow maxRow = BlueDataUtility.computeMaxRow(dp.deletedRows, myOutputColumn);
-            if (maxRow != null) {
-                bts.Log.assert(currMaxRow != null);
-                bts.Log.assert(BlueDataUtility.compareRows(maxRow, currMaxRow, myOutputColumn) <= 0);
-                if (BlueDataUtility.compareRows(maxRow, currMaxRow, myOutputColumn) == 0) {
-                    maxRow = BlueDataUtility.computeMaxRow(dp.newRows, myOutputColumn);
-                    if (maxRow == null)
-                        myOutputTable.removeRow(hashString);
-                    else
-                        myOutputTable.addRow(maxRow, hashString);
-                }
-            }
-            maxRow = BlueDataUtility.computeMaxRow(dp.addedRows, myOutputColumn);
-            // if the row was replaced in the above lines, then currMaxRow < maxRow, so the if below would never return a true,
-            // otherwise if maxRow > currMaxRow then its replaced
-            if (maxRow != null && (currMaxRow == null || BlueDataUtility.compareRows(maxRow, currMaxRow, myOutputColumn) > 0))
-                myOutputTable.addRow(maxRow, hashString);
-            return myProject(myOutputTable.getRow(hashString));
-        }
-    }
-}
+//       public override IBlueRow evaluate (DataPacket dp)  
+//       {
+//          myOutputTable = myOutputTable == null ? new GroupByOutputTable (dp.schema) :
+//             myOutputTable;
+//          string hashString = BlueDataUtility.rowToString (GroupByStream.getRepRow (dp), myGroupByCols);
+//          IBlueRow currMaxRow = myOutputTable.getRow (hashString);
+//          IBlueRow maxRow = BlueDataUtility.computeMaxRow (dp.deletedRows, myOutputColumn);
+//          if (maxRow != null)
+//          {
+//             BlueEventLog.logAssert (currMaxRow != null);
+//             BlueEventLog.logAssert (BlueDataUtility.compareRows (maxRow, currMaxRow, myOutputColumn) <= 0);
+//             if (BlueDataUtility.compareRows (maxRow, currMaxRow, myOutputColumn) == 0)
+//             {
+//                maxRow = BlueDataUtility.computeMaxRow (dp.newRows, myOutputColumn);
+//                if (maxRow == null)
+//                   myOutputTable.removeRow (hashString);
+//                else
+//                   myOutputTable.addRow (maxRow, hashString);
+//             }
+//          }
+//          maxRow = BlueDataUtility.computeMaxRow (dp.addedRows, myOutputColumn);
+//          // if the row was replaced in the above lines, then currMaxRow < maxRow, so the if below would never return a true,
+//          // otherwise if maxRow > currMaxRow then its replaced
+//          if (maxRow != null && (currMaxRow == null || BlueDataUtility.compareRows (maxRow, currMaxRow, myOutputColumn) > 0))
+//             myOutputTable.addRow (maxRow, hashString);
+//          return myProject (myOutputTable.getRow (hashString));
+//       }
+// 	}
+// }

src/DataStreamSuite/DataStream/Primitives/GroupBy/MaxStream.cs

@@ -1,15 +1,20 @@
-using System;
+// using System;
 
-namespace BCG {
-    public class MaxStream : ViewStream {
-        public MaxStream(ViewStream stream, string outputCol, string inputCols) : this(stream, outputCol, inputCols.Split(' ')) {
-        }
-        public MaxStream(ViewStream stream, string outputCol, string[] inputCols) {
-            mySetRealSource(new GroupByStream(stream,
-                                                new MaxGroupByEvaluator(outputCol, inputCols)));
-        }
+// namespace BCG
+// {
+// 	public class MaxStream : ViewStream
+// 	{
+//       public MaxStream(ViewStream stream, string outputCol, string inputCols) : this(stream, outputCol, inputCols.Split(' '))
+//       {
+//       }
+// 		public MaxStream (ViewStream stream, string outputCol, string[] inputCols)
+// 		{
+//          mySetRealSource (new GroupByStream (stream, 
+//                                              new MaxGroupByEvaluator (outputCol, inputCols)));
+// 		}
 
-        public MaxStream(ViewStream stream, string outputCol)
-           : this(new AddColumnsStream(stream, "fakeCol=0"), outputCol, new string[] { "fakeCol" }) { }
-    }
-}
+//       public MaxStream (ViewStream stream, string outputCol)
+//          : this (new AddColumnsStream (stream, "fakeCol=0"), outputCol, new string[]{"fakeCol"})
+//       {}
+// 	}
+// }

src/DataStreamSuite/DataStream/Primitives/GroupBy/MinGroupByEvaluator.cs

@@ -1,36 +1,40 @@
-using System;
+// using System;
+// using BCG.Utilities;
 
-using BCG.Utilities;
+// namespace BCG
+// {
+//    public class MinGroupByEvaluator : MaxGroupByEvaluator
+//    {
+//       public MinGroupByEvaluator (string outputCol, string[] groupByCols) 
+//          : base (outputCol, groupByCols)  {}
 
-namespace BCG {
-    public class MinGroupByEvaluator : MaxGroupByEvaluator {
-        public MinGroupByEvaluator(string outputCol, string[] groupByCols)
-           : base(outputCol, groupByCols) { }
+//       public MinGroupByEvaluator (string colMapEpression, string outputCol, string[] groupByCols) :
+//          base (colMapEpression, outputCol, groupByCols)  {}
 
-        public MinGroupByEvaluator(string colMapEpression, string outputCol, string[] groupByCols) :
-           base(colMapEpression, outputCol, groupByCols) { }
+//       public override IBlueRow evaluate (DataPacket dp)  
+//       {
+//          myOutputTable = myOutputTable == null ? new GroupByOutputTable (dp.schema) : myOutputTable;
+//          string stringRep = BlueDataUtility.rowToString (GroupByStream.getRepRow (dp), myGroupByCols);
+//          IBlueRow currMinRow = myOutputTable.getRow (stringRep);
+//          IBlueRow minRow = BlueDataUtility.computeMinRow (dp.deletedRows, myOutputColumn);
+//          if (minRow != null)
+//          {
+//             // BlueEventLog.logAssert (currMinRow != null);
+//             // BlueEventLog.logAssert (BlueDataUtility.compareRows (minRow, currMinRow, myOutputColumn) >= 0);
+//             if (BlueDataUtility.compareRows (minRow, currMinRow, myOutputColumn) == 0)
+//             {
+//                minRow = BlueDataUtility.computeMinRow (dp.newRows, myOutputColumn);
+//                if (minRow == null)
+//                   myOutputTable.removeRow (stringRep);
+//                else
+//                   myOutputTable.addRow (minRow, stringRep);
+//             }
+//          }
+//          minRow = BlueDataUtility.computeMinRow (dp.addedRows, myOutputColumn);
+//          if (minRow != null && (currMinRow == null || BlueDataUtility.compareRows (minRow, currMinRow, myOutputColumn) < 0))
+//             myOutputTable.addRow (minRow, stringRep);
+//          return myOutputTable.getRow (stringRep);
+//       }
 
-        public override IBlueRow evaluate(DataPacket dp) {
-            myOutputTable = myOutputTable == null ? new GroupByOutputTable(dp.schema) : myOutputTable;
-            string stringRep = BlueDataUtility.rowToString(GroupByStream.getRepRow(dp), myGroupByCols);
-            IBlueRow currMinRow = myOutputTable.getRow(stringRep);
-            IBlueRow minRow = BlueDataUtility.computeMinRow(dp.deletedRows, myOutputColumn);
-            if (minRow != null) {
-                // BlueEventLog.logAssert (currMinRow != null);
-                // BlueEventLog.logAssert (BlueDataUtility.compareRows (minRow, currMinRow, myOutputColumn) >= 0);
-                if (BlueDataUtility.compareRows(minRow, currMinRow, myOutputColumn) == 0) {
-                    minRow = BlueDataUtility.computeMinRow(dp.newRows, myOutputColumn);
-                    if (minRow == null)
-                        myOutputTable.removeRow(stringRep);
-                    else
-                        myOutputTable.addRow(minRow, stringRep);
-                }
-            }
-            minRow = BlueDataUtility.computeMinRow(dp.addedRows, myOutputColumn);
-            if (minRow != null && (currMinRow == null || BlueDataUtility.compareRows(minRow, currMinRow, myOutputColumn) < 0))
-                myOutputTable.addRow(minRow, stringRep);
-            return myOutputTable.getRow(stringRep);
-        }
-
-    }
-}
+//    }
+// }

src/DataStreamSuite/DataStream/Primitives/GroupBy/MinStream.cs

@@ -1,13 +1,17 @@
-using System;
+// using System;
 
-namespace BCG {
-    public class MinStream : ViewStream {
-        public MinStream(ViewStream stream, string outputCol, string[] inputCol) {
-            mySetRealSource(new GroupByStream(stream,
-                              new MinGroupByEvaluator(outputCol, inputCol)));
-        }
+// namespace BCG
+// {
+//    public class MinStream : ViewStream
+//    {
+//       public MinStream (ViewStream stream, string outputCol, string[] inputCol)
+//       {
+//          mySetRealSource (new GroupByStream (stream, 
+//                            new MinGroupByEvaluator (outputCol, inputCol)));
+//       }
 
-        public MinStream(ViewStream stream, string outputCol)
-                       : this(new AddColumnsStream(stream, "fakeCol=0"), outputCol, new string[] { "fakeCol" }) { }
-    }
-}
+//       public MinStream (ViewStream stream, string outputCol)
+//                      : this (new AddColumnsStream (stream, "fakeCol=0"), outputCol, new string[]{"fakeCol"})
+//       {}
+//    }
+// }

src/DataStreamSuite/DataStream/Primitives/GroupBy/SumGroupByEvaluator.cs

@@ -1,82 +1,94 @@
-using System;
+// using System;
 
-namespace BCG {
-    /// <summary>
-    /// Computes the sum for a given column
-    /// </summary>
-    public class SumGroupByEvaluator : GroupByEvaluator {
-        public SumGroupByEvaluator(string outputCol, string[] groupByCols) : base(outputCol, groupByCols) { }
+// namespace BCG
+// {
+//    /// <summary>
+//    /// Computes the sum for a given column
+//    /// </summary>
+//    public class SumGroupByEvaluator : GroupByEvaluator
+//    {
+//       public SumGroupByEvaluator (string outputCol, string[] groupByCols) : base (outputCol, groupByCols)
+//       {}
 
-        public SumGroupByEvaluator(string colMapEpression, string outputCol, string[] groupByCols)
-           : base(colMapEpression, outputCol, groupByCols) { }
+//       public SumGroupByEvaluator (string colMapEpression, string outputCol, string[] groupByCols)
+//          : base (colMapEpression, outputCol, groupByCols)   {}
 
-        public override IBlueRow evaluate(DataPacket dp) {
-            myOutputTable = myOutputTable == null ? new GroupByOutputTable(dp.schema) : myOutputTable;
-
-            IBlueRow sumRow = BlueDataUtility.computeSum(dp.newRows, myOutputColumn);
-            return sumRow;
-        }
-    }
+//       public override IBlueRow evaluate (DataPacket dp)  
+//       {
+//          myOutputTable = myOutputTable == null ? new GroupByOutputTable (dp.schema) : myOutputTable;
+         
+//          IBlueRow sumRow = BlueDataUtility.computeSum(dp.newRows, myOutputColumn);
+//          return sumRow;
+//       }
+//    }
 
 
-    public class MultiColumnSumGroupByEvaluator : IGroupByEvaluator {
-        private string[] myOutputCols, myGroupByCols;
-        public MultiColumnSumGroupByEvaluator(string spaceSeperatedOutputCols, string spaceSeperatedGroupByCols) {
-            myOutputCols = spaceSeperatedOutputCols.Split(' ');
-            myGroupByCols = spaceSeperatedGroupByCols.Split(' ');
-        }
+//    public class MultiColumnSumGroupByEvaluator : IGroupByEvaluator
+//    {
+//       private string[] myOutputCols, myGroupByCols;
+//       public MultiColumnSumGroupByEvaluator(string spaceSeperatedOutputCols, string spaceSeperatedGroupByCols)
+//       {
+//          myOutputCols = spaceSeperatedOutputCols.Split(' ');
+//          myGroupByCols = spaceSeperatedGroupByCols.Split(' ');
+//       }
 
-        public MultiColumnSumGroupByEvaluator(string[] output_columns, string[] groupby_columns) {
-            myOutputCols = output_columns;
-            myGroupByCols = groupby_columns;
-        }
+//       public MultiColumnSumGroupByEvaluator(string[] output_columns, string[] groupby_columns)
+//       {
+//          myOutputCols = output_columns;
+//          myGroupByCols = groupby_columns;
+//       }
 
-        #region IGroupByEvaluator Members
+//       #region IGroupByEvaluator Members
 
-        public IBlueRow evaluate(DataPacket packet) { return BlueDataUtility.computeSum(packet.newRows, myOutputCols); }
+//       public IBlueRow evaluate(DataPacket packet)  {return BlueDataUtility.computeSum(packet.newRows, myOutputCols);}
 
-        public string[] groupByColumns { get { return myGroupByCols; } }
+//       public string[] groupByColumns { get { return myGroupByCols; } }
 
-        public void clear() {/*stateless*/}
+//       public void clear()  {/*stateless*/}
 
-        public IBlueTableSchema getOutputSchema(IBlueTableSchema schema) { return schema; }
+//       public IBlueTableSchema getOutputSchema(IBlueTableSchema schema)  {return schema;}
 
-        #endregion
-    }
+//       #endregion
+//    }
 
-    public class SumGroupByEvaluator<T> : GroupByEvaluator {
-        public SumGroupByEvaluator(string outputCol, string[] groupByCols)
-           : base(outputCol, groupByCols) { }
+//    public class SumGroupByEvaluator<T> : GroupByEvaluator
+//    {
+//       public SumGroupByEvaluator(string outputCol, string[] groupByCols)
+//          : base(outputCol, groupByCols)
+//       { }
 
-        public SumGroupByEvaluator(string colMapEpression, string outputCol, string[] groupByCols)
-           : base(colMapEpression, outputCol, groupByCols) { }
+//       public SumGroupByEvaluator(string colMapEpression, string outputCol, string[] groupByCols)
+//          : base(colMapEpression, outputCol, groupByCols) { }
 
-        public override IBlueRow evaluate(DataPacket dp) {
-            myOutputTable = myOutputTable == null ? new GroupByOutputTable(dp.schema) : myOutputTable;
+//       public override IBlueRow evaluate(DataPacket dp)
+//       {
+//          myOutputTable = myOutputTable == null ? new GroupByOutputTable(dp.schema) : myOutputTable;
 
-            IBlueRow sumRow = BlueDataUtility.computeSum<T>(dp.newRows, myOutputColumn);
-            return sumRow;
-        }
-    }
+//          IBlueRow sumRow = BlueDataUtility.computeSum<T>(dp.newRows, myOutputColumn);
+//          return sumRow;
+//       }
+//    }
 
-    public class MultiColumnSumGroupByEvaluator<T> : IGroupByEvaluator {
-        private string[] myOutputCols, myGroupByCols;
-        public MultiColumnSumGroupByEvaluator(string spaceSeperatedOutputCols, string spaceSeperatedGroupByCols) {
-            myOutputCols = spaceSeperatedOutputCols.Split(' ');
-            myGroupByCols = spaceSeperatedGroupByCols.Split(' ');
-        }
+//    public class MultiColumnSumGroupByEvaluator<T> : IGroupByEvaluator
+//    {
+//       private string[] myOutputCols, myGroupByCols;
+//       public MultiColumnSumGroupByEvaluator(string spaceSeperatedOutputCols, string spaceSeperatedGroupByCols)
+//       {
+//          myOutputCols = spaceSeperatedOutputCols.Split(' ');
+//          myGroupByCols = spaceSeperatedGroupByCols.Split(' ');
+//       }
 
-        #region IGroupByEvaluator Members
+//       #region IGroupByEvaluator Members
 
-        public IBlueRow evaluate(DataPacket packet) { return BlueDataUtility.computeSum<T>(packet.newRows, myOutputCols); }
+//       public IBlueRow evaluate(DataPacket packet)  {return BlueDataUtility.computeSum<T>(packet.newRows, myOutputCols);}
 
-        public string[] groupByColumns { get { return myGroupByCols; } }
+//       public string[] groupByColumns { get { return myGroupByCols; } }
 
-        public void clear() {/*stateless*/}
+//       public void clear()  {/*stateless*/}
 
-        public IBlueTableSchema getOutputSchema(IBlueTableSchema schema) { return schema; }
+//       public IBlueTableSchema getOutputSchema(IBlueTableSchema schema)  {return schema;}
 
-        #endregion
-    }
+//       #endregion
+//    }
 
-}
+// }

src/DataStreamSuite/DataStream/Primitives/GroupBy/SumStream.cs

@@ -1,89 +1,107 @@
-using System;
-using System.Collections.Generic;
+// using System;
+// using System.Collections.Generic;
 
-namespace BCG {
-    public class NullMapper : IStreamMapper {
-        private Dictionary<string, bool> _valid_cols = new Dictionary<string, bool>();
+// namespace BCG
+// {
+//    public class NullMapper : IStreamMapper
+//    {
+//       private Dictionary<string, bool> _valid_cols = new Dictionary<string, bool>();
 
-        public NullMapper(string[] valid_cols) {
-            foreach (string c in valid_cols)
-                _valid_cols[c] = true;
-            _valid_cols["id"] = true;
-        }
+//       public NullMapper(string[] valid_cols)
+//       {
+//          foreach (string c in valid_cols)
+//             _valid_cols[c] = true;
+//          _valid_cols["id"] = true;
+//       }
 
-        public IBlueTableSchema getOutputSchema(IBlueTableSchema input_schema) { return input_schema; }
+//       public IBlueTableSchema getOutputSchema(IBlueTableSchema input_schema) { return input_schema; }
 
-        public IBlueRow map(IBlueRow row) {
-            MutableBlueRow out_row = new MutableBlueRow(row);
-            foreach (string c in row.schema.columnNames)
-                if (!_valid_cols.ContainsKey(c))
-                    out_row.setValue(c, DBNull.Value);
-            return out_row;
-        }
-    }
+//       public IBlueRow map(IBlueRow row)
+//       {
+//          MutableBlueRow out_row = new MutableBlueRow(row);
+//          foreach (string c in row.schema.columnNames)
+//             if (!_valid_cols.ContainsKey(c))
+//                out_row.setValue(c, DBNull.Value);
+//          return out_row;
+//       }
+//    }
 
-    public class SumStream : ViewStream {
-        public SumStream(ViewStream stream, string spaceSeparatedOutputCols, string spaceSeparatedGroupByCols) {
-            string[] outputCols = spaceSeparatedOutputCols.Split(' ');
-            string[] groupByCols = spaceSeparatedGroupByCols.Split(' ');
-            mySetRealSource(new GroupByStream(stream,
-                                              outputCols.Length == 1 ? new SumGroupByEvaluator(outputCols[0], groupByCols) as IGroupByEvaluator :
-                                              new MultiColumnSumGroupByEvaluator(spaceSeparatedOutputCols, spaceSeparatedGroupByCols)));
-        }
+//    public class SumStream : ViewStream
+//    {
+//       public SumStream(ViewStream stream, string spaceSeparatedOutputCols, string spaceSeparatedGroupByCols)
+//       {
+//          string[] outputCols = spaceSeparatedOutputCols.Split(' ');
+//          string[] groupByCols = spaceSeparatedGroupByCols.Split(' ');
+//          mySetRealSource(new GroupByStream(stream,
+//                                            outputCols.Length == 1 ? new SumGroupByEvaluator(outputCols[0], groupByCols) as IGroupByEvaluator :
+//                                            new MultiColumnSumGroupByEvaluator(spaceSeparatedOutputCols, spaceSeparatedGroupByCols)));
+//       }
 
-        public SumStream(ViewStream stream, string[] output_columns) {
-            ViewStream rs = new AddColumnsStream(stream, "fake_col=0");
-            rs = new SumStream(rs, output_columns, new string[] { "fake_col" }, false);
-            rs = new ProjectStream(rs, "!fake_col");
-            mySetRealSource(rs);
-        }
+//       public SumStream(ViewStream stream, string[] output_columns)
+//       {
+//          ViewStream rs = new AddColumnsStream(stream, "fake_col=0");
+//          rs = new SumStream(rs, output_columns, new string[] { "fake_col" }, false);
+//          rs = new ProjectStream(rs, "!fake_col");
+//          mySetRealSource(rs);
+//       }
 
-        public SumStream(ViewStream stream, string space_seperated_output_columns, string space_seperated_groupby_columns, bool force_null_into_other_columns) {
-            string[] output_columns = space_seperated_output_columns.Split(' ');
-            string[] groupby_columns = String.IsNullOrEmpty(space_seperated_groupby_columns) ? new string[] { } : space_seperated_groupby_columns.Split(' ');
-            mySetRealSource(new SumStream(stream, output_columns, groupby_columns, force_null_into_other_columns));
-        }
+//       public SumStream(ViewStream stream, string space_seperated_output_columns, string space_seperated_groupby_columns, bool force_null_into_other_columns)
+//       {
+//          string[] output_columns = space_seperated_output_columns.Split(' ');
+//          string[] groupby_columns = String.IsNullOrEmpty (space_seperated_groupby_columns) ? new string[]{} : space_seperated_groupby_columns.Split (' ');
+//          mySetRealSource (new SumStream (stream, output_columns, groupby_columns, force_null_into_other_columns));
+//       }
 
-        public SumStream(ViewStream stream, string[] output_columns, string[] groupby_columns, bool force_null_into_other_columns) {
-            ViewStream rs = null;
-            groupby_columns = groupby_columns ?? new string[] { };
-            if (groupby_columns.Length == 0) {
-                // rs = new AddColumnsStream(stream, "fake_col=0");
-                // rs = new GroupByStream(rs, new MultiColumnSumGroupByEvaluator(output_columns, new string[] { "fake_col" }));
-                // rs = new ProjectStream(rs, "!fake_col");
-            } else {
-                rs = new GroupByStream(stream, new MultiColumnSumGroupByEvaluator(output_columns, groupby_columns));
-            }
-            if (force_null_into_other_columns) {
-                List<string> valid_cols = new List<string>(output_columns);
-                valid_cols.AddRange(groupby_columns);
-                rs = new MapStream(rs, new NullMapper(valid_cols.ToArray()));
-            }
-            mySetRealSource(rs);
-        }
+//       public SumStream(ViewStream stream, string[] output_columns, string[] groupby_columns, bool force_null_into_other_columns)
+//       {
+//          ViewStream rs = null;
+//          groupby_columns = groupby_columns ?? new string[] {};
+//          if (groupby_columns.Length == 0)
+//          {
+//             rs = new AddColumnsStream(stream, "fake_col=0");
+//             rs = new GroupByStream(rs, new MultiColumnSumGroupByEvaluator(output_columns, new string[] {"fake_col"}));
+//             rs = new ProjectStream(rs, "!fake_col");
+//          }
+//          else
+//          {
+//             rs = new GroupByStream(stream, new MultiColumnSumGroupByEvaluator(output_columns, groupby_columns));
+//          }
+//          if (force_null_into_other_columns)
+//          {
+//             List<string> valid_cols = new List<string>(output_columns);
+//             valid_cols.AddRange(groupby_columns);
+//             rs = new MapStream(rs, new NullMapper(valid_cols.ToArray()));
+//          }
+//          mySetRealSource(rs);
+//       }
 
-        public SumStream(ViewStream stream, string outputCol)
-        : this(new AddColumnsStream(stream, "fakeCol=0"), outputCol, "fakeCol") {
-        }
-    }
+//    public SumStream (ViewStream stream, string outputCol)
+//    : this (new AddColumnsStream (stream, "fakeCol=0"), outputCol, "fakeCol")
+//    {
+//    }
+// }
 
-    public class IntSumStream : ViewStream {
-        public IntSumStream(ViewStream stream, string spaceSeparatedOutputCols, string spaceSeparatedGroupByCols) {
-            string[] outputCols = spaceSeparatedOutputCols.Split(' ');
-            string[] groupByCols = spaceSeparatedGroupByCols.Split(' ');
-            mySetRealSource(new GroupByStream(stream,
-                                              outputCols.Length == 1 ? new SumGroupByEvaluator<int>(outputCols[0], groupByCols) as IGroupByEvaluator :
-                                              new MultiColumnSumGroupByEvaluator<int>(spaceSeparatedOutputCols, spaceSeparatedGroupByCols)));
-        }
+// public class IntSumStream : ViewStream
+// {
+//    public IntSumStream (ViewStream stream, string spaceSeparatedOutputCols, string spaceSeparatedGroupByCols)
+//    {
+//       string[] outputCols = spaceSeparatedOutputCols.Split(' ');
+//       string[] groupByCols = spaceSeparatedGroupByCols.Split(' ');
+//       mySetRealSource(new GroupByStream(stream,
+//                                         outputCols.Length == 1 ? new SumGroupByEvaluator<int>(outputCols[0], groupByCols) as IGroupByEvaluator:
+//                                         new MultiColumnSumGroupByEvaluator<int>(spaceSeparatedOutputCols, spaceSeparatedGroupByCols)));
+//    }
 
-    }
+// }
 
 
-    public class SumStream<T> : ViewStream {
-        public SumStream(ViewStream stream, string outputCol, string spaceSeparatedInputCols) {
-            string[] inputCols = spaceSeparatedInputCols.Split(' ');
-            mySetRealSource(new GroupByStream(stream,
-                                                new SumGroupByEvaluator<T>(outputCol, inputCols)));
-        }
-    }
-}
+//    public class SumStream<T> : ViewStream
+//    {
+//       public SumStream(ViewStream stream, string outputCol, string spaceSeparatedInputCols)
+//       {
+//          string[] inputCols = spaceSeparatedInputCols.Split(' ');
+//          mySetRealSource(new GroupByStream(stream,
+//                                              new SumGroupByEvaluator<T>(outputCol, inputCols)));
+//       }
+//    }
+// }

src/DataStreamSuite/DataStream/Primitives/InterRow/RowDiffStream.cs

@@ -1,79 +0,0 @@
-using BCG;
-
-using System.Collections;
-using System.Collections.Generic;
-using System.Linq;
-
-using ZstdSharp.Unsafe;
-
-namespace bts {
-    public class RowDiffStream : ViewStream {
-        private ViewStream _vs;
-        private readonly string _order_col, _col_1, _col_2;
-        private readonly int _diff;
-        private IBlueTableSchema _out_schema;
-        private readonly string _diff_col_name;
-        public RowDiffStream(ViewStream vs, string order_col, string col_1, string col_2, int diff, string diff_col_name) {
-            _vs = vs;
-            _order_col = order_col;
-            _col_1 = col_1;
-            _col_2 = col_2;
-            _diff = diff;
-            _diff_col_name = diff_col_name;
-        }
-
-        private string _diff_col_name_perc {
-            get {
-                return $"{_diff_col_name}_percentage";
-            }
-        }
-
-        protected override void myOnFirstSubscriber() {
-            _vs.onChange += _handle_vs;
-        }
-
-        protected override void myOnLastSubscriber() {
-            _vs.onChange -= _handle_vs;
-        }
-
-        private void _handle_vs(object dp_obj) {
-            var dp = dp_obj as DataPacket;
-            IBlueRowCollection nrows = dp.newRows;
-            _out_schema = BlueDataUtility.add_column(nrows.schema, _diff_col_name, typeof(double));
-            _out_schema = BlueDataUtility.add_column(_out_schema, _diff_col_name_perc, typeof(double));
-            List<IBlueRow> row_list = new List<IBlueRow>();
-            foreach (IBlueRow r in nrows)
-                row_list.Add(r);
-            //OrderBy sorts in ascedning order
-            var sorted_rows = row_list.OrderBy(r => r[_order_col]).ToList();
-            var sorted_row_array = sorted_rows.ToArray();
-            // reverse it to make it descending    
-            sorted_rows.Reverse();
-            //sort rows so we can do the diff
-            OrderedBlueTable rv = new OrderedBlueTable(_out_schema);
-            int i = 0;
-            foreach (IBlueRow r in sorted_rows) {
-                MutableBlueRow nr = new MutableBlueRow(_out_schema, r);
-                //So, in the expression "x - y," where y is subtracted from x, x is the minuend, and y is the subtrahend.
-                // we assume x - y, and name variables like that
-                double x = r.get<double>(_col_1);
-                int y_row_idx = i + _diff;
-                double diff = System.Double.NaN;
-                double perc = System.Double.NaN;
-                if (y_row_idx < sorted_rows.Count()) {
-                    var y = sorted_rows[y_row_idx].get<double>(_col_2);
-                    diff = x - y;
-                    perc = 100.0 * diff / y;
-                }
-                nr.setValue(_diff_col_name, diff);
-                nr.setValue(_diff_col_name_perc, perc);
-                rv.addRow(nr);
-                i++;
-            }
-
-            var dp_out = BlueDataUtility.build_dp_inc(new_rows: rv, myLastValue as DataPacket, true);
-            myFireEvent(dp_out);
-
-        }
-    }
-}

src/DataStreamSuite/DataStream/Primitives/MongoDB/MongoCollectionStream.cs

@@ -1,203 +0,0 @@
-namespace Bts.DataStream;
-
-using BCG;
-
-using bts;
-
-using Bts.DataStreamCore;
-
-using mbts;
-
-using MongoDB.Bson;
-using MongoDB.Driver;
-
-using Spark.Core.Disposables;
-
-using System;
-using System.Collections.Generic;
-using System.Diagnostics.CodeAnalysis;
-using System.Reactive.Subjects;
-using System.Threading;
-using System.Threading.Tasks;
-
-public class MongoCollectionStream : ViewStream, IObservable<DataPacket> {
-   readonly string collectionName;
-   readonly string databaseName;
-   readonly string url;
-   readonly object syncRoot = new();
-   readonly CancellationTokenSource cts = new();
-   readonly Subject<DataPacket> subject = new();
-
-   bool hasStarted = false;
-   IBlueRowFactory<BsonDocument>? rowFactory;
-   MongoClient? clientBacking;
-   IMongoDatabase? databaseBacking;
-   IMongoCollection<BsonDocument>? collectionBacking = null;
-   BlueTableImmutable? table = null;
-
-   public MongoCollectionStream()
-      : this(
-         Environment.GetEnvironmentVariable("MONGO_DB_COLL") ?? string.Empty,
-         Environment.GetEnvironmentVariable("MONGO_DB_NAME") ?? string.Empty,
-         Environment.GetEnvironmentVariable("MONGO_DB_URL") ?? string.Empty) {
-   }
-
-   public MongoCollectionStream(string collectionName)
-      : this(
-         collectionName,
-         Environment.GetEnvironmentVariable("MONGO_DB_NAME") ?? string.Empty,
-         Environment.GetEnvironmentVariable("MONGO_DB_URL") ?? string.Empty) {
-   }
-
-   public MongoCollectionStream(string collectionName, string databaseName)
-      : this(
-         collectionName,
-         databaseName,
-         Environment.GetEnvironmentVariable("MONGO_DB_URL") ?? string.Empty) {
-   }
-
-   public MongoCollectionStream(string collectionName, string databaseName, string url)
-      : this(collectionName, databaseName, url, null) { }
-
-   internal MongoCollectionStream(
-      string collectionName, string databaseName, string url, IBlueRowFactory<BsonDocument>? rowFactory) {
-      this.collectionName = collectionName;
-      this.databaseName = databaseName;
-      this.url = url;
-      this.rowFactory = rowFactory;
-      mySetRealSource(new ObservableViewStream(this));
-   }
-
-   MongoClient client => clientBacking ??= new(url);
-   IMongoDatabase database => databaseBacking ??= client.GetDatabase(databaseName);
-   IMongoCollection<BsonDocument>? collection {
-      get => Volatile.Read(ref collectionBacking);
-      set => Volatile.Write(ref collectionBacking, value);
-   }
-
-   public IDisposable Subscribe(IObserver<DataPacket> observer) {
-      lock (syncRoot) {
-         if (hasStarted) {
-            if (table is not null) {
-               // The first update has already been fired. Seed observer w/ current state.
-               observer.OnNext(new DataPacket(table, table, new BlueTable(table.schema)));
-            }
-            return subject.Subscribe(observer);
-         }
-         hasStarted = true;
-      }
-      var subscription = subject.Subscribe(observer);
-      // Retrieve collection snapshot to bootstrap sequence
-      initTable().ignore();
-      // Watch changes
-      Task.Run(watchChanges);
-      return Disposable.Create(Dispose);
-
-      void Dispose() {
-         Log.Gen.info("Disposing MongoDbChangeObservable subscription");
-         subscription.Dispose();
-         cts.Cancel();
-         cts.Dispose();
-      }
-   }
-
-   protected virtual IBlueRowFactory<BsonDocument> getRowFactory(IEnumerable<BsonDocument> docs) =>
-      MongoSchemaAdaptor.InferSchema(docs);
-
-   async Task watchChanges() {
-      Log.Gen.info(
-         $"MongoCollectionStreamSource. Watching changes. thread: {Environment.CurrentManagedThreadId}");
-      while (collection is null) {
-         await Task.Delay(100);
-      }
-      Log.notnull(collection);
-      using var cursor = collection.Watch(new ChangeStreamOptions {
-         FullDocument = ChangeStreamFullDocumentOption.UpdateLookup
-      });
-      try {
-         while (await cursor.MoveNextAsync(cts.Token)) {
-            // Ignore while waiting for init
-            lock (syncRoot) {
-               if (table is null) {
-                  continue;
-               }
-            }
-            // schemaAdaptor is initialized after table; redundantly assert both
-            // are non-null to be safe, and clarify expectations.
-            Log.notnull(rowFactory);
-            Log.notnull(table);
-
-            var bd = table.ToDataPacketBuilder();
-            foreach (var ch in cursor.Current) {
-               var opDescJson = ch.OperationDescription?.ToJson() ?? string.Empty;
-               var updDescJson = ch.UpdateDescription?.ToJson() ?? string.Empty;
-               Log.Gen.info(
-                  $"MongoCollectionStream.watchChanges. Doc changed. type: {ch.OperationType} " +
-                  $"changes:\n\t\t{updDescJson}");
-               switch (ch.OperationType) {
-                  case ChangeStreamOperationType.Insert: {
-                        bd.AddOrUpdate(Row());
-                        break;
-                     }
-                  case ChangeStreamOperationType.Update: {
-                        bd.Remove(OldRow().Id());
-                        bd.AddOrUpdate(Row());
-                        break;
-                     }
-                  case ChangeStreamOperationType.Replace: {
-                        bd.Remove(OldRow().Id());
-                        bd.AddOrUpdate(Row());
-                        break;
-                     }
-                  case ChangeStreamOperationType.Delete: {
-                        bd.Remove(OldRow().Id());
-                        break;
-                     }
-                  default:
-                     Log.assert(false, $"Unsupported op type '{ch.OperationType}'");
-                     break;
-               }
-
-               IBlueRow Row() =>
-                  rowFactory.GetBlueRow(ch.FullDocumentBeforeChange);
-
-               IBlueRow OldRow() =>
-                  rowFactory.GetBlueRow(ch.FullDocumentBeforeChange);
-            }
-            var dp = bd.ToDataPacket();
-            if (!dp.HasChanges()) {
-               Log.Gen.warn(
-                  $"MongoDbChangeObservable Ignoring update because it didn't change any rows.");
-               continue;
-            }
-            lock (syncRoot) {
-               table = (BlueTableImmutable)dp.newRows;
-               subject.OnNext(dp);
-            }
-         }
-      } catch (TaskCanceledException ex) {
-         Log.Gen.warn(
-            $"MongoDbChangeObservable 'Watch' task cancelled (this is " +
-            $"expected if subscription was disposed). {ex}");
-      }
-   }
-
-
-   [MemberNotNull(nameof(collection))]
-   async Task initTable() {
-      collection = database.GetCollection<BsonDocument>(collectionName);
-      var cursor = collection.Find(Builders<BsonDocument>.Filter.Empty).ToCursor();
-      var docList = await cursor.ToListAsync();
-      rowFactory ??= MongoSchemaAdaptor.InferSchema(docList);
-      var firstTable = new BlueTableImmutable(rowFactory.Schema).ToDataPacketBuilder();
-      foreach (var doc in docList) {
-         var row = rowFactory.GetBlueRow(doc);
-         firstTable.AddOrUpdate(row);
-      }
-      var dp = firstTable.ToDataPacket();
-      lock (syncRoot) {
-         table = (BlueTableImmutable)dp.newRows;
-         subject.OnNext(dp);
-      }
-   }
-}

src/DataStreamSuite/DataStream/Primitives/MongoDB/MongoDBStream.cs

@@ -1,210 +1,212 @@
-namespace BCG;
-
-using BCG.Utilities.General;
-
-using mbts.spark;
-
-using MongoDB.Bson;
-using MongoDB.Driver;
-
 using System;
 using System.Collections.Generic;
+using MongoDB.Driver;
+using MongoDB.Bson;
+using BCG;
+using BCG.Utilities.General;
+using ZstdSharp.Unsafe;
+using System.Runtime.CompilerServices;
+using mbts.spark;
 
-[DataStreamUserRespresentation("mongodb_sql")]
-public class MongoDBSQLStream : ViewStream, IWorker {
-   private ViewStreamDatabase _vsd;
-   private readonly string _connection_string, _db_name, _collection_name;
-   private MongoClient _client;
-   private BlueTable _table;
 
-   // Access the database
-   private IMongoDatabase _database;
-   private IDictionary<string, int> _id_map = new Dictionary<string, int>();
-   // mongodb://192.168.1.80:27017
+namespace BCG {
+    [DataStreamUserRespresentation("mongodb_sql")]
+    public class MongoDBSQLStream : ViewStream, IWorker {
+        private ViewStreamDatabase _vsd;
+        private readonly string _connection_string, _db_name, _collection_name;
+        private MongoClient _client;
+        private BlueTable _table;
 
-   public MongoDBSQLStream(string connection_string, string db_name, string collection_name)
-   : this(connection_string, db_name, collection_name, ViewStreamDatabase.instance) {
+        // Access the database
+        private IMongoDatabase _database;
+        private IDictionary<string, int> _id_map = new Dictionary<string, int>();
+        // mongodb://192.168.1.80:27017
 
-   }
-   public MongoDBSQLStream(string connection_string, string db_name, string collection_name, ViewStreamDatabase vsd) {
-      bts.Log.Gen.info($"MongoDBStream constructed connection_string: {connection_string} " +
-                              $"db_name: {db_name}");
-      _connection_string = connection_string;
-      _db_name = db_name;
-      _collection_name = collection_name;
-      _vsd = vsd;
-      _client = new MongoClient(_connection_string);
-      _database = _client.GetDatabase(_db_name);
-   }
+        public MongoDBSQLStream(string connection_string, string db_name, string collection_name)
+        : this(connection_string, db_name, collection_name, ViewStreamDatabase.instance) {
 
-   protected override void myOnFirstSubscriber() {
-      MasterWorker.getInstance(false).addWorker(this, TimeSpan.FromSeconds(1));
-   }
+        }
+        public MongoDBSQLStream(string connection_string, string db_name, string collection_name, ViewStreamDatabase vsd) {
+            bts.Log.Gen.info($"MongoDBStream constructed connection_string: {connection_string} " +
+                                    $"db_name: {db_name}");
+            _connection_string = connection_string;
+            _db_name = db_name;
+            _collection_name = collection_name;
+            _vsd = vsd;
+            _client = new MongoClient(_connection_string);
+            _database = _client.GetDatabase(_db_name);
+        }
 
-   protected override void myOnLastSubscriber() {
-      MasterWorker.getInstance(false).removeWorker(this);
-   }
+        protected override void myOnFirstSubscriber() {
+            MasterWorker.getInstance(false).addWorker(this, TimeSpan.FromSeconds(1));
+        }
 
-   public void doWork() {
-      _read_mogodb();
-   }
+        protected override void myOnLastSubscriber() {
+            MasterWorker.getInstance(false).removeWorker(this);
+        }
 
-   private void _read_mogodb() {
-      HighPerformanceTimer timer = new HighPerformanceTimer();
-      timer.start();
-      // Access the collection
-      IMongoCollection<BsonDocument> collection = _database.GetCollection<BsonDocument>(name: _collection_name);
+        public void doWork() {
+            _read_mogodb();
+        }
 
-      // Fetch all documents from the collection
-      var documents = collection.Find(Builders<BsonDocument>.Filter.Empty).ToCursor();
-      _table = _table == null ? null : new BlueTable(_table.schema);
-      // Iterate through the documents and print values
-      foreach (var document in documents.ToEnumerable()) {
-         bts.Log.Gen.info($"document: {document} json: {document.ToJson()}");
-         if (_table == null) {
-            _init_table(document);
-         }
-         _add_row(document);
-      }
-      timer.stop();
-      bts.Log.Gen.info($"MongoDBStream time to read {_collection_name} {timer.lastIntervalTime} msecs");
-      timer.start();
-      var dp = BlueDataUtility.build_dp_inc(_table, myLastValue == null ? null : myLastValue as DataPacket, true);
-      timer.stop();
-      bts.Log.Gen.info($"MongoDBStream time to compare tables :{timer.lastIntervalTime} msecs");
-      if (dp.addedRows.Count == 0 && dp.deletedRows.Count == 0)
-         return;
-      myFireEvent(dp);
-   }
+        private void _read_mogodb() {
+            HighPerformanceTimer timer = new HighPerformanceTimer();
+            timer.start();
+            // Access the collection
+            IMongoCollection<BsonDocument> collection = _database.GetCollection<BsonDocument>(name: _collection_name);
 
-   private void _init_table(BsonDocument doc_row) {
-      bts.Log.assert(_table == null);
-      List<ColumnInfo> cis = new List<ColumnInfo>();
-      cis.Add((new ColumnInfo("id", 0, typeof(Int32))));
-      foreach (var n in doc_row.Names) {
-         cis.Add(new ColumnInfo(n, cis.Count, typeof(string)));
-      }
-      IBlueTableSchema schema = new BlueTableSchema(cis.ToArray(), _collection_name);
-      _table = new BlueTable(schema);
-   }
+            // Fetch all documents from the collection
+            var documents = collection.Find(Builders<BsonDocument>.Filter.Empty).ToCursor();
+            _table = _table == null ? null : new BlueTable(_table.schema);
+            // Iterate through the documents and print values
+            foreach (var document in documents.ToEnumerable()) {
+                bts.Log.Gen.info($"document: {document} json: {document.ToJson()}");
+                if (_table == null) {
+                    _init_table(document);
+                }
+                _add_row(document);
+            }
+            timer.stop();
+            bts.Log.Gen.info($"MongoDBStream time to read {_collection_name} {timer.lastIntervalTime} msecs");
+            timer.start();
+            var dp = BlueDataUtility.build_dp_inc(_table, myLastValue == null ? null : myLastValue as DataPacket, true);
+            timer.stop();
+            bts.Log.Gen.info($"MongoDBStream time to compare tables :{timer.lastIntervalTime} msecs");
+            if (dp.addedRows.Count == 0 && dp.deletedRows.Count == 0)
+                return;
+            myFireEvent(dp);
+        }
 
-   private void _add_row(BsonDocument doc) {
-      var key_val = doc["_id"];
-      var key = key_val.ToString();
-      int id;
-      if (!_id_map.TryGetValue(key, out id)) {
-         id = _id_map.Count;
-         _id_map.Add(key, id);
-      }
-      MutableBlueRow nr = new MutableBlueRow(_table.schema);
-      nr.setValue("id", id);
-      foreach (var n in doc.Names) {
-         var val = doc[n].ToString();
-         nr.setValue(n, val);
-      }
-      _table.addRow(nr);
-   }
-}
+        private void _init_table(BsonDocument doc_row) {
+            bts.Log.assert(_table == null);
+            List<ColumnInfo> cis = new List<ColumnInfo>();
+            cis.Add((new ColumnInfo("id", 0, typeof(Int32))));
+            foreach (var n in doc_row.Names) {
+                cis.Add(new ColumnInfo(n, cis.Count, typeof(string)));
+            }
+            IBlueTableSchema schema = new BlueTableSchema(cis.ToArray(), _collection_name);
+            _table = new BlueTable(schema);
+        }
 
-[DataStreamUserRespresentation("mongodb")]
-public class MongoDBStream : ViewStream, IWorker {
-   private ViewStreamDatabase _vsd;
-   private readonly string _connection_string, _db_name, _collection_name;
-   private MongoClient _client;
-   private BlueTable _table;
+        private void _add_row(BsonDocument doc) {
+            var key_val = doc["_id"];
+            var key = key_val.ToString();
+            int id;
+            if (!_id_map.TryGetValue(key, out id)) {
+                id = _id_map.Count;
+                _id_map.Add(key, id);
+            }
+            MutableBlueRow nr = new MutableBlueRow(_table.schema);
+            nr.setValue("id", id);
+            foreach (var n in doc.Names) {
+                var val = doc[n].ToString();
+                nr.setValue(n, val);
+            }
+            _table.addRow(nr);
+        }
+    }
 
-   // Access the database
-   private IMongoDatabase _database;
-   private IDictionary<string, int> _id_map = new Dictionary<string, int>();
-   // mongodb://192.168.1.80:27017
+    [DataStreamUserRespresentation("mongodb")]
+    public class MongoDBStream : ViewStream, IWorker {
+        private ViewStreamDatabase _vsd;
+        private readonly string _connection_string, _db_name, _collection_name;
+        private MongoClient _client;
+        private BlueTable _table;
 
-   public MongoDBStream(string connection_string, string db_name, string collection_name)
-   : this(connection_string, db_name, collection_name, ViewStreamDatabase.instance) {
+        // Access the database
+        private IMongoDatabase _database;
+        private IDictionary<string, int> _id_map = new Dictionary<string, int>();
+        // mongodb://192.168.1.80:27017
 
-   }
-   public MongoDBStream(string connection_string, string db_name, string collection_name, ViewStreamDatabase vsd) {
-      bts.Log.Gen.info($"MongoDBStream constructed connection_string: {connection_string} " +
-                              $"db_name: {db_name}");
-      _connection_string = connection_string;
-      _db_name = db_name;
-      _collection_name = collection_name;
-      _vsd = vsd;
-      _client = new MongoClient(_connection_string);
-      _database = _client.GetDatabase(_db_name);
-   }
+        public MongoDBStream(string connection_string, string db_name, string collection_name)
+        : this(connection_string, db_name, collection_name, ViewStreamDatabase.instance) {
 
-   protected override void myOnFirstSubscriber() {
-      MasterWorker.getInstance(false).addWorker(this, TimeSpan.FromSeconds(1));
-   }
+        }
+        public MongoDBStream(string connection_string, string db_name, string collection_name, ViewStreamDatabase vsd) {
+            bts.Log.Gen.info($"MongoDBStream constructed connection_string: {connection_string} " +
+                                    $"db_name: {db_name}");
+            _connection_string = connection_string;
+            _db_name = db_name;
+            _collection_name = collection_name;
+            _vsd = vsd;
+            _client = new MongoClient(_connection_string);
+            _database = _client.GetDatabase(_db_name);
+        }
 
-   protected override void myOnLastSubscriber() {
-      MasterWorker.getInstance(false).removeWorker(this);
-   }
+        protected override void myOnFirstSubscriber() {
+            MasterWorker.getInstance(false).addWorker(this, TimeSpan.FromSeconds(1));
+        }
 
-   public void doWork() {
-      _read_mogodb();
-   }
+        protected override void myOnLastSubscriber() {
+            MasterWorker.getInstance(false).removeWorker(this);
+        }
 
-   private void _read_mogodb() {
-      HighPerformanceTimer timer = new HighPerformanceTimer();
-      timer.start();
-      // Access the collection
-      IMongoCollection<BsonDocument> collection = _database.GetCollection<BsonDocument>(name: _collection_name);
+        public void doWork() {
+            _read_mogodb();
+        }
 
-      // Fetch all documents from the collection
-      var documents = collection.Find(Builders<BsonDocument>.Filter.Empty).ToCursor();
-      _table = _table == null ? null : new BlueTable(_table.schema);
-      // Iterate through the documents and print values
-      foreach (var document in documents.ToEnumerable()) {
-         bts.Log.Gen.info($"document: {document} json: {document.ToJson()}");
-         if (_table == null) {
-            _init_table(document);
-         }
-         _add_doc(document);
-      }
-      timer.stop();
-      bts.Log.Gen.info($"MongoDBStream time to read {_collection_name} {timer.lastIntervalTime} msecs");
-      timer.start();
-      var dp = BlueDataUtility.build_dp_inc(_table, myLastValue == null ? null : myLastValue as DataPacket, true);
-      timer.stop();
-      bts.Log.Gen.info($"MongoDBStream time to compare tables :{timer.lastIntervalTime} msecs");
-      if (dp.addedRows.Count == 0 && dp.deletedRows.Count == 0)
-         return;
-      myFireEvent(dp);
-   }
+        private void _read_mogodb() {
+            HighPerformanceTimer timer = new HighPerformanceTimer();
+            timer.start();
+            // Access the collection
+            IMongoCollection<BsonDocument> collection = _database.GetCollection<BsonDocument>(name: _collection_name);
 
-   private void _init_table(BsonDocument doc_row) {
-      bts.Log.assert(_table == null);
-      var schema = BlueDataUtility.form_schema($"{_database}_{_table}",
-      Tuple.Create("id", TypeCode.Int32),
-      Tuple.Create("doc_id", TypeCode.String),
-      Tuple.Create("key", TypeCode.String),
-      Tuple.Create("value", TypeCode.String));
-      _table = new BlueTable(schema);
-   }
+            // Fetch all documents from the collection
+            var documents = collection.Find(Builders<BsonDocument>.Filter.Empty).ToCursor();
+            _table = _table == null ? null : new BlueTable(_table.schema);
+            // Iterate through the documents and print values
+            foreach (var document in documents.ToEnumerable()) {
+                bts.Log.Gen.info($"document: {document} json: {document.ToJson()}");
+                if (_table == null) {
+                    _init_table(document);
+                }
+                _add_doc(document);
+            }
+            timer.stop();
+            bts.Log.Gen.info($"MongoDBStream time to read {_collection_name} {timer.lastIntervalTime} msecs");
+            timer.start();
+            var dp = BlueDataUtility.build_dp_inc(_table, myLastValue == null ? null : myLastValue as DataPacket, true);
+            timer.stop();
+            bts.Log.Gen.info($"MongoDBStream time to compare tables :{timer.lastIntervalTime} msecs");
+            if (dp.addedRows.Count == 0 && dp.deletedRows.Count == 0)
+                return;
+            myFireEvent(dp);
+        }
 
-   private void _add_doc(BsonDocument doc) {
-      var doc_id_bson = doc["_id"];
-      var doc_id = doc_id_bson.ToString();
-      Func<string, string, int> get_id = (string doc_id, string key) => {
-         string hash_key = $"{doc_id}_{key}";
-         int id = 0;
-         if (!_id_map.TryGetValue(hash_key, out id)) {
-            id = _id_map.Count;
-            _id_map.Add(hash_key, id);
-         }
-         return id;
-      };
-      foreach (var n in doc.Names) {
-         string key = n;
-         var id = get_id(doc_id, key);
-         MutableBlueRow nr = new MutableBlueRow(_table.schema);
-         var val = doc[n].ToString();
-         nr.setValue("id", id);
-         nr.setValue("doc_id", doc_id);
-         nr.setValue("key", key);
-         nr.setValue("value", val);
-         _table.addRow(nr);
-      }
-   }
-}
+        private void _init_table(BsonDocument doc_row) {
+            bts.Log.assert(_table == null);
+            var schema = BlueDataUtility.form_schema($"{_database}_{_table}",
+            Tuple.Create("id", TypeCode.Int32),
+            Tuple.Create("doc_id", TypeCode.String),
+            Tuple.Create("key", TypeCode.String),
+            Tuple.Create("value", TypeCode.String));
+            _table = new BlueTable(schema);
+        }
+
+        private void _add_doc(BsonDocument doc) {
+            var doc_id_bson = doc["_id"];
+            var doc_id = doc_id_bson.ToString();
+            Func<string, string, int> get_id = (string doc_id, string key) => {
+                string hash_key = $"{doc_id}_{key}";
+                int id = 0;
+                if (!_id_map.TryGetValue(hash_key, out id)) {
+                    id = _id_map.Count;
+                    _id_map.Add(hash_key, id);
+                }
+                return id;
+            };
+            foreach (var n in doc.Names) {
+                string key = n;
+                var id = get_id(doc_id, key);
+                MutableBlueRow nr = new MutableBlueRow(_table.schema);
+                var val = doc[n].ToString();
+                nr.setValue("id", id);
+                nr.setValue("doc_id", doc_id);
+                nr.setValue("key", key);
+                nr.setValue("value", val);
+                _table.addRow(nr);
+            }
+        }
+    }
+
+}

src/DataStreamSuite/DataStream/Primitives/MongoDB/MongoDbExtensions.cs

@@ -1,63 +0,0 @@
-namespace Bts.DataStream;
-
-using MongoDB.Bson;
-
-using System;
-
-public static class MongoDbExtensions {
-   public static BsonArray GetArray(this BsonDocument doc, string elementName) =>
-      doc[elementName].AsBsonArray;
-
-   public static BsonArray? GetArrayOrDefault(this BsonDocument doc, string elementName) =>
-      doc.TryGetValue(elementName, out var val) && val.IsBsonArray ?
-         val.AsBsonArray : null;
-
-   public static string GetString(this BsonDocument doc, string elementName) =>
-      doc[elementName].AsString;
-
-   public static string? GetStringOrDefault(this BsonDocument doc, string elementName) =>
-      doc.TryGetValue(elementName, out var val) && val.IsString ?
-         val.AsString : null;
-
-   public static int GetInt32(this BsonDocument doc, string elementName) =>
-      doc[elementName].AsInt32;
-
-   public static int? GetInt32OrDefault(this BsonDocument doc, string elementName) =>
-      doc.TryGetValue(elementName, out var val) && val.IsInt32 ?
-         val.AsInt32 : null;
-
-   public static long GetInt64(this BsonDocument doc, string elementName) =>
-      doc[elementName].AsInt64;
-
-   public static long? GetInt64OrDefault(this BsonDocument doc, string elementName) =>
-      doc.TryGetValue(elementName, out var val) && val.IsInt64 ?
-         val.AsInt64 : null;
-
-   public static double GetDouble(this BsonDocument doc, string elementName) =>
-      doc[elementName].AsDouble;
-
-   public static double? GetDoubleOrDefault(this BsonDocument doc, string elementName) =>
-      doc.TryGetValue(elementName, out var val) && val.IsDouble ?
-         val.AsDouble : null;
-
-   public static DateTime GetDateTime(this BsonDocument doc, string elementName) =>
-      doc[elementName].ToUniversalTime();
-
-   public static DateTime? GetDateTimeOrDefault(this BsonDocument doc, string elementName) =>
-      doc.TryGetValue(elementName, out var val) && val.IsValidDateTime ?
-         val.ToUniversalTime() : null;
-
-   public static DateTime GetTimestamp(this BsonDocument doc, string elementName) =>
-      doc[elementName].ToUniversalTime();
-
-   public static DateTime? GetTimestampOrDefault(this BsonDocument doc, string elementName) =>
-      doc.TryGetValue(elementName, out var val) && val.IsValidDateTime ?
-         val.ToUniversalTime() : null;
-
-   public static ObjectId GetObjectId(this BsonDocument doc, string elementName) =>
-      doc[elementName].AsObjectId;
-
-   public static ObjectId? GetObjectIdOrDefault(this BsonDocument doc, string elementName) =>
-      doc.TryGetValue(elementName, out var val) && val.IsObjectId ?
-         val.AsObjectId : null;
-}

src/DataStreamSuite/DataStream/Primitives/MongoDB/MongoSchemaAdaptor.cs

@@ -1,57 +0,0 @@
-namespace Bts.DataStream;
-
-using BCG;
-
-using bts;
-
-using MongoDB.Bson;
-
-using System;
-using System.Collections.Generic;
-
-public static class MongoSchemaAdaptor {
-   public static BlueTableSchemaAdaptor<BsonDocument> InferSchema(IEnumerable<BsonDocument> docs) {
-      var schemaAdaptor = new BlueTableSchemaAdaptor<BsonDocument>("MongoCollectionStream");
-      foreach (var doc in docs) {
-         foreach (var el in doc) {
-            if (schemaAdaptor.Contains(el.Name)) {
-               continue;
-            }
-            switch (el.Value.BsonType) {
-               case BsonType.Array:
-                  schemaAdaptor.AddColumn(el.Name, d => d.GetArrayOrDefault(el.Name)?.ToString() ?? string.Empty);
-                  break;
-               case BsonType.String:
-                  schemaAdaptor.AddColumn(el.Name, d => d.GetStringOrDefault(el.Name) ?? string.Empty);
-                  break;
-               case BsonType.Int32:
-                  schemaAdaptor.AddColumn(el.Name, d => d.GetInt32OrDefault(el.Name));
-                  break;
-               case BsonType.Int64:
-                  schemaAdaptor.AddColumn(el.Name, d => d.GetInt64OrDefault(el.Name));
-                  break;
-               case BsonType.Double:
-                  schemaAdaptor.AddColumn(el.Name, d => d.GetDoubleOrDefault(el.Name));
-                  break;
-               case BsonType.DateTime:
-                  schemaAdaptor.AddColumn(el.Name, d => d.GetDateTimeOrDefault(el.Name));
-                  break;
-               case BsonType.Timestamp:
-                  schemaAdaptor.AddColumn(el.Name, d => d.GetTimestampOrDefault(el.Name));
-                  break;
-               case BsonType.ObjectId:
-                  schemaAdaptor.AddColumn(el.Name, d => d.GetObjectIdOrDefault(el.Name)?.ToString());
-                  break;
-               case BsonType.Null:
-                  Log.Gen.warn(
-                     $"Ignoring column '{el.Name}' because at least one element is null");
-                  break;
-               default:
-                  throw new InvalidOperationException(
-                     $"Unsupported BsonType '{el.Value.BsonType}'");
-            }
-         }
-      }
-      return schemaAdaptor;
-   }
-}

src/DataStreamSuite/DataStream/Primitives/SideEffectStreams/SchemaGuesserStream.cs

@@ -1,34 +0,0 @@
-using BCG;
-
-namespace bts {
-    public class SchemaGuesserStream : ViewStream {
-
-        private IBlueRowCollection _rows = null;
-        private ViewStream _vs;
-        public SchemaGuesserStream(ViewStream vs) {
-            _vs = vs;
-        }
-
-        protected override void myOnFirstSubscriber() {
-            _vs.onChange += _on_data;
-        }
-        protected override void myOnLastSubscriber() {
-            _vs.onChange -= _on_data;
-        }
-
-        private void _on_data(object obj) {
-            DataPacket dp = obj as DataPacket;
-            if (_rows == null) {
-                _rows = BlueDataUtility.get_typed_table(dp.newRows);
-            }
-            var dp_out = BlueDataUtility.build_dp_inc(_rows, myLastValue as DataPacket);
-            myFireEvent(dp_out);
-
-
-        }
-
-
-
-
-    }
-}

src/DataStreamSuite/DataStream/Primitives/Sort/SortStream.cs

@@ -1,99 +1,99 @@
-using Bts.ColumnSortExpressions;
-
-using System;
-using System.Collections;
-using System.Collections.Generic;
-using System.Diagnostics.CodeAnalysis;
-
-namespace BCG;
-
-/// <summary>
-/// Defines a Sort stream, with performance of O(n) for all the increments. It uses simple merge sort
-/// to merge the additions to the accumulates table
-/// </summary>
-public class SortStream : ViewStream {
-   ViewStream stream;
-   IList<ColumnSortExpression> sortCols;
-   IComparer? comparer;
-   DataPacket? lastFiredPacket;
-   DataPacket? lastReceivedPacket;
-
-   public SortStream(ViewStream stream, string sortExpression) {
-      this.stream = stream;
-      SortExpression = sortExpression;
-   }
-
-   public SortStream(ViewStream vs, string colName, bool isAsc) {
-      stream = vs;
-      SortExpression = isAsc ? "+" + colName : "-" + colName;
-   }
-
-   public SortStream(ViewStream stream, IComparer comparer) {
-      this.stream = stream;
-      this.comparer = comparer;
-      sortCols = Array.Empty<ColumnSortExpression>();
-   }
-
-   [MemberNotNull(nameof(sortCols))]
-   public string SortExpression {
-      set {
-         comparer = null;
-         lastFiredPacket = null;
-         sortCols = ColumnSortExpressionInterpreter.Evaluate(value);
-         if (lastReceivedPacket is not null) {
-            myChangeHandler(lastReceivedPacket);
-         }
-      }
-   }
-
-   public IComparer Comparer {
-      set {
-         comparer = value;
-         lastFiredPacket = null;
-         if (lastReceivedPacket is not null) {
-            myChangeHandler(lastReceivedPacket);
-         }
-      }
-   }
-
-   public void myChangeHandlerTest(object o) =>
-      myChangeHandler(o);
-
-   [MemberNotNull(nameof(lastFiredPacket))]
-   internal void myChangeHandler(object o) {
-      lastReceivedPacket = (DataPacket)o;
-      // First callback, so we need to sort the whole table, this is the
-      // only time we will do that for all the other sorting operations would
-      // be sorting the increments and inserting them into the whole list
-      if (lastFiredPacket is null) {
-         lastFiredPacket = new() {
-            newRows = comparer is null ?
-               new SortedTable(lastReceivedPacket.newRows, sortCols) :
-               new SortedTable(lastReceivedPacket.newRows, comparer)
-         };
-         lastFiredPacket.addedRows = new OrderedBlueTable(lastFiredPacket.newRows);
-         lastFiredPacket.deletedRows = new BlueTable(lastFiredPacket.newRows.schema);
-         myFireEvent(lastFiredPacket.Clone());
-         return;
-      }
-      // Deletes
-      lastFiredPacket.deletedRows =
-         comparer == null ?
-         new SortedTable(lastReceivedPacket.deletedRows, sortCols) :
-         new SortedTable(lastReceivedPacket.deletedRows, comparer);
-      ((SortedTable)lastFiredPacket.newRows).deleteRows(lastFiredPacket.deletedRows);
-      // Adds
-      lastFiredPacket.addedRows = new BlueTable(lastReceivedPacket.addedRows);
-      ((SortedTable)lastFiredPacket.newRows).addRows(lastFiredPacket.addedRows);
-      myFireEvent(lastFiredPacket.Clone());
-   }
-
-   protected override void myOnFirstSubscriber() =>
-      stream.onChange += myChangeHandler;
-
-   protected override void myOnLastSubscriber() {
-      lastFiredPacket = null;
-      lastReceivedPacket = null;
-      stream.onChange -= myChangeHandler;
-   }
-}
+using Bts.ColumnSortExpressions;
+
+using System;
+using System.Collections;
+using System.Collections.Generic;
+using System.Diagnostics.CodeAnalysis;
+
+namespace BCG;
+
+/// <summary>
+/// Defines a Sort stream, with performance of O(n) for all the increments. It uses simple merge sort
+/// to merge the additions to the accumulates table
+/// </summary>
+public class SortStream : ViewStream {
+   ViewStream stream;
+   IList<ColumnSortExpression> sortCols;
+   IComparer? comparer;
+   DataPacket? lastFiredPacket;
+   DataPacket? lastReceivedPacket;
+
+   public SortStream(ViewStream stream, string sortExpression) {
+      this.stream = stream;
+      SortExpression = sortExpression;
+   }
+
+   public SortStream(ViewStream vs, string colName, bool isAsc) {
+      stream = vs;
+      SortExpression = isAsc ? "+" + colName : "-" + colName;
+   }
+
+   public SortStream(ViewStream stream, IComparer comparer) {
+      this.stream = stream;
+      this.comparer = comparer;
+      sortCols = Array.Empty<ColumnSortExpression>();
+   }
+
+   [MemberNotNull(nameof(sortCols))]
+   public string SortExpression {
+      set {
+         comparer = null;
+         lastFiredPacket = null;
+         sortCols = ColumnSortExpressionInterpreter.Evaluate(value);
+         if (lastReceivedPacket is not null) {
+            myChangeHandler(lastReceivedPacket);
+         }
+      }
+   }
+
+   public IComparer Comparer {
+      set {
+         comparer = value;
+         lastFiredPacket = null;
+         if (lastReceivedPacket is not null) {
+            myChangeHandler(lastReceivedPacket);
+         }
+      }
+   }
+
+   public void myChangeHandlerTest(object o) =>
+      myChangeHandler(o);
+
+   [MemberNotNull(nameof(lastFiredPacket))]
+   internal void myChangeHandler(object o) {
+      lastReceivedPacket = (DataPacket)o;
+      // First callback, so we need to sort the whole table, this is the
+      // only time we will do that for all the other sorting operations would
+      // be sorting the increments and inserting them into the whole list
+      if (lastFiredPacket is null) {
+         lastFiredPacket = new() {
+            newRows = comparer is null ?
+               new SortedTable(lastReceivedPacket.newRows, sortCols) :
+               new SortedTable(lastReceivedPacket.newRows, comparer)
+         };
+         lastFiredPacket.addedRows = new OrderedBlueTable(lastFiredPacket.newRows);
+         lastFiredPacket.deletedRows = new BlueTable(lastFiredPacket.newRows.schema);
+         myFireEvent(lastFiredPacket.Clone());
+         return;
+      }
+      // Deletes
+      lastFiredPacket.deletedRows =
+         comparer == null ?
+         new SortedTable(lastReceivedPacket.deletedRows, sortCols) :
+         new SortedTable(lastReceivedPacket.deletedRows, comparer);
+      ((SortedTable)lastFiredPacket.newRows).deleteRows(lastFiredPacket.deletedRows);
+      // Adds
+      lastFiredPacket.addedRows = new BlueTable(lastReceivedPacket.addedRows);
+      ((SortedTable)lastFiredPacket.newRows).addRows(lastFiredPacket.addedRows);
+      myFireEvent(lastFiredPacket.Clone());
+   }
+
+   protected override void myOnFirstSubscriber() =>
+      stream.onChange += myChangeHandler;
+
+   protected override void myOnLastSubscriber() {
+      lastFiredPacket = null;
+      lastReceivedPacket = null;
+      stream.onChange -= myChangeHandler;
+   }
+}

src/DataStreamSuite/DataStream/Primitives/Top/TopStream.cs

@@ -1,43 +1,49 @@
-using System;
+// using System;
 
-namespace BCG {
-    public class TopStream : MethodDrivenStream {
-        private int myTopCount;
+// namespace BCG
+// {
+//    public class TopStream : MethodDrivenStream
+//    {
+//       private int myTopCount;
 
-        public TopStream(ViewStream stream, int topCount) : base(stream) { myTopCount = topCount; }
+//       public TopStream (ViewStream stream, int topCount) : base (stream)  {myTopCount = topCount;}
 
-        public TopStream(ViewStream stream,
-           int topCount,
-           string sortExpression)
-           :
-           this(sortExpression == null ?
-           stream :
-           new SortStream(stream, sortExpression), topCount) { }
+//       public TopStream (ViewStream stream, 
+//          int topCount, 
+//          string sortExpression) 
+//          :
+//          this (sortExpression == null? 
+//          stream :
+//          new SortStream (stream, sortExpression), topCount)
+//       {}
 
-        protected override void myOnTableClear() {
-            myLastFiredPacket.deletedRows = myLastFiredPacket.newRows;
-            myLastFiredPacket.newRows = new OrderedBlueTable(myLastFiredPacket.deletedRows.schema);
-            myLastFiredPacket.addedRows = new OrderedBlueTable(myLastFiredPacket.deletedRows.schema);
-        }
+//       protected override void myOnTableClear () 
+//       {
+//          myLastFiredPacket.deletedRows = myLastFiredPacket.newRows;
+//          myLastFiredPacket.newRows = new OrderedBlueTable (myLastFiredPacket.deletedRows.schema);
+//          myLastFiredPacket.addedRows = new OrderedBlueTable (myLastFiredPacket.deletedRows.schema);
+//       }
 
-        protected override void myHandleFirstCall() {
-            myLastFiredPacket = new DataPacket();
-            myLastFiredPacket.newRows = new OrderedBlueTable(myLastArrivedPacket.newRows, myTopCount);
-            myLastFiredPacket.addedRows = new OrderedBlueTable(myLastFiredPacket.newRows);
-            myLastFiredPacket.deletedRows = new OrderedBlueTable(myLastFiredPacket.newRows.schema);
-        }
+//       protected override void myHandleFirstCall () 
+//       {
+//          myLastFiredPacket = new DataPacket ();
+//          myLastFiredPacket.newRows = new OrderedBlueTable (myLastArrivedPacket.newRows, myTopCount);
+//          myLastFiredPacket.addedRows = new OrderedBlueTable(myLastFiredPacket.newRows);
+//          myLastFiredPacket.deletedRows = new OrderedBlueTable (myLastFiredPacket.newRows.schema);
+//       }
 
-        protected override void myHandleIncrementalAdd() { }
+//       protected override void myHandleIncrementalAdd ()     {}
 
-        protected override void myHandleIncrementalDelete() {
-            IBlueRowCollection oldRows = myLastFiredPacket.newRows;
-            myLastFiredPacket.newRows = new OrderedBlueTable(myLastArrivedPacket.newRows, myTopCount);
-            BlueDataUtility.computeDifferences(myLastFiredPacket.newRows,
-               oldRows,
-               myLastFiredPacket.schema.columnNames,
-               out myLastFiredPacket.deletedRows,
-               out myLastFiredPacket.addedRows);
-            myFireSwitch = myLastFiredPacket.addedRows.Count + myLastFiredPacket.deletedRows.Count == 0 ? false : true;
-        }
-    }
-}
+//       protected override void myHandleIncrementalDelete()   
+//       {
+//          IBlueRowCollection oldRows = myLastFiredPacket.newRows;
+//          myLastFiredPacket.newRows = new OrderedBlueTable (myLastArrivedPacket.newRows, myTopCount);
+//          BlueDataUtility.computeDifferences (myLastFiredPacket.newRows,
+//             oldRows,
+//             myLastFiredPacket.schema.columnNames,
+//             out myLastFiredPacket.deletedRows,
+//             out myLastFiredPacket.addedRows);
+//          myFireSwitch = myLastFiredPacket.addedRows.Count + myLastFiredPacket.deletedRows.Count == 0 ? false : true;
+//       }
+//    }
+// }

src/DataStreamSuite/DataStreamCore/BlueData/BlueDataUtility.cs

@@ -6,7 +6,6 @@ using System.Reflection;
 using System.IO;
 using System.Diagnostics;
 using System.Linq;
-using System.Globalization;
 
 //using BCG.Utilities;
 using BCG.Utilities.General;
@@ -293,118 +292,21 @@ namespace BCG {
          return TypeCode.Empty;
       }
 
-      // public static Type guess_type(string val) {
-      //    //DateTime
-      //    {
-      //       DateTime dt;
-      //       var success = DateTime.TryParseExact(val, "yyyy-MM-dd HH:mm:ss", null, DateTimeStyles.AllowWhiteSpaces, out dt);
-      //       if (success) {
-      //          return typeof(DateTime);
-      //       }
-      //       DateTimeOffset dtoff;
-      //       success = DateTimeOffset.TryParseExact(val, "yyyy-MM-dd HH:mm:sszzz",
-      //                                              null, System.Globalization.DateTimeStyles.None, out dtoff);
-
-      //       if (success) {
-      //          return typeof(DateTime);
-      //       }
-      //    }
-      //    //int
-      //    {
-      //       int i;
-      //       var success = Int32.TryParse(val, out i);
-      //       if (success) {
-      //          return typeof(Int32);
-      //       }
-      //    }
-      //    //double
-      //    {
-      //       double d;
-      //       var success = Double.TryParse(val, out d);
-      //       if (success) {
-      //          return typeof(Double);
-      //       }
-      //    }
-      //    //bool
-      //    {
-      //       bool b;
-      //       var success = Boolean.TryParse(val, out b);
-      //       if (success) {
-      //          return typeof(bool);
-      //       }
-      //    }
-      //    //string
-      //    return typeof(string);
-      // }
-
-      private static Type[] _parsable_types = { typeof(DateTime), typeof(int), typeof(double), typeof(bool) };
-      public static Type guess_type(string val) {
-         foreach (var t in _parsable_types) {
-            var type_val = convert_to_type(t, val);
-            if (type_val != null)
-               return t;
-         }
-         return typeof(string);
-
-      }
-
-      public static IBlueTableSchema get_typed_schema(IBlueRowCollection rows) {
-         // just use one row to guess
+      public static IBlueTableSchema to_typed_schema(IBlueRowCollection rows) {
          List<ColumnInfo> cis = new List<ColumnInfo>();
-         foreach (IBlueRow r in rows) {
-            foreach (var ci in r.schema.columnInfo) {
-               if (ci.colType != typeof(System.String)) {
-                  cis.Add(ci);
-                  continue;
-               }
-               Type type = guess_type(r[ci.colName] as string);
-               cis.Add(new ColumnInfo(ci.colName, ci.colNo, type));
-            }
-            return new BlueTableSchema(cis.ToArray(), rows.schema.tableName);
+         foreach (var col in rows.schema.columnInfo) {
+            var text_vals = BlueDataUtility.get_colvalues<string>(rows, col.colName);
+            TypeCode type_code = convert_column_type(text_vals);
+            cis.Add(new ColumnInfo(col.colName, cis.Count, _typecode_to_type[type_code]));
          }
-         return null;
+         return new BlueTableSchema(cis.ToArray(), rows.schema.tableName);
       }
 
-      static object convert_to_type(Type t, string value) {
-         // Attempt to convert the string to the specified type
-         try {
-            return Convert.ChangeType(value, t);
-         } catch (Exception) {
-            if (t == typeof(double)) {
-               // some doubles come with currency and ',' we clean that up here (there might be better ways to do this)
-               string cleaned_val = new string(value.Where(c => c != '$' && c != ',').ToArray());
-               cleaned_val = cleaned_val.Replace('−', '-');
-               double rv_d;
-               if (Double.TryParse(cleaned_val, out rv_d))
-                  return rv_d;
-               else
-                  return null;
-            }
-            return null;
-            // Handle conversion failure gracefully
-            // throw new ArgumentException("Failed to convert the string to the specified type.");
-         }
-      }
-
-      // converts a row where all columns are string types to a schema where the best types are guesed
-      public static IBlueRowCollection get_typed_table(IBlueRowCollection string_rows) {
-         Func<IBlueRow, IBlueTableSchema, IBlueRow> get_typed_row = (IBlueRow string_row, IBlueTableSchema typed_schema) => {
-            MutableBlueRow rv = new MutableBlueRow(typed_schema);
-            foreach (ColumnInfo ci in typed_schema.columnInfo) {
-               var typed_val = convert_to_type(ci.colType, value: string_row[ci.colName].ToString());
-               rv.setValue(ci.colName, typed_val);
-            }
-            return rv;
-         };
-         IBlueTableSchema typed_schema = get_typed_schema(string_rows);
-         var rv = new BlueTable(typed_schema);
-         foreach (IBlueRow r in string_rows) {
-            var typed_row = get_typed_row(r, typed_schema);
-            rv.addRow(typed_row);
-         }
-         return rv;
-      }
+      // public static IBlueRow convert_string_row(IBlueRow r, IBlueTableSchema out_schema) {
+      //    foreach (var ci in out_schema.columnInfo){
 
+      //    }
+      // }
 
 
       /// <summary>
@@ -2210,33 +2112,5 @@ namespace BCG {
       }
 
       #endregion
-
-      public class RowComparer<T> : IComparer<IBlueRow> {
-         private readonly IEnumerable<Tuple<string, bool>> _sort_cols;
-         public RowComparer(IEnumerable<Tuple<string, bool>> sort_cols) {
-            _sort_cols = sort_cols;
-         }
-         public int Compare(IBlueRow? x, IBlueRow? y) {
-            Func<object, object, int> compare_cols = (o1, o2) => {
-               return (o1 as IComparable).CompareTo(o2);
-            };
-            foreach (var c in _sort_cols) {
-               int i = compare_cols(x[c.Item1], y[c.Item1]);
-
-            }
-            return 1;
-         }
-
-      }
-      public static IList<IBlueRow> get_sorted_row(IBlueRowCollection rows, IEnumerable<Tuple<string, bool>> sort_cols) {
-         List<IBlueRow> row_list = new List<IBlueRow>();
-         foreach (IBlueRow r in rows) {
-            row_list.Add(r);
-         }
-         var row_comparer = new RowComparer<IBlueRow>(sort_cols);
-         ParallelSort.QuicksortSequential<IBlueRow>(row_list, row_comparer);
-         return row_list;
-
-      }
    }
 }

src/DataStreamSuite/DataStreamCore/BlueData/BlueRowEqualityComparer.cs

@@ -1,24 +0,0 @@
-namespace Bts.DataStreamCore;
-
-using BCG;
-
-using System;
-using System.Collections.Generic;
-using System.Diagnostics.CodeAnalysis;
-using System.Linq;
-
-public class BlueRowEqualityComparer : IEqualityComparer<IBlueRow> {
-   public static BlueRowEqualityComparer Instance { get; } = new();
-
-   public bool Equals(IBlueRow x, IBlueRow y) =>
-      x.schema.columnInfo.SequenceEqual(y.schema.columnInfo) &&
-      x.Values.Cast<object>().SequenceEqual(y.Values.Cast<object>());
-
-   public int GetHashCode([DisallowNull] IBlueRow obj) {
-      var hash = new HashCode();
-      foreach (var v in obj.Values) {
-         hash.Add(v?.GetHashCode() ?? 0);
-      }
-      return hash.ToHashCode();
-   }
-}

src/DataStreamSuite/DataStreamCore/BlueData/BlueTable.cs

@@ -1,15 +1,8 @@
-using Bts.DataStreamCore;
-
-using Spark.DataStream;
-
 using System;
 using System.Collections;
 using System.Collections.Generic;
 using System.Data;
-using System.Diagnostics.CodeAnalysis;
-
 //using BCG.Utilities;
-
 using System.Linq;
 
 namespace BCG {
@@ -17,7 +10,7 @@ namespace BCG {
    /// Anologue of the dotNet DataTable.
    /// </summary>
    [Serializable]
-   public class BlueTable : IBlueRowCollection, IDictionary<int, IBlueRow> {
+   public class BlueTable : IBlueRowCollection {
       /// <summary>
       /// Hashtable indexed on "id" to speed up removes
       /// </summary>
@@ -93,12 +86,6 @@ namespace BCG {
                throw new BlueDataException("ID " + id.ToString() + " does not exist in table " + mySchema.tableName);
             return (IBlueRow)row;
          }
-         set {
-            if ((int)value["id"] != id) {
-               throw new ArgumentException("id arg not equal to row's actual id", nameof(id));
-            }
-            changeRow(value);
-         }
       }
 
       /// <summary>
@@ -215,8 +202,6 @@ namespace BCG {
 
       #endregion
 
-      public IDictionary<int, IBlueRow> AsDictionary() => this;
-
       #region ICollection members
       public virtual IEnumerator GetEnumerator() //{return myRows.GetEnumerator ();}
       { return myIdIndex.Values.GetEnumerator(); }
@@ -229,55 +214,5 @@ namespace BCG {
 
       public virtual int Count { get { return myIdIndex.Count; } }
       #endregion
-
-      #region IDictionary<int, IBlueRow> members
-      public void Add(int key, IBlueRow value) {
-         throw new NotImplementedException();
-      }
-
-      bool IDictionary<int, IBlueRow>.ContainsKey(int key) =>
-         containsID(key);
-
-      bool IDictionary<int, IBlueRow>.Remove(int key) =>
-         deleteRow(key);
-
-      bool IDictionary<int, IBlueRow>.TryGetValue(
-         int key, [MaybeNullWhen(false)] out IBlueRow value) =>
-         tryGetValue(key, out value);
-
-      void ICollection<KeyValuePair<int, IBlueRow>>.Add(KeyValuePair<int, IBlueRow> item) =>
-         addRow(item.Value);
-
-      void ICollection<KeyValuePair<int, IBlueRow>>.Clear() =>
-         clear();
-
-      bool ICollection<KeyValuePair<int, IBlueRow>>.Contains(KeyValuePair<int, IBlueRow> item) =>
-         tryGetValue(item.Key, out var r)
-         && BlueRowEqualityComparer.Instance.Equals(r, item.Value);
-
-      void ICollection<KeyValuePair<int, IBlueRow>>.CopyTo(
-         KeyValuePair<int, IBlueRow>[] array, int arrayIndex) {
-         foreach (DictionaryEntry de in myIdIndex) {
-            array[arrayIndex++] = new((int)de.Key, (IBlueRow)de.Value);
-         }
-      }
-
-      bool ICollection<KeyValuePair<int, IBlueRow>>.Remove(KeyValuePair<int, IBlueRow> item) =>
-         deleteRow(item.Key);
-
-      IEnumerator<KeyValuePair<int, IBlueRow>> IEnumerable<KeyValuePair<int, IBlueRow>>.GetEnumerator() {
-         foreach (DictionaryEntry de in myIdIndex) {
-            yield return new((int)de.Key, (IBlueRow)de.Value);
-         }
-      }
-
-      ICollection<int> IDictionary<int, IBlueRow>.Keys =>
-         GenericCollectionWrapper.Create<int>(myIdIndex.Keys);
-
-      ICollection<IBlueRow> IDictionary<int, IBlueRow>.Values =>
-         GenericCollectionWrapper.Create<IBlueRow>(myIdIndex.Values);
-
-      bool ICollection<KeyValuePair<int, IBlueRow>>.IsReadOnly => false;
-      #endregion
    }
 }

src/DataStreamSuite/DataStreamCore/BlueData/BlueTableImmutable.cs

@@ -1,80 +0,0 @@
-namespace Bts.DataStreamCore;
-
-using BCG;
-
-using System;
-using System.Collections;
-using System.Collections.Immutable;
-
-// I would have named this ImmutableBlueTable, but that was already taken.
-public class BlueTableImmutable : IBlueRowCollection, IImmutableRowCollection {
-   readonly ImmutableDictionary<int, IBlueRow> rows;
-
-   public BlueTableImmutable(IBlueTableSchema schema)
-      : this(schema, ImmutableDictionary<int, IBlueRow>.Empty) { }
-
-   public BlueTableImmutable(IBlueTableSchema schema, ImmutableDictionary<int, IBlueRow> rows) {
-      this.schema = schema;
-      this.rows = rows;
-   }
-
-   public int Count => rows.Count;
-   public bool IsSynchronized => false;
-   public IBlueTableSchema schema { get; }
-   public object SyncRoot => this;
-
-   public IBlueRow this[int id] => rows[id];
-
-   public bool containsID(int id) =>
-      rows.ContainsKey(id);
-
-   public void CopyTo(Array array, int index) {
-      foreach (var r in rows.Values) {
-         array.SetValue(r, index++);
-      }
-   }
-
-   public IEnumerator GetEnumerator() =>
-      rows.Values.GetEnumerator();
-
-   public DataPacketBuilder ToDataPacketBuilder() =>
-      new(this);
-
-   public class DataPacketBuilder {
-      readonly IBlueTableSchema schema;
-      readonly ImmutableDictionary<int, IBlueRow>.Builder table;
-      readonly ImmutableDictionary<int, IBlueRow>.Builder added =
-         ImmutableDictionary<int, IBlueRow>.Empty.ToBuilder();
-      readonly ImmutableDictionary<int, IBlueRow>.Builder removed =
-         ImmutableDictionary<int, IBlueRow>.Empty.ToBuilder();
-
-      public DataPacketBuilder(BlueTableImmutable table) {
-         schema = table.schema;
-         this.table = table.rows.ToBuilder();
-      }
-
-      public void AddOrUpdate(IBlueRow row) {
-         var id = row.Id();
-         added[id] = row;
-         if (table.TryGetValue(id, out var orig)) {
-            removed.Remove(id);
-         }
-      }
-
-      public void Remove(int id) {
-         added.Remove(id);
-         if (table.TryGetValue(id, out var orig)) {
-            removed[id] = orig;
-         }
-      }
-
-      public DataPacket ToDataPacket() {
-         table.RemoveRange(removed.Keys);
-         table.AddRange(added);
-         var newRows = new BlueTableImmutable(schema, table.ToImmutable());
-         var addedRows = new BlueTableImmutable(schema, added.ToImmutable());
-         var removedRows = new BlueTableImmutable(schema, removed.ToImmutable());
-         return new(newRows, addedRows, removedRows);
-      }
-   }
-}

src/DataStreamSuite/DataStreamCore/BlueData/BlueTableSchemaAdaptor.cs

@@ -1,84 +0,0 @@
-#nullable enable
-namespace Bts.DataStream;
-
-using BCG;
-
-using bts.collections;
-
-using System;
-using System.Collections.Generic;
-using System.Linq;
-
-public class BlueTableSchemaAdaptor<T> {
-   readonly string name;
-   readonly List<ColumnAdaptor> columns = [];
-   BlueTableSchema? schema;
-
-   public BlueTableSchemaAdaptor(string name) =>
-      this.name = name;
-
-   public IBlueTableSchema Schema =>
-      schema ??= new BlueTableSchema([.. columns.WrapSelectorRO(c => c.Info)], name);
-
-   public bool Contains(string name) =>
-      columns.Any(c => c.Info.colName == name);
-
-   public void AddColumn(string name, Func<T, int> getValue) =>
-      AddColumn(name, r => getValue(r), typeof(int));
-
-   public void AddColumn(string name, Func<T, int?> getValue) =>
-      AddColumn(name, r => getValue(r) ?? (object)DBNull.Value, typeof(int));
-
-   public void AddColumn(string name, Func<T, long> getValue) =>
-      AddColumn(name, r => getValue(r), typeof(long));
-
-   public void AddColumn(string name, Func<T, long?> getValue) =>
-      AddColumn(name, r => getValue(r) ?? (object)DBNull.Value, typeof(long));
-
-   public void AddColumn(string name, Func<T, double> getValue) =>
-      AddColumn(name, r => getValue(r), typeof(double));
-
-   public void AddColumn(string name, Func<T, double?> getValue) =>
-      AddColumn(name, r => getValue(r) ?? (object)DBNull.Value, typeof(double));
-
-   public void AddColumn(string name, Func<T, DateTime> getValue) =>
-      AddColumn(name, r => getValue(r), typeof(DateTime));
-
-   public void AddColumn(string name, Func<T, DateTime?> getValue) =>
-      AddColumn(name, r => getValue(r) ?? (object)DBNull.Value, typeof(DateTime));
-
-   public void AddColumn(string name, Func<T, string?> getValue) =>
-      AddColumn(name, r => getValue(r) ?? (object)DBNull.Value, typeof(string));
-
-   public void AddColumn(string name, Func<T, object?> getValue, Type dataType) {
-      if (!ColumnInfo.isAllowedType(dataType)) {
-         throw new ArgumentException(
-            $"'{dataType}' is not a supported column type", nameof(dataType));
-      }
-      // Invalidate cached schema (set it to null)
-      schema = null;
-      columns.Add(new ColumnAdaptor(new(name, columns.Count, dataType), r => getValue(r)));
-   }
-
-   public IBlueRow GetBlueRow(T item) {
-      var mbr = new MutableBlueRow(Schema);
-      foreach (var col in columns) {
-         mbr.setValue(col.Info.colNo, col.GetValue(item));
-      }
-      return mbr;
-   }
-
-   class ColumnAdaptor {
-      readonly Func<T, object?> getValue;
-
-      public ColumnAdaptor(ColumnInfo info, Func<T, object?> getValue) {
-         Info = info;
-         this.getValue = getValue;
-      }
-
-      public ColumnInfo Info { get; }
-
-      public object? GetValue(T row) =>
-         getValue(row);
-   }
-}

src/DataStreamSuite/DataStreamCore/BlueData/ColumnInfo.cs

@@ -74,13 +74,13 @@ namespace BCG
       //if it does, we are in serious trouble
       private short myLookupTableGroup;
 
-      [JsonPropertyName("colNo")]
+      [JsonPropertyName("index")]
       public int colNo{get{return myColNo;}}
 
-      [JsonPropertyName("colName")]
+      [JsonPropertyName("name")]
       public string colName {get{return myColName;}}
 
-      [JsonPropertyName("colType")]
+      [JsonPropertyName("type")]
       public Type colType {get{return myColType;}}
 
 

src/DataStreamSuite/DataStreamCore/BlueRowExtensions.cs

@@ -1,11 +0,0 @@
-namespace Bts.DataStreamCore;
-
-using BCG;
-
-using System;
-
-public static class BlueRowExtensions {
-   public static int Id(this IBlueRow row) =>
-      (int)(row["id"] ?? throw new ArgumentException(
-         "Row doesn't have an 'id' column", nameof(row)));
-}

src/DataStreamSuite/DataStreamCore/BlueRowJsonConverter.cs

@@ -1,61 +1,63 @@
-#nullable enable
-using BCG;
+// #nullable enable
+// using BCG;
 
-using System;
-using System.Text.Json;
-using System.Text.Json.Serialization;
+// using Spark.Core.Json;
 
-namespace Bts.DataStreamCore;
+// using System;
+// using System.Text.Json;
+// using System.Text.Json.Nodes;
+// using System.Text.Json.Serialization;
 
-class BlueRowJsonConverter : JsonConverter<IBlueRow> {
-   IBlueTableSchema? schema = null;
+// namespace Bts.DataStreamCore;
 
-   public override IBlueRow Read(
-      ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options) {
-      if (schema is null) {
-         throw new InvalidOperationException("Cannot read row from json because schema is null");
-      }
-      var mbr = new MutableBlueRow(schema);
-      int colNum = 0;
-      switch (reader.TokenType) {
-         case JsonTokenType.StartArray:
-            while (reader.Read() && reader.TokenType != JsonTokenType.EndArray) {
-               mbr.setValue(colNum++, readCellValue(ref reader));
-               colNum++;
-            }
-            break;
-         case JsonTokenType.StartObject:
-            while (reader.Read() && reader.TokenType != JsonTokenType.EndObject) {
-               if (reader.TokenType != JsonTokenType.PropertyName) {
-                  throw new InvalidOperationException(
-                     "Expected 'PropertyName' immediately after 'StartObject' token in row json");
-               }
-               var colName = reader.GetString();
-               if (!reader.Read()) {
-                  throw new InvalidOperationException(
-                     "Failed to read token immediately after 'PropertyName' token in row json");
-               }
-               mbr.setValue(colName, readCellValue(ref reader));
-            }
-            break;
-      }
-      if (mbr.Count != schema.columnInfo.Length) {
-         throw new InvalidOperationException(
-            $"Row json array length '{mbr.Count}' does not match the schema's " +
-            $"column length '{schema.columnInfo.Length}'.");
-      }
-      return mbr;
+// class BlueRowJsonConverter : JsonConverter<IBlueRow> {
+//    public override IBlueRow Read(
+//       ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options) {
+//       var row = JsonSerializer.Deserialize<JsonObject>(ref reader, options)!;
+//       var schemaj = row["schema"]!.GetValue<JsonObject>() ??
+//          throw new InvalidOperationException("Failed to read schema");
+//       // var columns = schemaj["columns"]
+//       var mbr = new MutableBlueRow(schema);
+//       int colNum = 0;
+//       switch (reader.TokenType) {
+//          case JsonTokenType.StartArray:
+//             while (reader.Read() && reader.TokenType != JsonTokenType.EndArray) {
+//                mbr.setValue(colNum++, readCellValue(ref reader));
+//                colNum++;
+//             }
+//             break;
+//          case JsonTokenType.StartObject:
+//             while (reader.Read() && reader.TokenType != JsonTokenType.EndObject) {
+//                if (reader.TokenType != JsonTokenType.PropertyName) {
+//                   throw new InvalidOperationException(
+//                      "Expected 'PropertyName' immediately after 'StartObject' token in row json");
+//                }
+//                var colName = reader.GetString();
+//                if (!reader.Read()) {
+//                   throw new InvalidOperationException(
+//                      "Failed to read token immediately after 'PropertyName' token in row json");
+//                }
+//                mbr.setValue(colName, readCellValue(ref reader));
+//             }
+//             break;
+//       }
+//       if (mbr.Count != schema.columnInfo.Length) {
+//          throw new InvalidOperationException(
+//             $"Row json array length '{mbr.Count}' does not match the schema's " +
+//             $"column length '{schema.columnInfo.Length}'.");
+//       }
+//       return mbr;
 
-      static object? readCellValue(ref Utf8JsonReader reader) =>
-         reader.TokenType switch {
-            JsonTokenType.String => reader.GetString(),
-            JsonTokenType.Number => reader.GetDouble(),
-            JsonTokenType.True or JsonTokenType.False => reader.GetBoolean(),
-            _ => throw new InvalidOperationException("Failed to read cell value"),
-         };
-   }
+//       static object? readCellValue(ref Utf8JsonReader reader) =>
+//          reader.TokenType switch {
+//             JsonTokenType.String => reader.GetString(),
+//             JsonTokenType.Number => reader.GetDouble(),
+//             JsonTokenType.True or JsonTokenType.False => reader.GetBoolean(),
+//             _ => throw new InvalidOperationException("Failed to read cell value"),
+//          };
+//    }
 
-   public override void Write(Utf8JsonWriter writer, IBlueRow value, JsonSerializerOptions options) {
-      throw new NotImplementedException();
-   }
-}
+//    public override void Write(Utf8JsonWriter writer, IBlueRow value, JsonSerializerOptions options) {
+//       throw new NotImplementedException();
+//    }
+// }

src/DataStreamSuite/DataStreamCore/DataPacketJsonConverter.cs

@@ -1,155 +1,186 @@
-#nullable enable
-using BCG;
+// #nullable enable
+// using BCG;
 
-using Spark.Core.Json;
+// using Spark.Core.Json;
 
-using System;
-using System.Collections.Generic;
-using System.Linq;
-using System.Text.Json;
-using System.Text.Json.Serialization;
+// using System;
+// using System.Collections.Generic;
+// using System.Text.Json;
+// using System.Text.Json.Nodes;
+// using System.Text.Json.Serialization;
 
-namespace Bts.DataStreamCore;
+// namespace Bts.DataStreamCore;
 
-class DataPacketJsonConverter : JsonConverter<DataPacket> {
-   /// <summary>
-   /// Whether or not to include a field for current rows (i.e., snapshot of current state). By
-   /// default, only incremental changes are included ('added' and 'deleted').
-   /// </summary>
-   public bool IncludeCurrent { get; set; }
+// public class DataStreamUpdateJsonConverter : JsonConverter<DataStreamUpdate> {
+//    /// <summary>
+//    /// Whether or not to include a field for current rows (i.e., snapshot of current state). By
+//    /// default, only incremental changes are included ('added' and 'deleted').
+//    /// </summary>
+//    public bool IncludeCurrent { get; set; }
 
-   /// <summary>
-   /// Whether to treat rows as objects (name/value pairs), or arrays (list of values). Arrays
-   /// result in smaller payloads, but aren't as easy to read as name/value pairs (for mere humans).
-   /// </summary>
-   public bool UseObjectRows { get; set; }
+//    /// <summary>
+//    /// Whether to treat rows as objects (name/value pairs), or arrays (list of values). Arrays
+//    /// result in smaller payloads, but aren't as easy to read as name/value pairs (for mere humans).
+//    /// </summary>
+//    public bool UseObjectRows { get; set; }
 
-   public override DataPacket? Read(ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options) {
-      if (!reader.ReadObject()) {
-         return null;
-      }
-      if (!reader.ReadPropertyName(out var prop)) {
-         return null;
-      }
-      if (!reader.Read()) {
-         return null;
-      }
-      if (prop == "added") {
-         var added = new List<MutableBlueRow>();
-         if (reader.TokenType != JsonTokenType.StartArray) {
-            return null;
-         }
-         while (true) {
-            if (!reader.Read()) {
-               break;
-            }
-            if (reader.TokenType != JsonTokenType.StartArray || !reader.Read()) {
-               return null;
-            }
-            switch (reader.TokenType) {
-               case JsonTokenType.String:
-                  // DAMNIT! Need the schema.
-                  break;
-            }
-         }
-      }
-      return null;
-   }
+//    public override DataStreamUpdate? Read(ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options) {
+//       var dpj = JsonSerializer.Deserialize<JsonObject>(ref reader);
+//       if (dpj is null) {
+//          return null;
+//       }
+//       var tableName = dpj["tableName"]!.AsString();
+//       var columns = dpj["columns"]!.AsArray()
+//          .Select((c, i) => {
+//             var dataType = c!["type"]!.AsEnum<DataStreamDataType>();
+//             return (info: new ColumnInfo(c["name"]!.AsString(), i, dataType.ToType()), dataType);
+//          })
+//          .ToArray();
+//       var schema = new BlueTableSchema(columns.Cut1().ToArray(), tableName);
+//       return new() {
+//          TableName = tableName,
+//          AddedRows = ToRows(dpj.GetArrayOrEmpty("added")).ToList(),
+//          DeletedRows = ToRows(dpj.GetArrayOrEmpty("deleted")).ToList(),
+//          Columns = [.. schema.columnInfo]
+//       };
 
-   public override void Write(Utf8JsonWriter writer, DataPacket value, JsonSerializerOptions options) {
-      using var _ = writer.WriteObject();
-      using (writer.WriteArray("schema")) {
-         foreach (var col in value.schema.columnInfo) {
-            using (writer.WriteObject()) {
-               writer.WriteString("name", col.colName);
-               writer.WriteNumber("index", col.colNo);
-               // TODO: use language agnostic type names (e.g., 'string', 'integer', 'float', 'date' etc.)
-               writer.WriteString("type", col.colType.ToString());
-            }
-         }
-      }
+//       IEnumerable<IBlueRow> ToRows(JsonArray rows) {
+//          foreach (var row in rows) {
+//             var mbr = new MutableBlueRow(schema);
+//             switch (row!.GetValueKind()) {
+//                case JsonValueKind.Object:
+//                   var rowObj = row.AsObject();
+//                   for (int i = 0; i < columns.Length; i++) {
+//                      SetCellValue(mbr, i, rowObj[columns[i].info.colName]!, columns[i].dataType);
+//                   }
+//                   break;
+//                case JsonValueKind.Array:
+//                   var rowArr = row.AsArray();
+//                   for (int i = 0; i < columns.Length; i++) {
+//                      SetCellValue(mbr, i, rowArr[i]!, columns[i].dataType);
+//                   }
+//                   break;
+//             }
+//             yield return mbr;
+//          }
+//       }
 
-      var cellWriters = value.schema.columnInfo.Select(getCellWriter).ToList();
-      using (writer.WriteArray("added")) {
-         foreach (IBlueRow row in value.addedRows) {
-            using (writer.WriteArray()) {
-               for (int i = 0; i < cellWriters.Count; i++) {
-                  cellWriters[i](row[i]);
-               }
-            }
-         }
-      }
-      using (writer.WriteArray("deleted")) {
-         foreach (IBlueRow row in value.deletedRows) {
-            using (writer.WriteArray()) {
-               for (int i = 0; i < cellWriters.Count; i++) {
-                  cellWriters[i](row[i]);
-               }
-            }
-         }
-      }
-      if (IncludeCurrent) {
-         using (writer.WriteArray("current")) {
-            foreach (IBlueRow row in value.newRows) {
-               using (writer.WriteArray()) {
-                  for (int i = 0; i < cellWriters.Count; i++) {
-                     cellWriters[i](row[i]);
-                  }
-               }
-            }
-         }
-      }
+//       static void SetCellValue(MutableBlueRow mbr, int i, JsonNode cell, DataStreamDataType dataType) {
+//          switch (dataType) {
+//             case DataStreamDataType.String:
+//                mbr.setValue(i, cell.GetValue<string>());
+//                break;
+//             case DataStreamDataType.Int32:
+//                mbr.setValue(i, cell.GetValue<int>());
+//                break;
+//             case DataStreamDataType.Int64:
+//                mbr.setValue(i, cell.GetValue<long>());
+//                break;
+//             case DataStreamDataType.Single:
+//                mbr.setValue(i, cell.GetValue<float>());
+//                break;
+//             case DataStreamDataType.Double:
+//                mbr.setValue(i, cell.GetValue<double>());
+//                break;
+//          }
+//       }
+//    }
 
-      Action<object?> getCellWriter(ColumnInfo colInfo) {
-         var writeVal = getCellValueWriter(colInfo);
-         if (!UseObjectRows) {
-            return writeVal;
-         }
-         return v => {
-            writer.WritePropertyName(colInfo.colName);
-            writeVal(v);
-         };
-      }
+//    public override void Write(Utf8JsonWriter writer, DataStreamUpdate value, JsonSerializerOptions options) {
+//       using var _ = writer.WriteObject();
+//       writer.WriteString("tableName", value.TableName);
+//       using (writer.WriteArray("columns")) {
+//          foreach (var col in value.Columns.empty_if_null()) {
+//             using (writer.WriteObject()) {
+//                writer.WriteString("name", col.colName);
+//                writer.WriteNumber("index", col.colNo);
+//                // TODO: use language agnostic type names (e.g., 'string', 'integer', 'float', 'date' etc.)
+//                writer.WriteString("type", col.colType.ToString());
+//             }
+//          }
+//       }
 
-      Action<object?> getCellValueWriter(ColumnInfo colInfo) {
-         var t = colInfo.colType;
-         if (t == typeof(int)) {
-            return nullCheckWrapper(v => writer.WriteNumberValue((int)v));
-         }
-         if (t == typeof(uint)) {
-            return nullCheckWrapper(v => writer.WriteNumberValue((uint)v));
-         }
-         if (t == typeof(long)) {
-            return nullCheckWrapper(v => writer.WriteNumberValue((long)v));
-         }
-         if (t == typeof(ulong)) {
-            return nullCheckWrapper(v => writer.WriteNumberValue((ulong)v));
-         }
-         if (t == typeof(short)) {
-            return nullCheckWrapper(v => writer.WriteNumberValue((short)v));
-         }
-         if (t == typeof(ushort)) {
-            return nullCheckWrapper(v => writer.WriteNumberValue((ushort)v));
-         }
-         if (t == typeof(double)) {
-            return nullCheckWrapper(v => writer.WriteNumberValue((double)v));
-         }
-         if (t == typeof(float)) {
-            return nullCheckWrapper(v => writer.WriteNumberValue((float)v));
-         }
-         if (t == typeof(bool)) {
-            return nullCheckWrapper(v => writer.WriteBooleanValue((bool)v));
-         }
-         return nullCheckWrapper(v => writer.WriteStringValue(v?.ToString()));
+//       var cellWriters = value.Columns!.Select(getCellWriter).ToList();
+//       using (writer.WriteArray("added")) {
+//          foreach (IBlueRow row in value.AddedRows!) {
+//             using (writer.WriteArray()) {
+//                for (int i = 0; i < cellWriters.Count; i++) {
+//                   cellWriters[i](row[i]);
+//                }
+//             }
+//          }
+//       }
+//       using (writer.WriteArray("deleted")) {
+//          foreach (IBlueRow row in value.DeletedRows!) {
+//             using (writer.WriteArray()) {
+//                for (int i = 0; i < cellWriters.Count; i++) {
+//                   cellWriters[i](row[i]);
+//                }
+//             }
+//          }
+//       }
+//       if (IncludeCurrent) {
+//          using (writer.WriteArray("current")) {
+//             foreach (IBlueRow row in value.newRows) {
+//                using (writer.WriteArray()) {
+//                   for (int i = 0; i < cellWriters.Count; i++) {
+//                      cellWriters[i](row[i]);
+//                   }
+//                }
+//             }
+//          }
+//       }
 
-         Action<object?> nullCheckWrapper(Action<object> writeNonNull) =>
-            v => {
-               if (v is null) {
-                  writer.WriteNullValue();
-               } else {
-                  writeNonNull(v);
-               }
-            };
-      }
-   }
-}
+//       Action<object?> getCellWriter(ColumnInfo colInfo) {
+//          var writeVal = getCellValueWriter(colInfo);
+//          if (!UseObjectRows) {
+//             return writeVal;
+//          }
+//          return v => {
+//             writer.WritePropertyName(colInfo.colName);
+//             writeVal(v);
+//          };
+//       }
+
+//       Action<object?> getCellValueWriter(ColumnInfo colInfo) {
+//          var t = colInfo.colType;
+//          if (t == typeof(int)) {
+//             return nullCheckWrapper(v => writer.WriteNumberValue((int)v));
+//          }
+//          if (t == typeof(uint)) {
+//             return nullCheckWrapper(v => writer.WriteNumberValue((uint)v));
+//          }
+//          if (t == typeof(long)) {
+//             return nullCheckWrapper(v => writer.WriteNumberValue((long)v));
+//          }
+//          if (t == typeof(ulong)) {
+//             return nullCheckWrapper(v => writer.WriteNumberValue((ulong)v));
+//          }
+//          if (t == typeof(short)) {
+//             return nullCheckWrapper(v => writer.WriteNumberValue((short)v));
+//          }
+//          if (t == typeof(ushort)) {
+//             return nullCheckWrapper(v => writer.WriteNumberValue((ushort)v));
+//          }
+//          if (t == typeof(double)) {
+//             return nullCheckWrapper(v => writer.WriteNumberValue((double)v));
+//          }
+//          if (t == typeof(float)) {
+//             return nullCheckWrapper(v => writer.WriteNumberValue((float)v));
+//          }
+//          if (t == typeof(bool)) {
+//             return nullCheckWrapper(v => writer.WriteBooleanValue((bool)v));
+//          }
+//          return nullCheckWrapper(v => writer.WriteStringValue(v?.ToString()));
+
+//          Action<object?> nullCheckWrapper(Action<object> writeNonNull) =>
+//             v => {
+//                if (v is null) {
+//                   writer.WriteNullValue();
+//                } else {
+//                   writeNonNull(v);
+//                }
+//             };
+//       }
+//    }
+// }

src/DataStreamSuite/DataStreamCore/DataStreamCore.csproj

@@ -4,6 +4,7 @@
     <AllowUnsafeBlocks>true</AllowUnsafeBlocks>
     <TargetFramework>net8.0</TargetFramework>
     <Nullable>disable</Nullable>
+    <ImplicitUsings>enable</ImplicitUsings>
   </PropertyGroup>
 
   <ItemGroup>

src/DataStreamSuite/DataStreamCore/DataStreamDataType.cs

@@ -0,0 +1,11 @@
+namespace Bts.DataStreamCore;
+
+public enum DataStreamDataType {
+   Any,
+   Bytes,
+   String,
+   Int32,
+   Int64,
+   Single,
+   Double
+}

src/DataStreamSuite/DataStreamCore/DataStreamExtensions.cs

@@ -0,0 +1,42 @@
+#nullable enable
+
+using Bts.DataStreamCore;
+
+public static class DataStreamExtensions {
+   public static Type ToType(this DataStreamDataType dataType) =>
+      dataType switch {
+         DataStreamDataType.Any => typeof(object),
+         DataStreamDataType.String => typeof(string),
+         DataStreamDataType.Int32 => typeof(int),
+         DataStreamDataType.Int64 => typeof(long),
+         DataStreamDataType.Single => typeof(float),
+         DataStreamDataType.Double => typeof(double),
+         _ => throw new ArgumentException($"DataType '{dataType}' not supported.", nameof(dataType)),
+      };
+
+   public static DataStreamDataType FromType(Type type) {
+      if (type == typeof(object)) {
+         return DataStreamDataType.Any;
+      }
+      if (type == typeof(string)) {
+         return DataStreamDataType.String;
+      }
+      if (type == typeof(int)) {
+         return DataStreamDataType.Int32;
+      }
+      if (type == typeof(long)) {
+         return DataStreamDataType.Int64;
+      }
+      if (type == typeof(float)) {
+         return DataStreamDataType.Single;
+      }
+      if (type == typeof(double)) {
+         return DataStreamDataType.Double;
+      }
+      if (type == typeof(byte[])) {
+         return DataStreamDataType.Bytes;
+      }
+      throw new ArgumentException(
+         $"Unable to convert type '{type}' to {typeof(DataStreamDataType)}");
+   }
+}

src/DataStreamSuite/DataStreamCore/DataStreamUpdate.cs

@@ -1,41 +1,41 @@
-#nullable enable
-using BCG;
-
-using Spark.Core.Json;
-
-using System;
-using System.Collections.Generic;
-using System.Linq;
-using System.Text.Json;
-using System.Text.Json.Serialization;
-
-namespace Bts.DataStreamCore;
-
-class ViewStreamUpdate {
-   [JsonPropertyName("tableName")]
-   public string? TableName { get; set; }
-
-   [JsonPropertyName("columns")]
-   public List<ColumnInfo>? Columns { get; set; }
-
-   [JsonPropertyName("addedRows")]
-   public List<Dictionary<string, object?>>? AddedRows { get; set; }
-
-   [JsonPropertyName("deletedRows")]
-   public List<Dictionary<string, object?>>? DeletedRows { get; set; }
-
-   public DataPacket ToDataPacket(BlueTable state, bool applyChanges = true) {
-      var schema = new BlueTableSchema([.. Columns!], TableName);
-      var addedRowsTable = new BlueTable(schema);
-      var deletedRowsTable = new BlueTable(schema);
-      if (applyChanges) {
-         state.deleteRows(deletedRowsTable);
-         state.addRows(addedRowsTable);
-      }
-      return new(state, addedRowsTable, deletedRowsTable);
-   }
-
-   public static string ToJson() {
-      return "";
-   }
-}
+#nullable enable
+using BCG;
+
+using Spark.Core.Json;
+
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text.Json;
+using System.Text.Json.Serialization;
+
+namespace Bts.DataStreamCore;
+
+public class DataStreamUpdate {
+   [JsonPropertyName("tableName")]
+   public string? TableName { get; set; }
+
+   [JsonPropertyName("columns")]
+   public List<ColumnInfo>? Columns { get; set; }
+
+   [JsonPropertyName("addedRows")]
+   public List<IBlueRow>? AddedRows { get; set; }
+
+   [JsonPropertyName("deletedRows")]
+   public List<IBlueRow>? DeletedRows { get; set; }
+
+   public DataPacket ToDataPacket(BlueTable state, bool applyChanges = true) {
+      var schema = new BlueTableSchema([.. Columns!], TableName);
+      var addedRowsTable = new BlueTable(schema);
+      var deletedRowsTable = new BlueTable(schema);
+      if (applyChanges) {
+         state.deleteRows(deletedRowsTable);
+         state.addRows(addedRowsTable);
+      }
+      return new(state, addedRowsTable, deletedRowsTable);
+   }
+
+   public static string ToJson() {
+      return "";
+   }
+}

src/DataStreamSuite/DataStreamCore/Usings.cs

@@ -0,0 +1 @@
+using System.Linq;

src/DataStreamSuite/DataStreamPublisher/Program.cs

@@ -1,5 +1,4 @@
-using BCG;
-using BCG.Utilities.General;
+using BCG;
 
 using bts;
 using bts.api.core;
@@ -14,8 +13,6 @@ using mbts.config;
 
 using System;
 using System.Collections.Generic;
-using System.IO;
-using System.Linq;
 using System.Threading.Tasks;
 
 await Parser.Default.ParseArguments<DataStreamPublisherConfig>(args)
@@ -72,8 +69,6 @@ static void start_echo_client() {
 void _publish_streams(IEnumerable<KeyValuePair<string, string>> prepublish_streams) {
    PubTableBroker pb_broker = new PubTableBroker();
    CloudStreamManager csm = new CloudStreamManager();
-   string csv_folder = @"C:\Temp\csvs_topublish";
-   CSVPublisher csv_publisher = new CSVPublisher(csv_folder, pb_broker);
    foreach (var kv in prepublish_streams) {
       var name = kv.Key;
       var exp = kv.Value;
@@ -94,36 +89,3 @@ void _publish_streams(IEnumerable<KeyValuePair<string, string>> prepublish_strea
 void on_change(object obj) {
    bts.Log.Gen.info($"DSPublisher on_change dp: {obj}");
 }
-
-public class CSVPublisher : IWorker {
-   private IDictionary<string, ViewPublisher> _pubs = new Dictionary<string, ViewPublisher>();
-   // read from config
-   private readonly string _csv_directory = @"C:\Temp\csvs_topublish\";
-
-   private PubTableBroker _pub_table_broker;
-   public CSVPublisher(string csv_directory, PubTableBroker pub_table_broker) {
-      _csv_directory = csv_directory;
-      _pub_table_broker = pub_table_broker;
-      MasterWorker.getInstance(false).addWorker(this, TimeSpan.FromSeconds(1));
-   }
-   public void doWork() {
-      var file_names = Directory.GetFiles(_csv_directory);
-      foreach (var fn in file_names) {
-         if (!_pubs.ContainsKey(fn)) {
-            _handle_new_file(fn);
-         }
-      }
-   }
-
-   private void _handle_new_file(string file_name) {
-      if (file_name.EndsWith(".csv")) {
-         ViewStream vs = new CSVReaderStream(file_name);
-         vs = new SchemaGuesserStream(vs);
-         var toks = file_name.Split('\\');
-         string table_name = toks.Last();
-         table_name = table_name.Remove(table_name.Length - ".csv".Length);
-         ViewPublisher vp = new ViewPublisher(table_name, _pub_table_broker, vs);
-         _pubs.Add(file_name, vp);
-      }
-   }
-}

src/DataStreamSuite/DataStreamSuite.sln

@@ -48,7 +48,7 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Bts.DataStream.Websocket",
 EndProject
 Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Bts.DataStream.WebsocketServer", "Bts.DataStream.WebsocketServer\Bts.DataStream.WebsocketServer.csproj", "{35823BE0-C883-476A-8B30-D6097CBA1A8D}"
 EndProject
-Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "spark_websocket", "..\client\spark_websocket\spark_websocket.csproj", "{09B0FF33-D34F-4BB4-BE96-B7D35960BB3B}"
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "spark_websocket", "..\client\spark_websocket\spark_websocket.csproj", "{EF492D73-C40F-4EEF-B917-4ADA7B30D728}"
 EndProject
 Global
 	GlobalSection(SolutionConfigurationPlatforms) = preSolution
@@ -151,9 +151,9 @@ Global
 		{35823BE0-C883-476A-8B30-D6097CBA1A8D}.Debug|Any CPU.Build.0 = Debug|Any CPU
 		{35823BE0-C883-476A-8B30-D6097CBA1A8D}.Release|Any CPU.ActiveCfg = Release|Any CPU
 		{35823BE0-C883-476A-8B30-D6097CBA1A8D}.Release|Any CPU.Build.0 = Release|Any CPU
-		{09B0FF33-D34F-4BB4-BE96-B7D35960BB3B}.Debug|Any CPU.ActiveCfg = Debug|x64
-		{09B0FF33-D34F-4BB4-BE96-B7D35960BB3B}.Debug|Any CPU.Build.0 = Debug|x64
-		{09B0FF33-D34F-4BB4-BE96-B7D35960BB3B}.Release|Any CPU.ActiveCfg = Release|x64
-		{09B0FF33-D34F-4BB4-BE96-B7D35960BB3B}.Release|Any CPU.Build.0 = Release|x64
+		{EF492D73-C40F-4EEF-B917-4ADA7B30D728}.Debug|Any CPU.ActiveCfg = Debug|x64
+		{EF492D73-C40F-4EEF-B917-4ADA7B30D728}.Debug|Any CPU.Build.0 = Debug|x64
+		{EF492D73-C40F-4EEF-B917-4ADA7B30D728}.Release|Any CPU.ActiveCfg = Release|x64
+		{EF492D73-C40F-4EEF-B917-4ADA7B30D728}.Release|Any CPU.Build.0 = Release|x64
 	EndGlobalSection
 EndGlobal

src/DataStreamSuite/ExpressionTester/ExpressionTester.csproj

@@ -0,0 +1,14 @@
+<Project Sdk="Microsoft.NET.Sdk">
+
+  <ItemGroup>
+    <ProjectReference Include="..\ExpressionTests\ExpressionTests.csproj" />
+  </ItemGroup>
+
+  <PropertyGroup>
+    <OutputType>Exe</OutputType>
+    <TargetFramework>net8.0</TargetFramework>
+    <ImplicitUsings>enable</ImplicitUsings>
+    <Nullable>enable</Nullable>
+  </PropertyGroup>
+
+</Project>

src/DataStreamSuite/ExpressionTester/Program.cs

@@ -0,0 +1,9 @@
+// See https://aka.ms/new-console-template for more information
+using ExpressionTests;
+
+Console.WriteLine("Hello, World!");
+
+var tester = new DataStreamExpressionTests();
+tester.Setup();
+tester.Test();
+tester.TestLineComments();

src/DataStreamSuite/ExpressionTests/ArithmeticExpressionTest.cs

@@ -0,0 +1,23 @@
+namespace ExpressionTests;
+
+using Bts.ArithmeticExpressions;
+
+public class ArithmeticExpressionTests {
+   [SetUp]
+   public void Setup() {
+   }
+
+   [Test]
+   public void Test() {
+      const string SIMPLE_SUM = "rv = 3 + 4";
+      var interpreter = ArithmeticProgramParser.Parse(SIMPLE_SUM);
+      Assert.NotNull(interpreter);
+      var env = new Dictionary<string, object?>();
+      var result = interpreter.Evaluate(env);
+      Assert.IsTrue(result.TryGetValue("rv", out var res) && Equals(res, 7.0));
+   }
+   
+   public void SimpleOperatorTest() {
+      // Assert.IsTrue(BooleanExpression);
+   }
+}

src/DataStreamSuite/ExpressionTests/DataStreamExpressionTests.cs

@@ -0,0 +1,84 @@
+namespace ExpressionTests;
+
+using BCG;
+
+using Bts.DataStreamExpressions;
+using Bts.ExpressionTests;
+
+using Spark.Core;
+
+public class DataStreamExpressionTests {
+   static string simpleExpression => "literal(\"n\", \"2.0\")";
+
+   [SetUp]
+   public void Setup() { }
+
+   void validateExpression(string expressionStr, bool expectSuccess = true) {
+      var env = new Dictionary<string, StatementExpression>();
+      if (expectSuccess) {
+         var expression = DataStreamExpressionsInterpreter.Evaluate(expressionStr);
+         Assert.NotNull(expression.getView(env));
+      } else {
+         try {
+            // var expression = DataStreamExpressionsInterpreter.TryEvaluate(expressionStr);
+            // Assert.IsNull(expression);
+         } catch {
+         }
+      }
+   }
+
+   void validateExpression(IEnumerable<string> lines, bool expectSuccess = true) {
+      validateExpression("\n".join(lines), expectSuccess);
+   }
+
+   [Test]
+   public void TestLineComments() {
+      var commentBodies = new[] { "comment", "comment w/ odd symbols $@!&*.;: (i.e., those!)" };
+      // Create 2 layers of permutations.
+      //    1. with and without white space between '//' and comment body
+      //    2. with and without leading white space
+      //    3. with and without trailing white space
+      var comments = commentBodies
+         .SelectMany(c => new[] { c, $" {c}", $" {c} ", $"{c} " })
+         .SelectMany(c => new[] { $"//{c}", $"// {c}" })
+         .SelectMany(c => new[] { c, $" {c}", $"{c} ", $" {c} " });
+      foreach (var c in comments) {
+         impl(c);
+      }
+      void impl(string comment) {
+         // Test single line comment (evaluation should fail)
+         var expLines = new List<string>() { comment };
+         validateExpression(expLines, false);
+
+         // Add valid expression after comment (evaluation should succeed)
+         expLines.Add(simpleExpression);
+         validateExpression(expLines);
+
+         // Add comment after expression 
+         expLines.Add(comment);
+         validateExpression(expLines);
+         expLines.Add(string.Empty);
+         validateExpression(expLines);
+      }
+   }
+
+   [Test]
+   public void Test() {
+      var expressionLines = new List<string>() {
+         "// fake comment w/ odd symbols $@!&*.;: (i.e., those!)",
+         "literal(\"n\", \"2.0\")",
+         "// com"
+      };
+      var expressionStr = "\n".join(expressionLines);
+      var expression = DataStreamExpressionsInterpreter.Evaluate(expressionStr + "\n");
+      var env = new Dictionary<string, StatementExpression>();
+      foreach (var s in expression.Statements) {
+         Console.WriteLine("Statement: " + s.ToString());
+      }
+      Assert.NotNull(expression.getView(env));
+   }
+
+   public void SimpleOperatorTest() {
+      // Assert.IsTrue(BooleanExpression);
+   }
+}

src/DataStreamSuite/ExpressionTests/EvaluatesRecursively.cs

@@ -0,0 +1,7 @@
+namespace Bts.ExpressionTests;
+
+class EvaluatesRecursively {
+   public void SetEvaluator(ExpressionEvaluator evaluator) {
+      
+   }
+}

src/DataStreamSuite/ExpressionTests/EvaluatesToFoo.cs

@@ -0,0 +1,24 @@
+namespace Bts.ExpressionTests;
+
+class EvaluatesToFoo : IPropertyEvaluator {
+   public Value? GetValue(string prop) {
+      if (!prop.StartsWith("Foo")) {
+         return null;
+      }
+      if (prop == "Foo.number") {
+         return new Value(42.0);
+      }
+      if (prop == "Foo.list") {
+         var values = new List<Value> {
+               new("foo"),
+               new("bar"),
+               new("baz")
+            };
+         return new(values);
+      }
+      if (prop == "Foo.bool") {
+         return new Value(true);
+      }
+      return new Value("Foo");
+   }
+}

src/DataStreamSuite/ExpressionTests/ExpressionEvaluator.cs

@@ -0,0 +1,13 @@
+namespace Bts.ExpressionTests;
+
+class ExpressionEvaluator {
+   readonly IPropertyEvaluator propEval;
+
+   public ExpressionEvaluator(IPropertyEvaluator propEval) {
+      this.propEval = propEval;
+   }
+   
+   public bool Eval(string expression) {
+      throw new NotImplementedException();
+   }
+}

src/DataStreamSuite/ExpressionTests/ExpressionTests.csproj

@@ -0,0 +1,27 @@
+<Project Sdk="Microsoft.NET.Sdk">
+
+  <PropertyGroup>
+    <TargetFramework>net8.0</TargetFramework>
+    <ImplicitUsings>enable</ImplicitUsings>
+    <Nullable>enable</Nullable>
+
+    <IsPackable>false</IsPackable>
+    <IsTestProject>true</IsTestProject>
+  </PropertyGroup>
+
+  <ItemGroup>
+    <PackageReference Include="Microsoft.NET.Test.Sdk" Version="17.5.0" />
+    <PackageReference Include="NUnit" Version="3.13.3" />
+    <PackageReference Include="NUnit3TestAdapter" Version="4.4.2" />
+    <PackageReference Include="NUnit.Analyzers" Version="3.6.1" />
+    <PackageReference Include="coverlet.collector" Version="3.2.0" />
+    <PackageReference Include="OneOf" Version="3.0.263" />
+  </ItemGroup>
+
+  <ItemGroup>
+    <ProjectReference Include="..\ArithmeticExpressions\ArithmeticExpressions.csproj" />
+    <ProjectReference Include="..\BooleanExpressions\BooleanExpressions.csproj" />
+    <ProjectReference Include="..\DataStreamExpressions\DataStreamExpressions.csproj" />
+  </ItemGroup>
+
+</Project>

src/DataStreamSuite/ExpressionTests/IPropertyEvaluator.cs

@@ -0,0 +1,5 @@
+namespace Bts.ExpressionTests;
+
+interface IPropertyEvaluator {
+   Value? GetValue(string property);
+}

src/DataStreamSuite/ExpressionTests/Usings.cs

@@ -0,0 +1 @@
+global using NUnit.Framework;

src/DataStreamSuite/ExpressionTests/Value.cs

@@ -0,0 +1,20 @@
+namespace Bts.ExpressionTests;
+
+using OneOf;
+
+enum ValueType {
+   Boolean,
+   Integer,
+   Real,
+   String,
+   List,
+   Any
+}
+
+class Value {
+   public Value(OneOf<bool, int, double, string, List<Value>>? value = null) {
+      this.value = value ?? new();
+   }
+
+   public OneOf<bool, int, double, string, List<Value>> value { get; }
+}

src/DataStreamSuite/ExpressionTests/expression_parser_tests.cpp

@@ -0,0 +1,179 @@
+#include "core/bts_core/utest/utest.h"
+
+#include "../IPropertyEvaluator.h"
+#include "../ExpressionEvaluator.h"
+#include "../ParserExceptions.h"
+
+#include "core/bts_core/utils/BTS_SharedPtr.h"
+#include <utility>
+
+using namespace bts::expression;
+
+namespace bts {
+
+   // Simple property evaluator for testing
+   class EvaluatesToFoo : public IPropertyEvaluator
+   {
+      public:
+      EvaluatesToFoo()
+      { }
+      ~EvaluatesToFoo()
+      { }
+      BTS_NO_COPY(EvaluatesToFoo);
+
+      bool get_value(const std::string& prop, Value* out_value) OVERRIDE
+      {
+         bool valid = prop.substr(0,3) == "Foo";
+         if (valid && out_value != nullptr)
+         {
+            if (prop == "Foo.number")
+            {
+               *out_value = Value(42.0);
+            }
+            else if (prop == "Foo.list")
+            {
+               std::vector<Value> values;
+               values.push_back(Value(std::string("foo")));
+               values.push_back(Value(std::string("bar")));
+               values.push_back(Value(std::string("baz")));
+               *out_value = Value(values);
+            }
+            else if (prop == "Foo.bool")
+            {
+               *out_value = Value(true);
+            }
+            else
+            {
+               // Foo.*
+               *out_value = Value(std::string("Foo"));
+            }
+         }
+
+         return valid;
+      }
+   };
+
+   class EvaluatesRecursively : public IPropertyEvaluator
+   {
+   public:
+      EvaluatesRecursively()
+         : _e(nullptr)
+      { }
+      ~EvaluatesRecursively()
+      { }
+      BTS_NO_COPY(EvaluatesRecursively);
+
+      void set_evaluator(ExpressionEvaluator* e)
+      {
+         _e = e;
+      }
+
+      bool get_value(const std::string& prop, Value* out_value) OVERRIDE
+      {
+         std::string recur = "42 == " + prop;
+         if (out_value)
+         {
+            *out_value = Value(_e->eval(recur));
+         }
+         return true;
+      }
+
+      ExpressionEvaluator* _e;
+   };
+
+   struct expression_tests : public TestSuite
+   {
+      expression_tests() { }
+      ~expression_tests() { }
+      BTS_NO_COPY(expression_tests);
+
+      void setup()
+      {
+      }
+
+      void teardown()
+      {
+      }
+
+      void simple_operator_test()
+      {
+         ExpressionEvaluator e(BTS_SharedPtr<IPropertyEvaluator> (new EvaluatesToFoo()));
+         test_assert(e.eval("!!true"));
+         test_assert(e.eval("3 == 3"));
+         test_assert(e.eval("4 > 3"));
+         test_assert(e.eval("3 < 6"));
+         test_assert(e.eval("3 <= 3.5"));
+         test_assert(e.eval("3 in [1,2,3]"));
+         test_assert(!e.eval("3 in [1,2,4]"));
+         test_assert(e.eval("'oo' in Foo.foo"));
+         test_assert(e.eval("'oo' == 'oo'"));
+         test_assert(e.eval("'afoobar' ~= 'foo.*'"));
+         test_assert(!e.eval("'afoobar' ~= '^foo.*'"));
+      }
+
+      void test_bool_conversion()
+      {
+         ExpressionEvaluator e(BTS_SharedPtr<IPropertyEvaluator> (new EvaluatesToFoo()));
+         test_assert(e.eval("true"));
+         test_assert(e.eval("1"));
+         test_assert(e.eval("1.5"));
+         test_assert(e.eval("'true string'"));
+         //test_assert(e.eval("[1,2,3]")); // list not valid here - only after "in"
+         test_assert(e.eval("Foo.x"));
+         test_assert(e.eval("Foo.list"));
+         test_assert(!e.eval("''"));
+      }
+
+      void property_validator_test()
+      {
+         ExpressionEvaluator e(BTS_SharedPtr<IPropertyEvaluator> (new EvaluatesToFoo()));
+         test_assert(e.check_valid("Foo.bar == 'Foo' or 'x' in [Foo.x, Foo.y, 'z']", nullptr));
+         std::string bad_property;
+         test_assert(!e.check_valid("Foo.bar == 'Foo' or 'x' in [Fuu.x, Foo.y, 'z']", &bad_property));
+         test_assert(bad_property == "Fuu.x");
+      }
+
+      void more_complex_test()
+      {
+         ExpressionEvaluator e(BTS_SharedPtr<IPropertyEvaluator> (new EvaluatesToFoo()));
+
+         test_assert(e.eval("Foo.bar == 'Foo' and 'b' in ['a', 'b', 'c', Foo.bar]"));
+         test_assert(e.eval("Foo.number > 2 or (3.5 <= 7 and Foo.number == 3) or 'x' in ['a', 'b', 'c']"));
+         test_assert(!e.eval("(3.5 <= 7 and Foo.number == 3) or 'x' in ['a', 'b', 'c']"));
+         test_assert(e.eval("'foo' in Foo.list and Foo.number == 42.0"));
+         test_assert(e.eval("!!('oo' in Foo.foo)"));
+         test_assert(e.eval("!('ooo' in Foo.foo)"));
+         test_assert(e.eval("!false in [false, false, false, false, true, Foo.bool]"));
+      }
+
+      void recursive_test()
+      {
+         BTS_SharedPtr<EvaluatesRecursively> badprops (new EvaluatesRecursively());
+         ExpressionEvaluator e(badprops);
+         badprops->set_evaluator(&e);
+
+         // Try to get the 'fuzzy'property recursively
+         bool success = false;
+         try
+         {
+            test_assert(e.eval("fuzzy == 42"));
+         }
+         catch(RecursivePropertyGetException)
+         {
+            success = true;
+         }
+         test_assert(success);
+      }
+
+   };
+
+
+   BEGIN_TEST_SUITE(expression_tests)
+      ADD_TEST(simple_operator_test);
+      ADD_TEST(test_bool_conversion);
+      ADD_TEST(property_validator_test);
+      ADD_TEST(more_complex_test);
+      ADD_TEST(recursive_test);
+   END_TEST_SUITE
+
+}

src/DataStreamSuite/PubSub/Testing/SimpleTCPServer.cs

@@ -2,20 +2,16 @@ using System;
 using System.Collections.Generic;
 using bts;
 using System.Threading;
-using System.Threading.Tasks;
 
 namespace BCG {
 
    public class SimpleTCPServer : ITCPConnectionHandler, ITCPMessageHandler {
-      readonly TimeSpan? echoDelay;
-
       int _n_msgs_received = 0;
       bool _isDisposed = false;
       AsyncTCPConnectionAcceptor_APM _acceptor;
       IDictionary<RemoteConnectionInfo, ITCPConnection> _conn_map = new Dictionary<RemoteConnectionInfo, ITCPConnection>();
 
-      public SimpleTCPServer(int port, TimeSpan? echoDelay = null) {
-         this.echoDelay = echoDelay;
+      public SimpleTCPServer(int port) {
          _acceptor = new AsyncTCPConnectionAcceptor_APM(port, this);
       }
 
@@ -38,27 +34,22 @@ namespace BCG {
       public void onData(RemoteMessage msg) {
          Console.WriteLine($"SimpleTCPServer.onData msg: {msg}");
          RemoteConnectionInfo rci = msg.remoteConnectionInfo;
-         if (!_conn_map.TryGetValue(rci, out var this_conn)) {
+         ITCPConnection this_conn;
+         bool found = _conn_map.TryGetValue(rci, out this_conn);
+         if (!found) {
             Console.WriteLine("Could not find connection !!!!!!");
             return;
          }
          _n_msgs_received++;
          Console.WriteLine($"\n\nSimpleTCPServerAsync.onData msgs_received: {_n_msgs_received} msg: {msg} ci: {rci} NOW echoing it back\n\n");
          if (msg.messageType == RemoteMessageType.Data) {
-            Echo();
+            this_conn.send(msg.message);
          } else {
             if (msg.messageType == RemoteMessageType.Disconnection) {
                var ci = msg.remoteConnectionInfo;
                _conn_map.Remove(ci);
             }
          }
-
-         async void Echo() {
-            if (echoDelay.HasValue) {
-               await Task.Delay(echoDelay.Value);
-            }
-            this_conn.send(msg.message);
-         }
       }
 
       public bool isDisposed => _isDisposed;

src/DataStreamSuite/PubSubServer/Configurations/SimpleEchoServer.json

@@ -1,4 +0,0 @@
-{
-  "simple_echo_server": true,
-  "echo_delay": "00:00:02"
-}

src/DataStreamSuite/PubSubServer/Program.cs

@@ -17,10 +17,9 @@ Parser.Default.ParseArguments<PubSubServerConfig>(args)
 
 int Run(PubSubServerConfig cfg) {
    try {
-      cfg.MergeWithConfigFile();
       Configuration.initialize();
       return cfg.SimpleEchoServer ?? false ?
-         RunEchoServer(cfg.Port ?? 12345, cfg.EchoDelay) :
+         RunEchoServer(cfg.Port ?? 12345) :
          RunPubSubServer(cfg);
    } catch (Exception ex) {
       // Write the error to console first, in case the logging system is compromised.
@@ -53,11 +52,11 @@ int RunPubSubServer(PubSubServerConfig cfg) {
    return 0;
 }
 
-int RunEchoServer(short port, TimeSpan? echoDelay) {
+int RunEchoServer(short port) {
    Console.WriteLine("Running as TCP Echo server, Hit enter to continue");
    Console.ReadLine();
    Console.WriteLine($"ready to listen to port: {port}");
-   var sts = new SimpleTCPServer(port, echoDelay);
+   var sts = new SimpleTCPServer(port);
    sts.init();
    Console.CancelKeyPress += OnCancelKeyPress;
    while (true) {

src/DataStreamSuite/PubSubServer/PubSubServerConfig.cs

@@ -38,13 +38,9 @@ public class PubSubServerConfig {
    public string? Sparkd { get; set; }
 
    [JsonPropertyName("simple_echo_server")]
-   [Option("simple-echo-server", HelpText = "Run a simple echo server (won't connect to sparkd)")]
+   [Option("simple-echo-server", HelpText = "Run a simple echo server (won'connect to sparkd)")]
    public bool? SimpleEchoServer { get; set; }
 
-   [JsonPropertyName("echo_delay")]
-   [Option("echo-delay", HelpText = "The amount of time to delay echo replies (must run with --simple-echo-server)")]
-   public TimeSpan? EchoDelay { get; set; }
-
    [JsonIgnore]
    [Option(
       "config",

src/DataStreamSuite/SharedStreams/Business/MarketData/Charting/TradeBarsStream.cs

@@ -1,161 +0,0 @@
-using BCG;
-using BCG.Utilities.General;
-
-using System;
-
-namespace bts {
-    [DataStreamUserRespresentation("tradebars")]
-    public class TradeBarsStream : ViewStream, IWorker {
-
-        public class TradeBar {
-            static double[] calculate_log_returns(double[] prices) {
-                // Calculate logarithmic returns
-                double[] returns = new double[prices.Length - 1];
-                for (int i = 1; i < prices.Length; i++) {
-                    returns[i - 1] = Math.Log(prices[i] / prices[i - 1]);
-                }
-                return returns;
-            }
-
-            static double calculate_volatility(double[] returns) {
-                // Calculate standard deviation of logarithmic returns
-                double mean = returns.Average();
-                double sum_of_squares = returns.Sum(x => Math.Pow(x - mean, 2));
-                double variance = sum_of_squares / (returns.Length - 1);
-                double volatility = Math.Sqrt(variance);
-
-                return volatility;
-            }
-            public void add_trade(IBlueRow r) {
-                double price = r.get<double>("price");
-                //prices.Add(price);
-                double qty = r.get<double>(col_name: "qty");
-                if (n_entries == 0) {
-                    open = price;
-                    low = price < low ? price : low;
-                    high = price > high ? price : high;
-                }
-                n_entries++;
-                volume += qty;
-                vwap_sum += price * qty;
-                close = price;
-                if ("Buy".Equals(r.get<string>("aggressor"))) {
-                    n_buys_aggs++;
-                } else {
-                    n_sell_aggs++;
-                }
-            }
-            private List<double> prices = new List<double>();
-            public readonly string security;
-            public double low = Double.MaxValue, high, open, close, median, std_dev, vwap_sum, volume;
-            public uint n_buys_aggs, n_sell_aggs, n_entries;
-            public DateTime bar_start = DateTime.Now;
-            public TradeBar(string security_) {
-                security = security_;
-            }
-            public double volatility {
-                get {
-                    //if (prices.Count == 1)
-                    return 0.0;
-                    var log_returns = calculate_log_returns(prices.ToArray());
-                    return calculate_volatility(log_returns);
-                }
-            }
-        }
-
-        private readonly int _bar_period_seconds;
-        private readonly ViewStreamDatabase _vsd;
-        private ViewStream _timesales_stream;
-        private ViewStream _bbo_stream;
-        private IDictionary<uint, TradeBar> _sec_bar_map = new Dictionary<uint, TradeBar>();
-        private bool _dirty = false;
-        private TimeSpan _fire_period;
-        private DateTime _last_fire_time = DateTime.Now;
-        private BlueTable _table = new BlueTable(BlueDataUtility.form_schema("TradeBars",
-        Tuple.Create("id", TypeCode.Int32),
-        Tuple.Create("bar_start", TypeCode.DateTime),
-        Tuple.Create("bar_end", TypeCode.DateTime),
-        Tuple.Create("day_of_week", TypeCode.String),
-        Tuple.Create("n_trades", TypeCode.UInt32),
-        Tuple.Create("security", TypeCode.String),
-        Tuple.Create("sec_key", TypeCode.UInt32),
-        Tuple.Create("low", TypeCode.Double),
-        Tuple.Create("high", TypeCode.Double),
-        Tuple.Create("open", TypeCode.Double),
-        Tuple.Create("close", TypeCode.Double),
-        Tuple.Create("vwap_sum", TypeCode.Double),
-        Tuple.Create("volume", TypeCode.Double),
-        Tuple.Create("n_buy_aggs", TypeCode.UInt32),
-        Tuple.Create("n_sell_aggs", TypeCode.UInt32),
-        Tuple.Create("volatility", TypeCode.Double)));
-
-
-        public TradeBarsStream(int bar_period_seconds)
-        : this(bar_period_seconds, ViewStreamDatabase.instance) { }
-        public TradeBarsStream(int bar_period_seconds, ViewStreamDatabase vsd) {
-            _vsd = vsd;
-            _bar_period_seconds = bar_period_seconds;
-            _timesales_stream = new TimeSalesStream(10000, vsd);
-            _fire_period = TimeSpan.FromSeconds(bar_period_seconds);
-            _bbo_stream = new BBOStream(_vsd);
-        }
-
-        protected override void myOnFirstSubscriber() {
-            _timesales_stream.onChange += _on_timesales;
-            _bbo_stream.onChange += _on_bbo;
-            _vsd.addWorker(this);
-        }
-
-        protected override void myOnLastSubscriber() {
-            _vsd.removeWorker(this);
-            _timesales_stream.onChange -= _on_timesales;
-            _bbo_stream.onChange -= _on_bbo;
-        }
-
-        public void doWork() {
-            if (DateTime.Now - _last_fire_time > _fire_period) {
-                if (_dirty) {
-                    _dirty = false;
-                    _last_fire_time = DateTime.Now;
-                    BlueTable added_rows = new BlueTable(_table.schema);
-                    foreach (var kv in _sec_bar_map) {
-                        var sec_key = kv.Key;
-                        var tbar = kv.Value;
-                        IBlueRow new_row = new BlueRow(_table.schema, _table.Count, tbar.bar_start, DateTime.Now,
-                        tbar.bar_start.DayOfWeek.ToString(),
-                                                        tbar.n_entries, tbar.security, sec_key, tbar.low, tbar.high, tbar.open,
-                                                        tbar.close, tbar.vwap_sum, tbar.volume, tbar.n_sell_aggs, tbar.n_sell_aggs,
-                                                        tbar.volatility);
-                        added_rows.addRow(new_row);
-                        _table.addRow(new_row);
-                    }
-                    _sec_bar_map.Clear();
-                    DataPacket dp_out = new DataPacket(_table, added_rows, new BlueTable(_table.schema));
-                    myFireEvent(dp_out);
-                }
-            }
-        }
-
-        private void _on_timesales(object obj) {
-            DataPacket dp = obj as DataPacket;
-            foreach (IBlueRow r in dp.addedRows) {
-                uint sec_key = r.get<UInt32>("sec_key");
-                TradeBar tbar = null;
-                if (!_sec_bar_map.TryGetValue(sec_key, out tbar)) {
-                    tbar = new TradeBar(r.get<string>("sec"));
-                    _sec_bar_map.Add(sec_key, tbar);
-                }
-                _dirty = true;
-                tbar.add_trade(r);
-            }
-
-
-        }
-
-        private void _on_bbo(object obj) {
-
-        }
-    }
-
-
-}

src/DataStreamSuite/SharedStreams/Business/MarketData/PriceStatsStream.cs

@@ -1,130 +1,127 @@
-namespace BCG {
-   using BCG.Utilities.General;
-
-   using bts.api;
-   using bts.api.domain;
-   using bts.api.synthetic;
-   using bts.md;
-   using bts.spark;
-   using bts.utils;
-
-   using mbts;
-   using mbts.co2;
-   using mbts.config;
-   using mbts.spark;
-   using mbts.ui;
-
-   using Spark.Core.Data;
-   using Spark.Core.Disposables;
-   using Spark.Core.Reactive;
-
-   using System;
-   using System.Collections.Concurrent;
-   using System.Collections.Generic;
-   using System.Collections.ObjectModel;
-   using System.Linq;
-   using System.Reactive.Linq;
-   using bts.api.core;
-   using Spark.Api.MarketData;
-
-   using SecurityKey = System.UInt32;
-   using System.Data;
-   using System.Reflection.Metadata;
-
-   [DataStreamUserRespresentation("price_stats")]
-   public class PriceStatsStream : ViewStream, IWorker {
-      private readonly TimeSpan _poll_period = TimeSpan.FromMilliseconds(500);
-      private DateTime _last_poll_time = DateTime.MinValue;
-      private ViewStreamDatabase _vsd;
-      private int _id = 0;
-      IReadOnlyDictionary<SecurityKey, PriceStats> _cached_state = null;
-      MarketDataClient _md_client;
-      private bool _dirty = false;
-      private IDisposable _sub = null;
-      private readonly IBlueTableSchema _pstats_scheam = BlueDataUtility.form_schema("bbo",
-                                                Tuple.Create("id", TypeCode.Int32),
-                                                Tuple.Create("sec_key", TypeCode.UInt32),
-                                                Tuple.Create("sec", TypeCode.String),
-                                                Tuple.Create("date", TypeCode.DateTime),
-                                                Tuple.Create("open", TypeCode.Double),
-                                                Tuple.Create("close", TypeCode.Double),
-                                                Tuple.Create("settlement", TypeCode.Double));
-      IDisposable? subscription;
-
-      // TODO: add cons for symbol, exchange etc 
-      public PriceStatsStream()
-      : this(ViewStreamDatabase.instance) {
-      }
-
-      public PriceStatsStream(ViewStreamDatabase vsd) {
-         bts.Log.Gen.info("PriceStatsStream.cons");
-         _vsd = vsd;
-         _md_client = new MarketDataClient(new MDSource(TimeSpan.FromSeconds(0.5)));
-      }
-
-      protected override void myOnFirstSubscriber() {
-         _vsd.addWorker(this);
-         subscription = _md_client.price_stats().Subscribe(_on_price_stats);
-      }
-
-      private void _log_updates(TableUpdate<PriceStats, uint> update) {
-         int i = 0;
-         // foreach (BookTuple bt in update.added) {
-         //    bts.Log.Gen.info($"PriceStatsStream added: idx: {i++} bid: {bt.book.bids[0].price} / {bt.book.bids[0].size} ask: {bt.book.asks[0].price} / {bt.book.asks[0].size}");
-         // }
-         // foreach (Change<BookTuple> c in update.changes) {
-         //    var bt = c.New;
-         //    bts.Log.Gen.info($"PriceStatsStream changes: idx: {i} bid: {bt.book.bids[0].price} / {bt.book.bids[0].size} ask: {bt.book.asks[0].price} / {bt.book.asks[0].size}");
-         // }
-      }
-
-      private void _on_price_stats(TableUpdate<PriceStats, uint> update) {
-         _log_updates(update);
-         // just cacching in here, will populate the rows in _poll_scan and fire the datapacket
-         bts.Log.Gen.info($"PriceStatsStream._on_price_stats state: {update.state.Count()} changes: {update.changes.Count()} added: {update.added.Count()}");
-         _cached_state = update.state;
-         _dirty = true;
-      }
-
-      protected override void myOnLastSubscriber() {
-         _vsd.removeWorker(this);
-         Disposable.dispose(ref _sub);
-      }
-
-      private void _scan() {
-         // this shouldn be made incremental using the TableUpdate
-         bts.Log.Gen.info("PriceStatsStream._scan_bbo");
-         if (_cached_state == null)
-            return;
-         BlueTable new_state = new BlueTable(_pstats_scheam);
-         foreach (var kv in _cached_state) {
-            SecurityKey skey = kv.Key;
-            int id = Convert.ToInt32(skey);
-            PriceStats pstats = kv.Value;
-            var today = DateTime.Now.Date;
-            IBlueRow r = new BlueRow(_pstats_scheam, id, skey, pstats.security.ToString(), today, pstats.open, pstats.close, pstats.settlement);
-            new_state.addRow(r);
-         }
-         DataPacket dp = null;
-         if (myLastValue == null) {
-            dp = new DataPacket(new_state, new_state, new BlueTable(_pstats_scheam));
-         } else {
-            DataPacket old_dp = myLastValue as DataPacket;
-            IBlueRowCollection added_rows, del_rows;
-            BlueDataUtility.compute_differences_fast(new_state, old_dp.newRows, out del_rows, out added_rows);
-            dp = new DataPacket(new_state, added_rows, del_rows);
-         }
-         if (dp.addedRows.Count == 0 && dp.deletedRows.Count == 0)
-            return;
-         myFireEvent(dp);
-      }
-
-      public void doWork() {
-         if (_dirty && DateTime.Now - _last_poll_time > _poll_period) {
-            _last_poll_time = DateTime.Now;
-            _dirty = false;
-            _scan();
-         }
-      }
-   }
-}
+namespace BCG {
+   using BCG.Utilities.General;
+
+   using bts.api;
+   using bts.api.domain;
+   using bts.api.synthetic;
+   using bts.md;
+   using bts.spark;
+   using bts.utils;
+
+   using mbts;
+   using mbts.co2;
+   using mbts.config;
+   using mbts.spark;
+   using mbts.ui;
+
+   using Spark.Core.Data;
+   using Spark.Core.Disposables;
+   using Spark.Core.Reactive;
+
+   using System;
+   using System.Collections.Concurrent;
+   using System.Collections.Generic;
+   using System.Collections.ObjectModel;
+   using System.Linq;
+   using System.Reactive.Linq;
+   using bts.api.core;
+   using Spark.Api.MarketData;
+
+   using SecurityKey = System.UInt32;
+   using System.Data;
+   using System.Reflection.Metadata;
+
+   [DataStreamUserRespresentation("price_stats")]
+   public class PriceStatsStream : ViewStream, IWorker {
+      private readonly TimeSpan _poll_period = TimeSpan.FromMilliseconds(500);
+      private DateTime _last_poll_time = DateTime.MinValue;
+      private ViewStreamDatabase _vsd;
+      private int _id = 0;
+      IReadOnlyDictionary<SecurityKey, PriceStats> _cached_state = null;
+      MarketDataClient _md_client;
+      private bool _dirty = false;
+      private readonly IBlueTableSchema _pstats_scheam = BlueDataUtility.form_schema("bbo",
+                                                Tuple.Create("id", TypeCode.Int32),
+                                                Tuple.Create("sec_key", TypeCode.Int32),
+                                                Tuple.Create("sec", TypeCode.String),
+                                                Tuple.Create("open", TypeCode.UInt32),
+                                                Tuple.Create("close", TypeCode.Double),
+                                                Tuple.Create("settlement", TypeCode.Single));
+      IDisposable? subscription;
+
+      // TODO: add cons for symbol, exchange etc 
+      public PriceStatsStream()
+      : this(ViewStreamDatabase.instance) {
+      }
+
+      public PriceStatsStream(ViewStreamDatabase vsd) {
+         bts.Log.Gen.info("PriceStatsStream.cons");
+         _vsd = vsd;
+         _md_client = new MarketDataClient(new MDSource(TimeSpan.FromSeconds(0.5)));
+      }
+
+      protected override void myOnFirstSubscriber() {
+         _vsd.addWorker(this);
+         subscription = _md_client.price_stats().Subscribe(_on_price_stats);
+      }
+
+      private void _log_updates(TableUpdate<PriceStats, uint> update) {
+         int i = 0;
+         // foreach (BookTuple bt in update.added) {
+         //    bts.Log.Gen.info($"PriceStatsStream added: idx: {i++} bid: {bt.book.bids[0].price} / {bt.book.bids[0].size} ask: {bt.book.asks[0].price} / {bt.book.asks[0].size}");
+         // }
+         // foreach (Change<BookTuple> c in update.changes) {
+         //    var bt = c.New;
+         //    bts.Log.Gen.info($"PriceStatsStream changes: idx: {i} bid: {bt.book.bids[0].price} / {bt.book.bids[0].size} ask: {bt.book.asks[0].price} / {bt.book.asks[0].size}");
+         // }
+      }
+
+      private void _on_price_stats(TableUpdate<PriceStats, uint> update) {
+         _log_updates(update);
+         // just cacching in here, will populate the rows in _poll_scan and fire the datapacket
+         bts.Log.Gen.info($"PriceStatsStream._on_price_stats state: {update.state.Count()} changes: {update.changes.Count()} added: {update.added.Count()}");
+         _cached_state = update.state;
+         _dirty = true;
+      }
+
+      protected override void myOnLastSubscriber() {
+         _vsd.removeWorker(this);
+         Disposable.dispose(ref subscription);
+      }
+
+      private void _scan() {
+         // this shouldn be made incremental using the TableUpdate
+         bts.Log.Gen.info("PriceStatsStream._scan_bbo");
+         if (_cached_state == null)
+            return;
+         BlueTable new_state = new BlueTable(_pstats_scheam);
+         foreach (var kv in _cached_state) {
+            SecurityKey skey = kv.Key;
+            int id = Convert.ToInt32(skey);
+            PriceStats pstats = kv.Value;
+            IBlueRow r = new BlueRow(_pstats_scheam, id, skey, pstats.security.ToString(), pstats.open, pstats.close, pstats.settlement);
+            new_state.addRow(r);
+         }
+         DataPacket dp = null;
+         if (myLastValue == null) {
+            dp = new DataPacket(new_state, new_state, new BlueTable(_pstats_scheam));
+         } else {
+            DataPacket old_dp = myLastValue as DataPacket;
+            IBlueRowCollection added_rows, del_rows;
+            BlueDataUtility.compute_differences_fast(new_state, old_dp.newRows, out del_rows, out added_rows);
+            dp = new DataPacket(new_state, added_rows, del_rows);
+         }
+         if (dp.addedRows.Count == 0 && dp.deletedRows.Count == 0)
+            return;
+         myFireEvent(dp);
+      }
+
+      public void doWork() {
+         if (_dirty && DateTime.Now - _last_poll_time > _poll_period) {
+            _last_poll_time = DateTime.Now;
+            _dirty = false;
+            _scan();
+         }
+      }
+   }
+}

src/DataStreamSuite/SharedStreams/SharedStreams.csproj

@@ -12,7 +12,6 @@
     <ProjectReference Include="..\DataStreamCore\DataStreamCore.csproj" />
     <ProjectReference Include="..\DataStream\DataStream.csproj" />
     <ProjectReference Include="..\PubSub\PubSub.csproj" />
-    <ProjectReference Include="..\..\client\spark_core\spark_core.csproj" />
   </ItemGroup>
 
   <ItemGroup>