WellKnownColumns.cs

Also change snake_case fn names to PascalCase

.devcontainer/devcontainer.json

@@ -22,6 +22,7 @@
       "extensions": [
         "eamodio.gitlens",
         "llvm-vs-code-extensions.vscode-clangd",
+        "ms-python.python",
         "ms-vscode.cmake-tools",
         "ms-vscode.cpptools",
         "xaver.clang-format"

.gitignore

@@ -55,6 +55,8 @@ trace_data/
 .cache/
 /src/Spark.Admin.WebApp/tsconfig.*.g.json
 /src/Spark.Admin.WebApp/src/bin/
+/src/client/**/*appsettings.Local.json
+/src/DataStreamSuite/**/*appsettings.Local.json
 **/CMakeDeps.cmake
 **/CMakeDeps_win.cmake
 .devcontainer/

.vscode/launch.json

@@ -27,7 +27,7 @@
           "--sparkd",
           "http://127.0.0.1:8000/config"
         ],
-        "cwd": "build/Debug/bin/net7.0",
+        "cwd": "build/Debug/bin/net8.0",
       },
       "stopAtEntry": false,
       "console": "integratedTerminal"
@@ -44,7 +44,7 @@
           "--sparkd",
           "http://127.0.0.1:8000/config"
         ],
-        "cwd": "build/Debug/bin/net7.0",
+        "cwd": "build/Debug/bin/net8.0",
       },
       "stopAtEntry": false,
       "console": "integratedTerminal"
@@ -165,12 +165,25 @@
       "program": "",
       "windows": {
         "program": "DataStreamQueryAnalyzer.exe",
-        "args": [
-          "--config",
-          "${workspaceFolder}/src/DataStreamSuite/DataStreamQueryAnalyzer/Configurations/DataStreamQueryAnalyzer.json"
-        ],
         "cwd": "build/Debug/bin/net8.0-windows",
         "env": {
+          "DOTNET_ENVIRONMENT": "${input:dataStreamQueryAnalyzerEnvironment}",
+          "PYTHON_EXE": "${workspaceFolder}/.venv/Scripts/python.exe"
+        },
+      },
+      "stopAtEntry": false,
+      "console": "integratedTerminal",
+    },
+    {
+      "name": "DataStreamQueryAnalyzer (Skip Build)",
+      "type": "coreclr",
+      "request": "launch",
+      "program": "",
+      "windows": {
+        "program": "DataStreamQueryAnalyzer.exe",
+        "cwd": "build/Debug/bin/net8.0-windows",
+        "env": {
+          "DOTNET_ENVIRONMENT": "${input:dataStreamQueryAnalyzerEnvironment}",
           "PYTHON_EXE": "${workspaceFolder}/.venv/Scripts/python.exe"
         },
       },
@@ -207,7 +220,7 @@
       "windows": {
         "program": "PubSubServer.exe",
       },
-      "cwd": "build/Debug/bin/net7.0",
+      "cwd": "build/Debug/bin/net8.0",
       "env": {
         "DOTNET_ENVIRONMENT": "${input:pubSubServerEnvironment}",
       },
@@ -223,6 +236,9 @@
         "preLaunchTask": "Build Spark (Debug)",
         "program": "Spark.exe",
         "cwd": "build/Debug/bin/net8.0-windows",
+        "env": {
+          "DOTNET_ENVIRONMENT": "${input:sparkEnvironment}",
+        }
       },
       "stopAtEntry": false,
       "console": "integratedTerminal"
@@ -235,6 +251,9 @@
       "windows": {
         "program": "Spark.exe",
         "cwd": "build/Debug/bin/net8.0-windows",
+        "env": {
+          "DOTNET_ENVIRONMENT": "${input:sparkEnvironment}",
+        }
       },
       "stopAtEntry": false,
       "console": "integratedTerminal"
@@ -384,6 +403,26 @@
         "Staging",
       ]
     },
+    {
+      "id": "dataStreamQueryAnalyzerEnvironment",
+      "type": "pickString",
+      "description": "DataStreamQueryAnalyzer Configuration",
+      "options": [
+        "Development",
+        "Local",
+      ],
+      "default": "Development",
+    },
+    {
+      "id": "sparkEnvironment",
+      "type": "pickString",
+      "description": "Spark Configuration",
+      "options": [
+        "Development",
+        "Local",
+      ],
+      "default": "Development",
+    },
     {
       "id": "sparkConfigServer",
       "type": "promptString",

.vscode/tasks.json

@@ -485,12 +485,12 @@
       "command": "",
       "windows": {
         "command": "DataStreamQueryAnalyzer.exe",
-        "args": [
-          "--config",
-          "${workspaceFolder}/src/DataStreamSuite/DataStreamQueryAnalyzer/Configurations/DataStreamQueryAnalyzer.json",
-        ],
         "options": {
-          "cwd": "${workspaceFolder}/build/${input:buildConfiguration}/bin/net8.0-windows"
+          "cwd": "${workspaceFolder}/build/${input:buildConfiguration}/bin/net8.0-windows",
+          "env": {
+            "DOTNET_ENVIRONMENT": "${input:dataStreamQueryAnalyzerEnvironment}",
+            "PYTHON_EXE": "${workspaceFolder}/.venv/Scripts/python.exe"
+          },
         },
       },
       "icon": {
@@ -549,12 +549,11 @@
       "windows": {
         "command": "DataStreamPublisher.exe",
       },
-      "args": [
-        "--config",
-        "${workspaceFolder}/src/DataStreamSuite/DataStreamPublisher/Configurations/${input:dataStreamPublisherConfig}.json",
-      ],
       "options": {
-        "cwd": "${workspaceFolder}/build/${input:buildConfiguration}/bin/net8.0"
+        "cwd": "${workspaceFolder}/build/${input:buildConfiguration}/bin/net8.0",
+        "env": {
+          "DOTNET_ENVIRONMENT": "${input:dataStreamPublisherEnvironment}",
+        },
       },
       "icon": {
         "id": "run",
@@ -572,12 +571,11 @@
       "windows": {
         "command": "DataStreamPublisher.exe",
       },
-      "args": [
-        "--config",
-        "${workspaceFolder}/src/DataStreamSuite/DataStreamPublisher/Configurations/${input:dataStreamPublisherConfig}.json",
-      ],
       "options": {
-        "cwd": "${workspaceFolder}/build/Debug/bin/net8.0"
+        "cwd": "${workspaceFolder}/build/Debug/bin/net8.0",
+        "env": {
+          "DOTNET_ENVIRONMENT": "${input:dataStreamPublisherEnvironment}",
+        },
       },
       "icon": {
         "id": "run",
@@ -595,12 +593,11 @@
       "windows": {
         "command": "DataStreamPublisher.exe",
       },
-      "args": [
-        "--config",
-        "${workspaceFolder}/src/DataStreamSuite/DataStreamPublisher/Configurations/${input:dataStreamPublisherConfig}.json",
-      ],
       "options": {
-        "cwd": "${workspaceFolder}/build/Release/bin/net8.0"
+        "cwd": "${workspaceFolder}/build/Release/bin/net8.0",
+        "env": {
+          "DOTNET_ENVIRONMENT": "${input:dataStreamPublisherEnvironment}",
+        },
       },
       "icon": {
         "id": "run",
@@ -701,12 +698,11 @@
       "windows": {
         "command": "PubSubServer.exe",
       },
-      "args": [
-        "--config",
-        "${workspaceFolder}/src/DataStreamSuite/PubSubServer/Configurations/${input:pubSubServerConfig}.json",
-      ],
       "options": {
-        "cwd": "${workspaceFolder}/build/${input:buildConfiguration}/bin/net8.0"
+        "cwd": "${workspaceFolder}/build/${input:buildConfiguration}/bin/net8.0",
+        "env": {
+          "DOTNET_ENVIRONMENT": "${input:pubSubServerEnvironment}",
+        },
       },
       "icon": {
         "id": "run",
@@ -724,12 +720,11 @@
       "windows": {
         "command": "PubSubServer.exe",
       },
-      "args": [
-        "--config",
-        "${workspaceFolder}/src/DataStreamSuite/PubSubServer/Configurations/${input:pubSubServerConfig}.json",
-      ],
       "options": {
-        "cwd": "${workspaceFolder}/build/Debug/bin/net8.0"
+        "cwd": "${workspaceFolder}/build/Debug/bin/net8.0",
+        "env": {
+          "DOTNET_ENVIRONMENT": "${input:pubSubServerEnvironment}",
+        },
       },
       "icon": {
         "id": "run",
@@ -746,12 +741,11 @@
       "windows": {
         "command": "PubSubServer.exe",
       },
-      "args": [
-        "--config",
-        "${workspaceFolder}/src/DataStreamSuite/PubSubServer/Configurations/${input:pubSubServerConfig}.json",
-      ],
       "options": {
-        "cwd": "${workspaceFolder}/build/Debug/bin/net8.0"
+        "cwd": "${workspaceFolder}/build/Debug/bin/net8.0",
+        "env": {
+          "DOTNET_ENVIRONMENT": "${input:pubSubServerEnvironment}",
+        },
       },
       "icon": {
         "id": "run",
@@ -769,12 +763,11 @@
       "windows": {
         "command": "PubSubServer.exe",
       },
-      "args": [
-        "--config",
-        "${workspaceFolder}/src/DataStreamSuite/PubSubServer/Configurations/${input:pubSubServerConfig}.json",
-      ],
       "options": {
-        "cwd": "${workspaceFolder}/build/Release/bin/net8.0"
+        "cwd": "${workspaceFolder}/build/Release/bin/net8.0",
+        "env": {
+          "DOTNET_ENVIRONMENT": "${input:pubSubServerEnvironment}",
+        },
       },
       "icon": {
         "id": "run",
@@ -791,12 +784,11 @@
       "windows": {
         "command": "PubSubServer.exe",
       },
-      "args": [
-        "--config",
-        "${workspaceFolder}/src/DataStreamSuite/PubSubServer/Configurations/${input:pubSubServerConfig}.json",
-      ],
       "options": {
-        "cwd": "${workspaceFolder}/build/Release/bin/net8.0"
+        "cwd": "${workspaceFolder}/build/Release/bin/net8.0",
+        "env": {
+          "DOTNET_ENVIRONMENT": "${input:pubSubServerEnvironment}",
+        },
       },
       "icon": {
         "id": "run",
@@ -1095,26 +1087,40 @@
       "default": "sparkd-debug-clang",
     },
     {
-      "id": "dataStreamPublisherConfig",
+      "id": "dataStreamPublisherEnvironment",
       "type": "pickString",
       "description": "Publisher Configuration",
       "options": [
-        "MarketDataPublisher",
-        "SimpleEchoClient",
-        "SimpleTestDataPublisher",
-        "TestDataPublisher",
+        "Development",
+        "MarketData",
+        "Production",
+        "ProductionAws",
+        "Staging",
       ],
       "default": "MarketDataPublisher",
     },
     {
-      "id": "pubSubServerConfig",
+      "id": "dataStreamQueryAnalyzerEnvironment",
+      "type": "pickString",
+      "description": "DataStreamQueryAnalyzer Configuration",
+      "options": [
+        "Development",
+        "Local",
+      ],
+      "default": "Development",
+    },
+    {
+      "id": "pubSubServerEnvironment",
       "type": "pickString",
       "description": "PubSub Configuration",
       "options": [
-        "PubSubServer",
-        "SimpleEchoClient",
+        "Development",
+        "EchoServer",
+        "Production",
+        "ProductionAws",
+        "Staging",
       ],
-      "default": "PubSubServer",
+      "default": "Development",
     },
     {
       "id": "sparkConfigQuery",

compose.yaml

@@ -134,6 +134,16 @@ services:
     environment:
       - DOTNET_ENVIRONMENT=MarketData
 
+  api-server:
+    depends_on:
+      - sparkd
+    ports:
+      - 8008:8008
+      - 9443:9443
+    build:
+      dockerfile: src/client/spark_api_server/Dockerfile
+    command: [ "dockerize", "-wait", "tcp://sparkd:8000", "-timeout", "10s", "./spark_api_server", "run" ]
+
   webapp:
     ports:
       - 8080:8080
@@ -148,18 +158,6 @@ services:
     environment:
       - ASPNETCORE_ENVIRONMENT=Staging
 
-  api-server:
-    depends_on:
-      - sparkd
-    ports:
-      - 8008:8008
-      - 9443:9443
-    build:
-      dockerfile: src/client/spark_api_server/Dockerfile
-    command: [ "dockerize", "-wait", "tcp://sparkd:8000", "-timeout", "10s", "./spark_api_server", "run" ]
-    environment:
-      - ASPNETCORE_ENVIRONMENT=Staging
-
 volumes:
   sparkd_data:
     name: "sparkd_${SPARKD__CONFIG_PORT:-8000}"

global.json

@@ -1,6 +0,0 @@
-{
-  "sdk": {
-    "version": "6.0.200",
-    "rollForward": "latestMajor"
-  }
-}

spark_client.sln

@@ -1,143 +1,164 @@
-Microsoft Visual Studio Solution File, Format Version 12.00
-# Visual Studio Version 17
-VisualStudioVersion = 17.3.32922.545
-MinimumVisualStudioVersion = 15.0.26124.0
-Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "spark_api", "src\client\spark_api\spark_api.csproj", "{DC9A8975-8211-4FC9-A9BA-C8B9D26AD481}"
-EndProject
-Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Spark", "src\client\spark\Spark.csproj", "{9A4D4D07-1AB6-4B2B-B111-A4F34D6C267E}"
-EndProject
-Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Spark.Console", "src\client\Spark.Console\Spark.Console.csproj", "{387566C6-C6F6-412C-A30C-9855BBCB4816}"
-EndProject
-Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "spark_core", "src\client\spark_core\spark_core.csproj", "{6E449EB6-20AC-4FA6-B8D9-10F4BB7E1FEE}"
-EndProject
-Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "spark_sdk", "src\client\spark_sdk\spark_sdk.csproj", "{6963C314-83C4-4667-B0D0-C2F7AE208C29}"
-EndProject
-Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "spark_test", "src\client\spark_test\spark_test.csproj", "{ECD78CC5-138F-4809-A764-2358C2AC2E82}"
-EndProject
-Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "spark_websocket", "src\client\spark_websocket\spark_websocket.csproj", "{3201359D-C071-4089-9EDE-B04D759F1508}"
-EndProject
-Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "spark_api_server", "src\client\spark_api_server\spark_api_server.csproj", "{2B8C5D01-FF78-4BFA-B332-5E430586DAF9}"
-EndProject
-Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "DataStreamCore", "src\DataStreamSuite\DataStreamCore\DataStreamCore.csproj", "{4E4F491F-C688-43C4-88DA-AC92BC5FA1B8}"
-EndProject
-Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "DataStream", "src\DataStreamSuite\DataStream\DataStream.csproj", "{2AC54ADC-3246-427E-9CA9-FEA2DBCB9432}"
-EndProject
-Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "SharedStreams", "src\DataStreamSuite\SharedStreams\SharedStreams.csproj", "{4E96E5CC-7791-40B3-A8A1-C34FBB736B3B}"
-EndProject
-Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "PubSub", "src\DataStreamSuite\PubSub\PubSub.csproj", "{51C40BA9-F226-4A35-939D-1ED6F5FF7299}"
-EndProject
-Global
-	GlobalSection(SolutionConfigurationPlatforms) = preSolution
-		Debug|x64 = Debug|x64
-		MinSizeRel|x64 = MinSizeRel|x64
-		Release|x64 = Release|x64
-		RelWithDebInfo|x64 = RelWithDebInfo|x64
-	EndGlobalSection
-	GlobalSection(ProjectConfigurationPlatforms) = postSolution
-		{DC9A8975-8211-4FC9-A9BA-C8B9D26AD481}.Debug|x64.ActiveCfg = Debug|x64
-		{DC9A8975-8211-4FC9-A9BA-C8B9D26AD481}.Debug|x64.Build.0 = Debug|x64
-		{DC9A8975-8211-4FC9-A9BA-C8B9D26AD481}.MinSizeRel|x64.ActiveCfg = Release|x64
-		{DC9A8975-8211-4FC9-A9BA-C8B9D26AD481}.MinSizeRel|x64.Build.0 = Release|x64
-		{DC9A8975-8211-4FC9-A9BA-C8B9D26AD481}.Release|x64.ActiveCfg = Release|x64
-		{DC9A8975-8211-4FC9-A9BA-C8B9D26AD481}.Release|x64.Build.0 = Release|x64
-		{DC9A8975-8211-4FC9-A9BA-C8B9D26AD481}.RelWithDebInfo|x64.ActiveCfg = Release|x64
-		{DC9A8975-8211-4FC9-A9BA-C8B9D26AD481}.RelWithDebInfo|x64.Build.0 = Release|x64
-		{9A4D4D07-1AB6-4B2B-B111-A4F34D6C267E}.Debug|x64.ActiveCfg = Debug|x64
-		{9A4D4D07-1AB6-4B2B-B111-A4F34D6C267E}.Debug|x64.Build.0 = Debug|x64
-		{9A4D4D07-1AB6-4B2B-B111-A4F34D6C267E}.MinSizeRel|x64.ActiveCfg = Release|x64
-		{9A4D4D07-1AB6-4B2B-B111-A4F34D6C267E}.MinSizeRel|x64.Build.0 = Release|x64
-		{9A4D4D07-1AB6-4B2B-B111-A4F34D6C267E}.Release|x64.ActiveCfg = Release|x64
-		{9A4D4D07-1AB6-4B2B-B111-A4F34D6C267E}.Release|x64.Build.0 = Release|x64
-		{9A4D4D07-1AB6-4B2B-B111-A4F34D6C267E}.RelWithDebInfo|x64.ActiveCfg = Release|x64
-		{9A4D4D07-1AB6-4B2B-B111-A4F34D6C267E}.RelWithDebInfo|x64.Build.0 = Release|x64
-		{387566C6-C6F6-412C-A30C-9855BBCB4816}.Debug|x64.ActiveCfg = Debug|x64
-		{387566C6-C6F6-412C-A30C-9855BBCB4816}.Debug|x64.Build.0 = Debug|x64
-		{387566C6-C6F6-412C-A30C-9855BBCB4816}.MinSizeRel|x64.ActiveCfg = Debug|x64
-		{387566C6-C6F6-412C-A30C-9855BBCB4816}.MinSizeRel|x64.Build.0 = Debug|x64
-		{387566C6-C6F6-412C-A30C-9855BBCB4816}.Release|x64.ActiveCfg = Release|x64
-		{387566C6-C6F6-412C-A30C-9855BBCB4816}.Release|x64.Build.0 = Release|x64
-		{387566C6-C6F6-412C-A30C-9855BBCB4816}.RelWithDebInfo|x64.ActiveCfg = Release|x64
-		{387566C6-C6F6-412C-A30C-9855BBCB4816}.RelWithDebInfo|x64.Build.0 = Release|x64
-		{6E449EB6-20AC-4FA6-B8D9-10F4BB7E1FEE}.Debug|x64.ActiveCfg = Debug|x64
-		{6E449EB6-20AC-4FA6-B8D9-10F4BB7E1FEE}.Debug|x64.Build.0 = Debug|x64
-		{6E449EB6-20AC-4FA6-B8D9-10F4BB7E1FEE}.MinSizeRel|x64.ActiveCfg = Release|x64
-		{6E449EB6-20AC-4FA6-B8D9-10F4BB7E1FEE}.MinSizeRel|x64.Build.0 = Release|x64
-		{6E449EB6-20AC-4FA6-B8D9-10F4BB7E1FEE}.Release|x64.ActiveCfg = Release|x64
-		{6E449EB6-20AC-4FA6-B8D9-10F4BB7E1FEE}.Release|x64.Build.0 = Release|x64
-		{6E449EB6-20AC-4FA6-B8D9-10F4BB7E1FEE}.RelWithDebInfo|x64.ActiveCfg = Release|x64
-		{6E449EB6-20AC-4FA6-B8D9-10F4BB7E1FEE}.RelWithDebInfo|x64.Build.0 = Release|x64
-		{6963C314-83C4-4667-B0D0-C2F7AE208C29}.Debug|x64.ActiveCfg = Debug|x64
-		{6963C314-83C4-4667-B0D0-C2F7AE208C29}.Debug|x64.Build.0 = Debug|x64
-		{6963C314-83C4-4667-B0D0-C2F7AE208C29}.MinSizeRel|x64.ActiveCfg = Release|x64
-		{6963C314-83C4-4667-B0D0-C2F7AE208C29}.MinSizeRel|x64.Build.0 = Release|x64
-		{6963C314-83C4-4667-B0D0-C2F7AE208C29}.Release|x64.ActiveCfg = Release|x64
-		{6963C314-83C4-4667-B0D0-C2F7AE208C29}.Release|x64.Build.0 = Release|x64
-		{6963C314-83C4-4667-B0D0-C2F7AE208C29}.RelWithDebInfo|x64.ActiveCfg = Release|x64
-		{6963C314-83C4-4667-B0D0-C2F7AE208C29}.RelWithDebInfo|x64.Build.0 = Release|x64
-		{ECD78CC5-138F-4809-A764-2358C2AC2E82}.Debug|x64.ActiveCfg = Debug|x64
-		{ECD78CC5-138F-4809-A764-2358C2AC2E82}.Debug|x64.Build.0 = Debug|x64
-		{ECD78CC5-138F-4809-A764-2358C2AC2E82}.MinSizeRel|x64.ActiveCfg = Debug|x64
-		{ECD78CC5-138F-4809-A764-2358C2AC2E82}.MinSizeRel|x64.Build.0 = Debug|x64
-		{ECD78CC5-138F-4809-A764-2358C2AC2E82}.Release|x64.ActiveCfg = Release|x64
-		{ECD78CC5-138F-4809-A764-2358C2AC2E82}.Release|x64.Build.0 = Release|x64
-		{ECD78CC5-138F-4809-A764-2358C2AC2E82}.RelWithDebInfo|x64.ActiveCfg = Release|x64
-		{ECD78CC5-138F-4809-A764-2358C2AC2E82}.RelWithDebInfo|x64.Build.0 = Release|x64
-		{3201359D-C071-4089-9EDE-B04D759F1508}.Debug|x64.ActiveCfg = Debug|x64
-		{3201359D-C071-4089-9EDE-B04D759F1508}.Debug|x64.Build.0 = Debug|x64
-		{3201359D-C071-4089-9EDE-B04D759F1508}.MinSizeRel|x64.ActiveCfg = Debug|x64
-		{3201359D-C071-4089-9EDE-B04D759F1508}.MinSizeRel|x64.Build.0 = Debug|x64
-		{3201359D-C071-4089-9EDE-B04D759F1508}.Release|x64.ActiveCfg = Release|x64
-		{3201359D-C071-4089-9EDE-B04D759F1508}.Release|x64.Build.0 = Release|x64
-		{3201359D-C071-4089-9EDE-B04D759F1508}.RelWithDebInfo|x64.ActiveCfg = Release|x64
-		{3201359D-C071-4089-9EDE-B04D759F1508}.RelWithDebInfo|x64.Build.0 = Release|x64
-		{2B8C5D01-FF78-4BFA-B332-5E430586DAF9}.Debug|x64.ActiveCfg = Debug|x64
-		{2B8C5D01-FF78-4BFA-B332-5E430586DAF9}.Debug|x64.Build.0 = Debug|x64
-		{2B8C5D01-FF78-4BFA-B332-5E430586DAF9}.MinSizeRel|x64.ActiveCfg = Debug|x64
-		{2B8C5D01-FF78-4BFA-B332-5E430586DAF9}.MinSizeRel|x64.Build.0 = Debug|x64
-		{2B8C5D01-FF78-4BFA-B332-5E430586DAF9}.Release|x64.ActiveCfg = Release|x64
-		{2B8C5D01-FF78-4BFA-B332-5E430586DAF9}.Release|x64.Build.0 = Release|x64
-		{2B8C5D01-FF78-4BFA-B332-5E430586DAF9}.RelWithDebInfo|x64.ActiveCfg = Release|x64
-		{2B8C5D01-FF78-4BFA-B332-5E430586DAF9}.RelWithDebInfo|x64.Build.0 = Release|x64
-		{4E4F491F-C688-43C4-88DA-AC92BC5FA1B8}.Debug|x64.ActiveCfg = Debug|Any CPU
-		{4E4F491F-C688-43C4-88DA-AC92BC5FA1B8}.Debug|x64.Build.0 = Debug|Any CPU
-		{4E4F491F-C688-43C4-88DA-AC92BC5FA1B8}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
-		{4E4F491F-C688-43C4-88DA-AC92BC5FA1B8}.MinSizeRel|x64.Build.0 = Debug|Any CPU
-		{4E4F491F-C688-43C4-88DA-AC92BC5FA1B8}.Release|x64.ActiveCfg = Release|Any CPU
-		{4E4F491F-C688-43C4-88DA-AC92BC5FA1B8}.Release|x64.Build.0 = Release|Any CPU
-		{4E4F491F-C688-43C4-88DA-AC92BC5FA1B8}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
-		{4E4F491F-C688-43C4-88DA-AC92BC5FA1B8}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
-		{2AC54ADC-3246-427E-9CA9-FEA2DBCB9432}.Debug|x64.ActiveCfg = Debug|Any CPU
-		{2AC54ADC-3246-427E-9CA9-FEA2DBCB9432}.Debug|x64.Build.0 = Debug|Any CPU
-		{2AC54ADC-3246-427E-9CA9-FEA2DBCB9432}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
-		{2AC54ADC-3246-427E-9CA9-FEA2DBCB9432}.MinSizeRel|x64.Build.0 = Debug|Any CPU
-		{2AC54ADC-3246-427E-9CA9-FEA2DBCB9432}.Release|x64.ActiveCfg = Release|Any CPU
-		{2AC54ADC-3246-427E-9CA9-FEA2DBCB9432}.Release|x64.Build.0 = Release|Any CPU
-		{2AC54ADC-3246-427E-9CA9-FEA2DBCB9432}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
-		{2AC54ADC-3246-427E-9CA9-FEA2DBCB9432}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
-		{4E96E5CC-7791-40B3-A8A1-C34FBB736B3B}.Debug|x64.ActiveCfg = Debug|Any CPU
-		{4E96E5CC-7791-40B3-A8A1-C34FBB736B3B}.Debug|x64.Build.0 = Debug|Any CPU
-		{4E96E5CC-7791-40B3-A8A1-C34FBB736B3B}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
-		{4E96E5CC-7791-40B3-A8A1-C34FBB736B3B}.MinSizeRel|x64.Build.0 = Debug|Any CPU
-		{4E96E5CC-7791-40B3-A8A1-C34FBB736B3B}.Release|x64.ActiveCfg = Release|Any CPU
-		{4E96E5CC-7791-40B3-A8A1-C34FBB736B3B}.Release|x64.Build.0 = Release|Any CPU
-		{4E96E5CC-7791-40B3-A8A1-C34FBB736B3B}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
-		{4E96E5CC-7791-40B3-A8A1-C34FBB736B3B}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
-		{51C40BA9-F226-4A35-939D-1ED6F5FF7299}.Debug|x64.ActiveCfg = Debug|Any CPU
-		{51C40BA9-F226-4A35-939D-1ED6F5FF7299}.Debug|x64.Build.0 = Debug|Any CPU
-		{51C40BA9-F226-4A35-939D-1ED6F5FF7299}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
-		{51C40BA9-F226-4A35-939D-1ED6F5FF7299}.MinSizeRel|x64.Build.0 = Debug|Any CPU
-		{51C40BA9-F226-4A35-939D-1ED6F5FF7299}.Release|x64.ActiveCfg = Release|Any CPU
-		{51C40BA9-F226-4A35-939D-1ED6F5FF7299}.Release|x64.Build.0 = Release|Any CPU
-		{51C40BA9-F226-4A35-939D-1ED6F5FF7299}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
-		{51C40BA9-F226-4A35-939D-1ED6F5FF7299}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
-	EndGlobalSection
-	GlobalSection(SolutionProperties) = preSolution
-		HideSolutionNode = FALSE
-	EndGlobalSection
-	GlobalSection(ExtensibilityGlobals) = postSolution
-		SolutionGuid = {422D1620-0166-4C26-9A81-CA8A2EB1AB58}
-	EndGlobalSection
-	GlobalSection(Performance) = preSolution
-		HasPerformanceSessions = true
-	EndGlobalSection
-EndGlobal
+Microsoft Visual Studio Solution File, Format Version 12.00
+# Visual Studio Version 17
+VisualStudioVersion = 17.3.32922.545
+MinimumVisualStudioVersion = 15.0.26124.0
+Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "spark_api", "src\client\spark_api\spark_api.csproj", "{DC9A8975-8211-4FC9-A9BA-C8B9D26AD481}"
+EndProject
+Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Spark", "src\client\spark\Spark.csproj", "{9A4D4D07-1AB6-4B2B-B111-A4F34D6C267E}"
+EndProject
+Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Spark.Console", "src\client\Spark.Console\Spark.Console.csproj", "{387566C6-C6F6-412C-A30C-9855BBCB4816}"
+EndProject
+Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "spark_core", "src\client\spark_core\spark_core.csproj", "{6E449EB6-20AC-4FA6-B8D9-10F4BB7E1FEE}"
+EndProject
+Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "spark_sdk", "src\client\spark_sdk\spark_sdk.csproj", "{6963C314-83C4-4667-B0D0-C2F7AE208C29}"
+EndProject
+Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "spark_test", "src\client\spark_test\spark_test.csproj", "{ECD78CC5-138F-4809-A764-2358C2AC2E82}"
+EndProject
+Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "spark_websocket", "src\client\spark_websocket\spark_websocket.csproj", "{3201359D-C071-4089-9EDE-B04D759F1508}"
+EndProject
+Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "spark_api_server", "src\client\spark_api_server\spark_api_server.csproj", "{2B8C5D01-FF78-4BFA-B332-5E430586DAF9}"
+EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "DataStreamCore", "src\DataStreamSuite\DataStreamCore\DataStreamCore.csproj", "{4E4F491F-C688-43C4-88DA-AC92BC5FA1B8}"
+EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "DataStream", "src\DataStreamSuite\DataStream\DataStream.csproj", "{2AC54ADC-3246-427E-9CA9-FEA2DBCB9432}"
+EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "SharedStreams", "src\DataStreamSuite\SharedStreams\SharedStreams.csproj", "{4E96E5CC-7791-40B3-A8A1-C34FBB736B3B}"
+EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "PubSub", "src\DataStreamSuite\PubSub\PubSub.csproj", "{51C40BA9-F226-4A35-939D-1ED6F5FF7299}"
+EndProject
+Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "src", "src", "{4564E677-343A-46CB-AF02-AFC4E38BF94B}"
+EndProject
+Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "client", "client", "{56A8A70A-DA03-4CED-A144-EC10F1379FD8}"
+EndProject
+Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "DataStreamSuite", "DataStreamSuite", "{915EF238-CA0D-4E73-AF50-88CB3F29C7AD}"
+EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Bts.Extensions.Hosting", "src\DataStreamSuite\Bts.Extensions.Hosting\Bts.Extensions.Hosting.csproj", "{4E0E51AA-7B59-4808-B7CC-9EAEDF397460}"
+EndProject
+Global
+	GlobalSection(SolutionConfigurationPlatforms) = preSolution
+		Debug|x64 = Debug|x64
+		MinSizeRel|x64 = MinSizeRel|x64
+		Release|x64 = Release|x64
+		RelWithDebInfo|x64 = RelWithDebInfo|x64
+	EndGlobalSection
+	GlobalSection(ProjectConfigurationPlatforms) = postSolution
+		{DC9A8975-8211-4FC9-A9BA-C8B9D26AD481}.Debug|x64.ActiveCfg = Debug|x64
+		{DC9A8975-8211-4FC9-A9BA-C8B9D26AD481}.Debug|x64.Build.0 = Debug|x64
+		{DC9A8975-8211-4FC9-A9BA-C8B9D26AD481}.MinSizeRel|x64.ActiveCfg = Release|x64
+		{DC9A8975-8211-4FC9-A9BA-C8B9D26AD481}.MinSizeRel|x64.Build.0 = Release|x64
+		{DC9A8975-8211-4FC9-A9BA-C8B9D26AD481}.Release|x64.ActiveCfg = Release|x64
+		{DC9A8975-8211-4FC9-A9BA-C8B9D26AD481}.Release|x64.Build.0 = Release|x64
+		{DC9A8975-8211-4FC9-A9BA-C8B9D26AD481}.RelWithDebInfo|x64.ActiveCfg = Release|x64
+		{DC9A8975-8211-4FC9-A9BA-C8B9D26AD481}.RelWithDebInfo|x64.Build.0 = Release|x64
+		{9A4D4D07-1AB6-4B2B-B111-A4F34D6C267E}.Debug|x64.ActiveCfg = Debug|x64
+		{9A4D4D07-1AB6-4B2B-B111-A4F34D6C267E}.Debug|x64.Build.0 = Debug|x64
+		{9A4D4D07-1AB6-4B2B-B111-A4F34D6C267E}.MinSizeRel|x64.ActiveCfg = Release|x64
+		{9A4D4D07-1AB6-4B2B-B111-A4F34D6C267E}.MinSizeRel|x64.Build.0 = Release|x64
+		{9A4D4D07-1AB6-4B2B-B111-A4F34D6C267E}.Release|x64.ActiveCfg = Release|x64
+		{9A4D4D07-1AB6-4B2B-B111-A4F34D6C267E}.Release|x64.Build.0 = Release|x64
+		{9A4D4D07-1AB6-4B2B-B111-A4F34D6C267E}.RelWithDebInfo|x64.ActiveCfg = Release|x64
+		{9A4D4D07-1AB6-4B2B-B111-A4F34D6C267E}.RelWithDebInfo|x64.Build.0 = Release|x64
+		{387566C6-C6F6-412C-A30C-9855BBCB4816}.Debug|x64.ActiveCfg = Debug|x64
+		{387566C6-C6F6-412C-A30C-9855BBCB4816}.Debug|x64.Build.0 = Debug|x64
+		{387566C6-C6F6-412C-A30C-9855BBCB4816}.MinSizeRel|x64.ActiveCfg = Debug|x64
+		{387566C6-C6F6-412C-A30C-9855BBCB4816}.MinSizeRel|x64.Build.0 = Debug|x64
+		{387566C6-C6F6-412C-A30C-9855BBCB4816}.Release|x64.ActiveCfg = Release|x64
+		{387566C6-C6F6-412C-A30C-9855BBCB4816}.Release|x64.Build.0 = Release|x64
+		{387566C6-C6F6-412C-A30C-9855BBCB4816}.RelWithDebInfo|x64.ActiveCfg = Release|x64
+		{387566C6-C6F6-412C-A30C-9855BBCB4816}.RelWithDebInfo|x64.Build.0 = Release|x64
+		{6E449EB6-20AC-4FA6-B8D9-10F4BB7E1FEE}.Debug|x64.ActiveCfg = Debug|x64
+		{6E449EB6-20AC-4FA6-B8D9-10F4BB7E1FEE}.Debug|x64.Build.0 = Debug|x64
+		{6E449EB6-20AC-4FA6-B8D9-10F4BB7E1FEE}.MinSizeRel|x64.ActiveCfg = Release|x64
+		{6E449EB6-20AC-4FA6-B8D9-10F4BB7E1FEE}.MinSizeRel|x64.Build.0 = Release|x64
+		{6E449EB6-20AC-4FA6-B8D9-10F4BB7E1FEE}.Release|x64.ActiveCfg = Release|x64
+		{6E449EB6-20AC-4FA6-B8D9-10F4BB7E1FEE}.Release|x64.Build.0 = Release|x64
+		{6E449EB6-20AC-4FA6-B8D9-10F4BB7E1FEE}.RelWithDebInfo|x64.ActiveCfg = Release|x64
+		{6E449EB6-20AC-4FA6-B8D9-10F4BB7E1FEE}.RelWithDebInfo|x64.Build.0 = Release|x64
+		{6963C314-83C4-4667-B0D0-C2F7AE208C29}.Debug|x64.ActiveCfg = Debug|x64
+		{6963C314-83C4-4667-B0D0-C2F7AE208C29}.Debug|x64.Build.0 = Debug|x64
+		{6963C314-83C4-4667-B0D0-C2F7AE208C29}.MinSizeRel|x64.ActiveCfg = Release|x64
+		{6963C314-83C4-4667-B0D0-C2F7AE208C29}.MinSizeRel|x64.Build.0 = Release|x64
+		{6963C314-83C4-4667-B0D0-C2F7AE208C29}.Release|x64.ActiveCfg = Release|x64
+		{6963C314-83C4-4667-B0D0-C2F7AE208C29}.Release|x64.Build.0 = Release|x64
+		{6963C314-83C4-4667-B0D0-C2F7AE208C29}.RelWithDebInfo|x64.ActiveCfg = Release|x64
+		{6963C314-83C4-4667-B0D0-C2F7AE208C29}.RelWithDebInfo|x64.Build.0 = Release|x64
+		{ECD78CC5-138F-4809-A764-2358C2AC2E82}.Debug|x64.ActiveCfg = Debug|x64
+		{ECD78CC5-138F-4809-A764-2358C2AC2E82}.Debug|x64.Build.0 = Debug|x64
+		{ECD78CC5-138F-4809-A764-2358C2AC2E82}.MinSizeRel|x64.ActiveCfg = Debug|x64
+		{ECD78CC5-138F-4809-A764-2358C2AC2E82}.MinSizeRel|x64.Build.0 = Debug|x64
+		{ECD78CC5-138F-4809-A764-2358C2AC2E82}.Release|x64.ActiveCfg = Release|x64
+		{ECD78CC5-138F-4809-A764-2358C2AC2E82}.Release|x64.Build.0 = Release|x64
+		{ECD78CC5-138F-4809-A764-2358C2AC2E82}.RelWithDebInfo|x64.ActiveCfg = Release|x64
+		{ECD78CC5-138F-4809-A764-2358C2AC2E82}.RelWithDebInfo|x64.Build.0 = Release|x64
+		{3201359D-C071-4089-9EDE-B04D759F1508}.Debug|x64.ActiveCfg = Debug|x64
+		{3201359D-C071-4089-9EDE-B04D759F1508}.Debug|x64.Build.0 = Debug|x64
+		{3201359D-C071-4089-9EDE-B04D759F1508}.MinSizeRel|x64.ActiveCfg = Debug|x64
+		{3201359D-C071-4089-9EDE-B04D759F1508}.MinSizeRel|x64.Build.0 = Debug|x64
+		{3201359D-C071-4089-9EDE-B04D759F1508}.Release|x64.ActiveCfg = Release|x64
+		{3201359D-C071-4089-9EDE-B04D759F1508}.Release|x64.Build.0 = Release|x64
+		{3201359D-C071-4089-9EDE-B04D759F1508}.RelWithDebInfo|x64.ActiveCfg = Release|x64
+		{3201359D-C071-4089-9EDE-B04D759F1508}.RelWithDebInfo|x64.Build.0 = Release|x64
+		{2B8C5D01-FF78-4BFA-B332-5E430586DAF9}.Debug|x64.ActiveCfg = Debug|x64
+		{2B8C5D01-FF78-4BFA-B332-5E430586DAF9}.Debug|x64.Build.0 = Debug|x64
+		{2B8C5D01-FF78-4BFA-B332-5E430586DAF9}.MinSizeRel|x64.ActiveCfg = Debug|x64
+		{2B8C5D01-FF78-4BFA-B332-5E430586DAF9}.MinSizeRel|x64.Build.0 = Debug|x64
+		{2B8C5D01-FF78-4BFA-B332-5E430586DAF9}.Release|x64.ActiveCfg = Release|x64
+		{2B8C5D01-FF78-4BFA-B332-5E430586DAF9}.Release|x64.Build.0 = Release|x64
+		{2B8C5D01-FF78-4BFA-B332-5E430586DAF9}.RelWithDebInfo|x64.ActiveCfg = Release|x64
+		{2B8C5D01-FF78-4BFA-B332-5E430586DAF9}.RelWithDebInfo|x64.Build.0 = Release|x64
+		{4E4F491F-C688-43C4-88DA-AC92BC5FA1B8}.Debug|x64.ActiveCfg = Debug|Any CPU
+		{4E4F491F-C688-43C4-88DA-AC92BC5FA1B8}.Debug|x64.Build.0 = Debug|Any CPU
+		{4E4F491F-C688-43C4-88DA-AC92BC5FA1B8}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
+		{4E4F491F-C688-43C4-88DA-AC92BC5FA1B8}.MinSizeRel|x64.Build.0 = Debug|Any CPU
+		{4E4F491F-C688-43C4-88DA-AC92BC5FA1B8}.Release|x64.ActiveCfg = Release|Any CPU
+		{4E4F491F-C688-43C4-88DA-AC92BC5FA1B8}.Release|x64.Build.0 = Release|Any CPU
+		{4E4F491F-C688-43C4-88DA-AC92BC5FA1B8}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
+		{4E4F491F-C688-43C4-88DA-AC92BC5FA1B8}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
+		{2AC54ADC-3246-427E-9CA9-FEA2DBCB9432}.Debug|x64.ActiveCfg = Debug|Any CPU
+		{2AC54ADC-3246-427E-9CA9-FEA2DBCB9432}.Debug|x64.Build.0 = Debug|Any CPU
+		{2AC54ADC-3246-427E-9CA9-FEA2DBCB9432}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
+		{2AC54ADC-3246-427E-9CA9-FEA2DBCB9432}.MinSizeRel|x64.Build.0 = Debug|Any CPU
+		{2AC54ADC-3246-427E-9CA9-FEA2DBCB9432}.Release|x64.ActiveCfg = Release|Any CPU
+		{2AC54ADC-3246-427E-9CA9-FEA2DBCB9432}.Release|x64.Build.0 = Release|Any CPU
+		{2AC54ADC-3246-427E-9CA9-FEA2DBCB9432}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
+		{2AC54ADC-3246-427E-9CA9-FEA2DBCB9432}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
+		{4E96E5CC-7791-40B3-A8A1-C34FBB736B3B}.Debug|x64.ActiveCfg = Debug|Any CPU
+		{4E96E5CC-7791-40B3-A8A1-C34FBB736B3B}.Debug|x64.Build.0 = Debug|Any CPU
+		{4E96E5CC-7791-40B3-A8A1-C34FBB736B3B}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
+		{4E96E5CC-7791-40B3-A8A1-C34FBB736B3B}.MinSizeRel|x64.Build.0 = Debug|Any CPU
+		{4E96E5CC-7791-40B3-A8A1-C34FBB736B3B}.Release|x64.ActiveCfg = Release|Any CPU
+		{4E96E5CC-7791-40B3-A8A1-C34FBB736B3B}.Release|x64.Build.0 = Release|Any CPU
+		{4E96E5CC-7791-40B3-A8A1-C34FBB736B3B}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
+		{4E96E5CC-7791-40B3-A8A1-C34FBB736B3B}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
+		{51C40BA9-F226-4A35-939D-1ED6F5FF7299}.Debug|x64.ActiveCfg = Debug|Any CPU
+		{51C40BA9-F226-4A35-939D-1ED6F5FF7299}.Debug|x64.Build.0 = Debug|Any CPU
+		{51C40BA9-F226-4A35-939D-1ED6F5FF7299}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
+		{51C40BA9-F226-4A35-939D-1ED6F5FF7299}.MinSizeRel|x64.Build.0 = Debug|Any CPU
+		{51C40BA9-F226-4A35-939D-1ED6F5FF7299}.Release|x64.ActiveCfg = Release|Any CPU
+		{51C40BA9-F226-4A35-939D-1ED6F5FF7299}.Release|x64.Build.0 = Release|Any CPU
+		{51C40BA9-F226-4A35-939D-1ED6F5FF7299}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
+		{51C40BA9-F226-4A35-939D-1ED6F5FF7299}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
+		{4E0E51AA-7B59-4808-B7CC-9EAEDF397460}.Debug|x64.ActiveCfg = Debug|Any CPU
+		{4E0E51AA-7B59-4808-B7CC-9EAEDF397460}.Debug|x64.Build.0 = Debug|Any CPU
+		{4E0E51AA-7B59-4808-B7CC-9EAEDF397460}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
+		{4E0E51AA-7B59-4808-B7CC-9EAEDF397460}.MinSizeRel|x64.Build.0 = Debug|Any CPU
+		{4E0E51AA-7B59-4808-B7CC-9EAEDF397460}.Release|x64.ActiveCfg = Release|Any CPU
+		{4E0E51AA-7B59-4808-B7CC-9EAEDF397460}.Release|x64.Build.0 = Release|Any CPU
+		{4E0E51AA-7B59-4808-B7CC-9EAEDF397460}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
+		{4E0E51AA-7B59-4808-B7CC-9EAEDF397460}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
+	EndGlobalSection
+	GlobalSection(SolutionProperties) = preSolution
+		HideSolutionNode = FALSE
+	EndGlobalSection
+	GlobalSection(ExtensibilityGlobals) = postSolution
+		SolutionGuid = {422D1620-0166-4C26-9A81-CA8A2EB1AB58}
+	EndGlobalSection
+	GlobalSection(Performance) = preSolution
+		HasPerformanceSessions = true
+	EndGlobalSection
+	GlobalSection(NestedProjects) = preSolution
+		{56A8A70A-DA03-4CED-A144-EC10F1379FD8} = {4564E677-343A-46CB-AF02-AFC4E38BF94B}
+		{915EF238-CA0D-4E73-AF50-88CB3F29C7AD} = {4564E677-343A-46CB-AF02-AFC4E38BF94B}
+		{4E0E51AA-7B59-4808-B7CC-9EAEDF397460} = {915EF238-CA0D-4E73-AF50-88CB3F29C7AD}
+	EndGlobalSection
+EndGlobal

src/DataStreamSuite/BooleanExpressions/Arguments/ArgumentComparer.cs

@@ -6,23 +6,80 @@ using Spark.Core.Collections;
 
 static class ArgumentComparer {
    public static int Compare(
-      IArgument value1, IArgument value2, IDictionary<string,object?> env) {
+      IArgument value1, IArgument value2, IDictionary<string, object?> env) {
       if (value1 is null) {
          throw new ArgumentNullException(nameof(value1));
       }
       if (value2 is null) {
          throw new ArgumentNullException(nameof(value2));
       }
-      var cmp1 = value1.Cast<IComparable>();
-      var cmp2 = value2.Cast<IComparable>();
+      var obj1 = value1.GetValue(env);
+      var obj2 = value2.GetValue(env);
+      if (obj1 is null) {
+         if (obj2 is null) {
+            return 0;
+         }
+         return -1;
+      }
+      if (obj2 is null) {
+         return 1;
+      }
+      return compareObjs(obj1, obj2);
+   }
+
+   static int compareObjs(object obj1, object obj2) {
       try {
-         var v1 = cmp1.GetValue(env);
-         var v2 = cmp2.GetValue(env);
-         return v1.GetType().IsNumeric() ?
-            new NumericComparer().Compare(v1, v2) : v1.CompareTo(v2);
+         var typ1 = obj1.GetType();
+         var typ2 = obj2.GetType();
+         bool areSameTypes = typ1 == typ2;
+         if (areSameTypes) {
+            return ((IComparable)obj1).CompareTo(obj2);
+         }
+         if (typ1.IsNumeric()) {
+            return tryCompareNumeric(obj1, obj2);
+         } else if (typ2.IsNumeric()) {
+            return -tryCompareNumeric(obj2, obj1);
+         } else if (typ1.IsEnum) {
+            if (typ2 == typeof(string)) {
+               return -((string)obj2).CompareTo(obj1.ToString());
+            }
+         } else if (typ1 == typeof(string)) {
+            return ((string)obj1).CompareTo(obj2.ToString());
+         } else if (typ1 == typeof(DateTime)) {
+            if (typ2 == typeof(TimeSpan)) {
+               return ((DateTime)obj1).Ticks.CompareTo(((TimeSpan)obj2).Ticks);
+            }
+         } else if (typ1 == typeof(TimeSpan)) {
+            if (typ2 == typeof(DateTime)) {
+               return ((TimeSpan)obj1).Ticks.CompareTo(((DateTime)obj2).Ticks);
+            }
+         }
       } catch (Exception ex) {
          Log.Ui.error($"Comparison failed: {ex}");
          throw;
       }
+      throw new ArgumentException("Failed to compare obj1 and obj2", nameof(obj1));
+   }
+
+   static int tryCompareNumeric(object numericObj1, object obj2) {
+      var typ2 = obj2.GetType();
+      if (typ2.IsNumeric()) {
+         return new NumericComparer().Compare(numericObj1, obj2);
+      } else if (typ2.IsEnum) {
+         var undTyp2 = typ2.GetEnumUnderlyingType();
+         if (undTyp2 == typeof(byte)) {
+            return new NumericComparer().Compare(numericObj1, (byte)obj2);
+         }
+         if (undTyp2 == typeof(short)) {
+            return new NumericComparer().Compare(numericObj1, (short)obj2);
+         }
+         if (undTyp2 == typeof(int)) {
+            return new NumericComparer().Compare(numericObj1, (int)obj2);
+         }
+         if (undTyp2 == typeof(long)) {
+            return new NumericComparer().Compare(numericObj1, (long)obj2);
+         }
+      }
+      throw new ArgumentException("Failed to compare numeric obj1 and obj2", nameof(obj2));
    }
 }

src/DataStreamSuite/BooleanExpressions/BooleanExpressionTokens.lex

@@ -41,7 +41,6 @@ Identifier               {Letter}({Letter}*[0-9]*_*)*
 OpenParenthesis          \(
 CloseParenthesis         \)
 Comma                    ,
-Negate                   '!'
 MapsTo                   ':'
 IsOfType                 '/'
 
@@ -94,10 +93,6 @@ IsOfType                 '/'
    return (int)Token.COMMA;
 }
 
-{Negate} {
-   return (int)Token.NEGATE;
-}
-
 {MapsTo} {
    return (int)Token.MAPS_TO;
 }

src/DataStreamSuite/BooleanExpressions/BoundBooleanExpression.cs

@@ -2,6 +2,8 @@ namespace Bts.BooleanExpressions;
 
 using Bts.BooleanExpressions.Arguments;
 
+using Spark.Core.Collections;
+
 class BoundBooleanExpression : IBooleanExpression {
    readonly IBooleanExpression expression;
    readonly IDictionary<string, FunctionDefinition> definitions;
@@ -17,10 +19,10 @@ class BoundBooleanExpression : IBooleanExpression {
    }
 
    public bool Evaluate(IDictionary<string, object?> env) {
-      foreach (var kv in env) {
-         identifiers[kv.Key] = kv.Value;
+      if (identifiers.Count > 0) {
+         env = AggregateDictionary.Create(env, identifiers);
       }
-      env = new EvaluationEnvironment(identifiers, definitions);
+      env = new EvaluationEnvironment(env, definitions);
       return expression.Evaluate(env);
    }
 

src/DataStreamSuite/Bts.DataStreamWebApp.Client/src/DataTable.jsx

@@ -1,7 +1,7 @@
 import PropTypes from 'prop-types';
 import { useTheme } from '@mui/material/styles';
 import { DataGrid } from '@mui/x-data-grid';
-import './DataGrid.css'
+import './DataGrid.css';
 import { useMemo, useState } from 'react';
 
 export default function DataTable({ columns, rows }) {
@@ -23,7 +23,7 @@ export default function DataTable({ columns, rows }) {
         headerName: col.name,
         width: columnWidths[col.name],
       };
-    }
+    };
   }, [columnWidths]);
 
   return (

src/DataStreamSuite/Bts.DataStreamWebApp.Client/src/ReactDataGrid.jsx

@@ -13,7 +13,7 @@ function getGridColDef(col) {
 export default function ReactDataGrid({columns, rows}) {
   return (
     <DataGrid className={"data-grid"} rows={rows} columns={columns.map(getGridColDef)} />
-  )
+  );
 }
 
 ReactDataGrid.propTypes = {

src/DataStreamSuite/Bts.DataStreamWebApp.Client/src/SideBar.jsx

@@ -3,7 +3,7 @@ import Button from '@mui/material/Button';
 import { useTheme } from '@mui/material/styles';
 import PropTypes from 'prop-types';
 import Tooltip from '@mui/material/Tooltip';
-import ChevronLeftRounded from "@mui/icons-material/ChevronLeftRounded"
+import ChevronLeftRounded from "@mui/icons-material/ChevronLeftRounded";
 import ChevronRightRounded from '@mui/icons-material/ChevronRightRounded';
 import FileCopyIcon from '@mui/icons-material/FileCopy';
 import './SideBar.css';
@@ -51,7 +51,7 @@ const Sidebar = ({ onSelection, onOpened, onClosed }) => {
       } catch (error) {
         console.error('Error fetching WebSocket config:', error);
       }
-    }
+    };
     fetchCannedQueries();
   }, []);
 

src/DataStreamSuite/Bts.DataStreamWebApp.Client/vite.config.js

@@ -52,7 +52,18 @@ export default defineConfig(({ $$command, mode, $$isSsrBuild, $$isPreview }) =>
       proxy: {
         '^/api': {
           target,
-          secure: false
+          secure: false,
+          configure: (proxy) => {
+            proxy.on('error', (err) => {
+              console.log('proxy error', err);
+            });
+            proxy.on('proxyReq', (proxyReq, req) => {
+              console.log(`Forwarding ${proxyReq.method} ${proxyReq.protocol}//${proxyReq.host}${proxyReq.path} to ${req.socket.remoteAddress}${req.url}`);
+            });
+            proxy.on('proxyRes', (proxyRes, req, res) => {
+              console.log(`Forwarding response ${req.socket.localAddress} to ${res.socket.remoteAddress}. Status=${proxyRes.statusCode}`);
+            });
+          },
         },
         '^/ws': {
           target,

src/DataStreamSuite/Bts.DataStreamWebApp.Server/Bts.DataStreamWebApp.Server.csproj

@@ -1,7 +1,7 @@
 <Project Sdk="Microsoft.NET.Sdk.Web">
 
   <PropertyGroup>
-    <TargetFramework>net7.0</TargetFramework>
+    <TargetFramework>net8.0</TargetFramework>
     <Nullable>enable</Nullable>
     <ImplicitUsings>enable</ImplicitUsings>
     <InvariantGlobalization>true</InvariantGlobalization>
@@ -12,10 +12,10 @@
 
   <ItemGroup>
     <PackageReference Include="CommandLineParser" Version="2.9.1" />
-    <PackageReference Include="Microsoft.AspNetCore.SpaProxy" Version="7.0.20" />
-    <PackageReference Include="Microsoft.AspNetCore.SpaServices.Extensions" Version="7.0.20" />
+    <PackageReference Include="Microsoft.AspNetCore.SpaProxy" Version="8.0.18" />
+    <PackageReference Include="Microsoft.AspNetCore.SpaServices.Extensions" Version="8.0.18" />
     <PackageReference Include="Newtonsoft.Json" Version="13.0.3" />
-    <PackageReference Include="Swashbuckle.AspNetCore" Version="6.4.0" />
+    <PackageReference Include="Swashbuckle.AspNetCore" Version="9.0.3" />
   </ItemGroup>
 
   <ItemGroup>

src/DataStreamSuite/Bts.DataStreamWebApp.Server/Controllers/DataStreamSubscriptionController.cs

@@ -1,30 +1,34 @@
-namespace Bts.DataStreamWebApp.Server.Controllers;
-
-using Microsoft.AspNetCore.Mvc;
-
-[Route("/ws")]
-public class DataStreamSubscriptionController : ControllerBase {
-   readonly ILogger logger;
-
-   public DataStreamSubscriptionController(ILogger<DataStreamSubscriptionController> logger) {
-      this.logger = logger;
-      logger.LogInformation("Creating {ControllerType}", GetType().Name);
-   }
-
-   [Route("/ws")]
-   public async Task Get(CancellationToken cancellationToken) {
-      if (!HttpContext.WebSockets.IsWebSocketRequest) {
-         logger.LogError("Expected websocket request on path '{}'", HttpContext.Request.Path);
-         return;
-      }
-      logger.LogInformation("Establishing websocket connection...");
-      try {
-         using var webSocket = await HttpContext.WebSockets.AcceptWebSocketAsync();
-         logger.LogInformation("Websocket connection established.");
-         using var server = new DataStreamWebSocketServer(webSocket, logger);
-         await server.Run(cancellationToken);
-      } catch (Exception ex) {
-         logger.LogError("DataStreamSubscriptionController.Run threw exception: {Exception}", ex);
-      }
-   }
-}
+namespace Bts.DataStreamWebApp.Server.Controllers;
+
+using Bts.DataStreamCore;
+
+using Microsoft.AspNetCore.Mvc;
+
+[Route("/ws")]
+public class DataStreamSubscriptionController : ControllerBase {
+   readonly ILogger logger;
+   readonly IDataStreamFactory streamFactory;
+
+   public DataStreamSubscriptionController(ILogger<DataStreamSubscriptionController> logger, IEnumerable<IDataStreamFactory> streamFactories) {
+      this.logger = logger;
+      logger.LogInformation("Creating {ControllerType}", GetType().Name);
+      streamFactory = DataStreamFactory.Create(streamFactories);
+   }
+
+   [Route("/ws")]
+   public async Task Get(CancellationToken cancellationToken) {
+      if (!HttpContext.WebSockets.IsWebSocketRequest) {
+         logger.LogError("Expected websocket request on path '{}'", HttpContext.Request.Path);
+         return;
+      }
+      logger.LogInformation("Establishing websocket connection...");
+      try {
+         using var webSocket = await HttpContext.WebSockets.AcceptWebSocketAsync();
+         logger.LogInformation("Websocket connection established.");
+         using var server = new DataStreamWebSocketServer(webSocket, streamFactory, logger);
+         await server.Run(cancellationToken);
+      } catch (Exception ex) {
+         logger.LogError("DataStreamSubscriptionController.Run threw exception: {Exception}", ex);
+      }
+   }
+}

src/DataStreamSuite/Bts.DataStreamWebApp.Server/DataStreamWebSocketServer.cs

@@ -2,6 +2,7 @@ namespace Bts.DataStreamWebApp.Server.Controllers;
 
 using bts.websocket_core.json_rpc;
 
+using Bts.DataStreamCore;
 using Bts.DataStreamWebApp.Server.Messages;
 
 using System.Net.WebSockets;
@@ -13,9 +14,11 @@ public class DataStreamWebSocketServer : IDisposable, IAsyncDisposable {
    readonly ILogger logger;
    readonly WebSocket webSocket;
    readonly Dictionary<string, DataStreamSubscription> subscriptions = new();
+   readonly IDataStreamFactory streamFactory;
 
-   public DataStreamWebSocketServer(WebSocket webSocket, ILogger logger) {
+   public DataStreamWebSocketServer(WebSocket webSocket, IDataStreamFactory streamFactory, ILogger logger) {
       this.webSocket = webSocket;
+      this.streamFactory = streamFactory;
       this.logger = logger;
    }
 
@@ -67,7 +70,7 @@ public class DataStreamWebSocketServer : IDisposable, IAsyncDisposable {
       }
       var subId = subReq.SubscriptionId ?? throw new InvalidOperationException("SubscriptionId is null (parse false)");
       disposeSubscription(subId);
-      if (!subReq.ParseQuery(out var vs, out parseErr, out errCode)) {
+      if (!subReq.ParseQuery(streamFactory, out var vs, out parseErr, out errCode)) {
          logger.LogError("Invalid 'subscribe' request: {Reason}", parseErr);
          var rpcError = JsonRpcResponse.CreateError<SubscriptionError>(
             req.Id, new() { Code = (long)errCode, Message = "Failed to parse subscription request", Data = parseErr });

src/DataStreamSuite/Bts.DataStreamWebApp.Server/Dockerfile

@@ -7,5 +7,5 @@ ARG CONF_FILE="src/DataStreamSuite/Bts.DataStreamWebApp.Server/spark.conf"
 RUN dotnet dev-certs https --trust
 COPY --chown=bts:bts "${SOURCE_DIR}" "bin"
 COPY --chown=bts:bts "${WEB_ROOT}" "bin/wwwroot"
-COPY --chown=bts:bts "${CONF_FILE}" ".local/share/bts/Bts.DataStreamWebApp.Server/conf/spark.conf"
+COPY --chown=bts:bts "${CONF_FILE}" ".local/share/DataStream/conf/spark.conf"
 WORKDIR "/home/bts/bin"

src/DataStreamSuite/Bts.DataStreamWebApp.Server/Messages/SubscribeRequest.cs

@@ -4,6 +4,7 @@ using BCG;
 
 using bts.websocket_core.json_rpc;
 
+using Bts.DataStreamCore;
 using Bts.DataStreamExpressions;
 
 using System.Diagnostics.CodeAnalysis;
@@ -37,7 +38,12 @@ public class SubscribeRequest {
       return true;
    }
 
-   internal bool ParseQuery([NotNullWhen(true)] out ViewStream? stream, [NotNullWhen(false)] out SubscriptionError? error, out ErrorCode errorCode) {
+   internal bool ParseQuery(
+      IDataStreamFactory streamFactory,
+      [NotNullWhen(true)] out ViewStream? stream,
+      [NotNullWhen(false)] out SubscriptionError? error,
+      out ErrorCode errorCode
+   ) {
       stream = null;
       errorCode = ErrorCode.None;
       error = null;
@@ -54,7 +60,7 @@ public class SubscribeRequest {
       }
       Exception? exception = null;
       try {
-         stream = exp.getView(new Dictionary<string, StatementExpression>(0));
+         stream = exp.Evaluate(streamFactory) as ViewStream;
       } catch (Exception ex) {
          exception = ex;
       }

src/DataStreamSuite/Bts.DataStreamWebApp.Server/Program.cs

@@ -1,5 +1,6 @@
 using Bts.DataStreamCore.Services;
 using Bts.PubSub.Services;
+using Bts.SharedStreams.Services;
 
 using Spark.Api.Services;
 using Spark.Core.Services;
@@ -78,5 +79,9 @@ static void ConfigureServices(IServiceCollection services, IConfiguration config
       .AddSparkApi(config.GetSection("SparkApi"))
       .AddMasterWorker(config.GetSection("MasterWorker"))
       .AddViewStreamDatabase()
+      .AddPrimitiveStreams(config.GetSection("PrimitiveStreams"))
+      .AddMarketDataStreams()
+      .AddNaturalLanguageStreams(config.GetSection("NaturalLanguageStream"))
+      .AddCompositeDataStreamFactory()
       .AddPubSubClient(config.GetSection("PubSub"));
 }

src/DataStreamSuite/Bts.DataStreamWebApp.Server/appsettings.NaturalLanguageCustom.json

@@ -0,0 +1,45 @@
+{
+  "NaturalLanguagePrompt": {
+    "prompts": [
+      {
+        "category": "Impact of financial events on the market",
+        "description": "Show how trading metrics (e.g., price movement, volume) are affected by events related to finance (rate announcements, earnings reports, etc.).",
+        "prompt": "Show me the price movement and volume of Crude Oil futures during the 5-minute window following the last 5 OPEC meetings",
+        "examples": [
+          {
+            "natural_language": "Show me the price movement and volume of Crude Oil futures during the 5-minute window following the last 5 OPEC meetings",
+            "dsl": [
+              "time_idx=EventTimeIndex.OpecMeetings(which: \"last 5\", duration: \"+5min\")",
+              "contract_idx=ActiveFutureContracts(predicate: \"product==CL\")",
+              "Bars(time_index: time_idx, contract_index: contract_idx, fields: [\"price_movement\", \"volume\"])"
+            ]
+          },
+          {
+            "natural_language": "How do OPEC meetings affect the price of Crude futures? Examine the 10 minutes after each meeting this year.",
+            "dsl": [
+              "time_idx=EventTimeIndex.OpecMeetings(duration: \"+10min\", predicate: \"timestamp.year==2024\")",
+              "contract_idx=ActiveFutureContracts(predicate: \"product==CL\")",
+              "Bars(time_index: time_idx, contract_index: contract_idx, fields: [\"price_movement\", \"volume\"])"
+            ]
+          },
+          {
+            "natural_language": "Show me how the SP 500 price moves after FOMC rate announcements that day",
+            "dsl": [
+              "time_idx=EventTimeIndex.FomcRateAnnouncements(which: \"timestamp.year==2024\", duration: \"+10min\")",
+              "contract_idx=ActiveFutureContracts(predicate: \"product==ES\")",
+              "Bars(time_index: time_idx, contract_index: contract_idx, fields: [\"price_movement\", \"volume\"])"
+            ]
+          },
+          {
+            "natural_language": "Analyze the impact of the FED's interest rate hikes in 2022 on the 10-year Treasury yield, focusing on the 2-week period following each hike.",
+            "dsl": [
+              "time_idx=FedInterestRateAnnouncements(predicate: \"timestamp.year==2022 && change_pct>0\", duration: \"+2 weeks\")",
+              "contract_idx=ActiveFutureContracts(predicate: \"product==ZN\")",
+              "Bars(time_index: time_idx, contract_index: contract_idx)"
+            ]
+          }
+        ]
+      }
+    ]
+  }
+}

src/DataStreamSuite/Bts.DataStreamWebApp.Server/appsettings.json

@@ -11,8 +11,14 @@
       "Http": {
         "Url": "http://0.0.0.0:8080"
       },
+      "HttpIpv6": {
+        "Url": "http://[::1]:8080"
+      },
       "Https": {
         "Url": "https://0.0.0.0:8443"
+      },
+      "HttpsIpv6": {
+        "Url": "https://[::1]:8443"
       }
     }
   },
@@ -22,6 +28,358 @@
   "ViewStreamDatabase": {
     "Interval": "00:00:00.1"
   },
+  "PrimitiveStream": {
+    "Assemblies": [
+      "DataStream.dll",
+      "SharedStreams.dll"
+    ],
+    "ExcludedTypes": [
+      "SettlementPriceStream",
+      "OHLCVStream"
+    ]
+  },
+  "StreamDatabase": {
+    "Path": "/home/bts/.local/share/bts/silver_pine/store.db",
+    "Query": "select * from metadata"
+  },
+  "NaturalLanguageStream": {
+    "prompts": [
+      {
+        "category": "Settlement",
+        "MulitLineDescription": [
+          "This is a few shot learning example for an NL to DSL. The DSL function in question is Settlement (symbol_or_contract, datetime1). ",
+          "The first argument is a future (like CL, for crude oil) or a specific future contract like crude oil Dec 24 future (CLZ4). ",
+          "The second argument is the date at which the settlement is to be fetched. There is an overloaded function, Settlement ",
+          "(symbol_or_contract, begin_dt, end_dt) which gets the settlements for all dates from begin_dt to end_dt. This will also have ",
+          "datetimes expressed as Today, Yesterday, Last Friday, First Friday of the month, Last Thursday of August etc. You will need to know ",
+          "what Today is, since there will be referecnes to last week, yesterday etc. These examples are assuming Today is 10/11/2024, however ",
+          "you should use the Today's real date"
+        ],
+        "prompt": "What was the settlement for SP 500 Future on June 8th?",
+        "examples": [
+          {
+            "natural_language": "Find Settlement price of the SP500 future on 04/22/2023.",
+            "dsl": "Settlement(\"ES\", 4/22/2023)"
+          },
+          {
+            "natural_language": "Find Settlement price of the Silver Dec 24 Future on 04/22/2023.",
+            "dsl": "Settlement(\"SIZ4\", 4/22/2023)"
+          },
+          {
+            "natural_language": "Find Yesterday's Settlement of the Crude Oil Future.",
+            "dsl": "Settlement(\"CL\", 10/10/2024)"
+          },
+          {
+            "natural_language": "Find today's settlement of the 10 year treasury future.",
+            "dsl": "Settlement(\"ZN\", 10/11/2024)"
+          },
+          {
+            "natural_language": "Find the Settlement of Natural Gas Dec 24 future on the first Friday of Sept.",
+            "dsl": "Settlement(\"NGZ4\", 9/6/2024)"
+          },
+          {
+            "natural_language": "Find the Settlement of WTI Jan 25 future in the first Week of Sept.",
+            "dsl": "Settlement(\"CLF5\", 9/2/2024, 9/6/2024)"
+          },
+          {
+            "natural_language": "Find the Settlement price for the Sp 500 Dec 24 Mar 25  spread for the first two weeks of September.",
+            "dsl": "Settlement(\"ESZ4-ESH5\", 9/14/2024, 9/1/2024)"
+          },
+          {
+            "natural_language": "Find the Settlement of Natural Gas future on the first Friday of Sept.",
+            "dsl": "Settlement(\"NG\", 9/6/2024)"
+          }
+        ]
+      },
+      {
+        "category": "BBO",
+        "MulitLineDescription": [
+          "This is a few shot learning example for an NL to DSL. The DSL function in question is Bbo(args...). ",
+          "It accepts the following parameters: ",
+          "symbol (a product code such as ZN for 10 year T-Note futures), ",
+          "contract (a specific contract, such as ZNZ4 for the 10 Year T-Note future Dec 2024 exp), ",
+          "venue (name of the exchange, such as CME, NASDAQ, NYSE, NYMEX, DERIBIT etc.), ",
+          "interval (sample frequency of the data), ",
+          "begin and end (inclusive beginning and end of the time window to query, respectively), ",
+          "Arguments are specified by name, for example: Bbo(symbol: \"ZN\", venue: \"CME\", interval: \"PT1M\", begin: \"2024-10-10\", end: \"2024-10-11\"). ",
+          "Depending on the wording of the request, some arguments may be omitted. ",
+          "The begin, end, and interval args should be expressed in iso 8601 format. ",
+          "Interpret dates according to the trading schedule rather than calendar date whenever possible. ",
+          "In some prompts, the time window will be expressed in relative terms (e.g., \"last Monday\", or \"yesterday\") ",
+          "which must be translated to exact dates. To correctly interpret the dsl in the examples, assume the current time is 2024-10-11T12:00-05:00."
+        ],
+        "prompt": "Show me bbo for SP 500 futures on Sep 23rd of this year.",
+        "examples": [
+          {
+            "natural_language": "Get SP500 future bbo from Wednesday.",
+            "dsl": "Bbo(symbol: \"ES\", venue: \"CME\", begin: \"2024-10-08T17:00:00-06:00\", end: \"2024-10-09T16:00:00-06:00\")"
+          },
+          {
+            "natural_language": "Get bbo for 10 year T-Notes in the first minute after the market opened yesterday.",
+            "dsl": "Bbo(symbol: \"ES\", venue: \"CME\", begin: \"2024-10-10T17:00:00-06:00\", end: \"2024-10-10T17:01:00-06:00\", exclusive_end: true)"
+          },
+          {
+            "natural_language": "Retrieve NVIDIA L1 books from Oct 3",
+            "dsl": "Bbo(symbol: \"NVDA\", venue: \"NASDAQ\", begin: \"2024-10-02TT18:00:00-05:00\", end: \"2024-10-03T17:00:00-05:00\")"
+          },
+          {
+            "natural_language": "Show me L1 data for Crude oil futures this week",
+            "dsl": "Bbo(symbol: \"CL\", venue: \"CME\", begin: \"2024-10-06:T17:00:00-06:00\")"
+          },
+          {
+            "natural_language": "Fetch hourly bbo for Gold futures this week",
+            "dsl": "Bbo(symbol: \"GC\", venue: \"CME\", begin: \"2024-10-06:T17:00:00-06:00\", interval: \"PT1H\")"
+          },
+          {
+            "natural_language": "What was the minute by minute bbo for Tesla yesterday?",
+            "dsl": "Bbo(symbol: \"TSLA\", venue: \"NASDAQ\", begin: \"2024-10-09T18:00:00-05:00\", end: \"2024-10-10T17:00:00-05:00\", interval: \"PT1M\")"
+          }
+        ]
+      },
+      {
+        "category": "Order flow imbalance",
+        "MulitLineDescription": [
+          "This is a few shot learning example for an NL to DSL. The DSL function in question is OrderImbalance(args...). ",
+          "It accepts the following parameters: ",
+          "symbol (a product code such as ZN for 10 year T-Note futures), ",
+          "contract (a specific contract, such as ZNZ4 for the 10 Year T-Note future Dec 2024 exp), ",
+          "venue (name of the exchange, such as CME, NASDAQ, NYSE, NYMEX, DERIBIT etc.), ",
+          "interval (sample frequency of the data), ",
+          "levels (the number of price levels to examine)",
+          "begin and end (inclusive beginning and end of the time window to query, respectively), ",
+          "inclusive_end (a bool flag that, when true, makes the end time inclusive). ",
+          "Arguments are specified by name, for example: OrderImbalance(symbol: \"ZN\", venue: \"CME\", interval: \"PT1M\", begin: \"2024-10-10\", end: \"2024-10-11\"). ",
+          "Depending on the wording of the request, some arguments may be omitted. ",
+          "The begin, end, and interval args should be expressed in iso 8601 format. ",
+          "Interpret dates according to the trading schedule rather than calendar date whenever possible. ",
+          "In some prompts, the time window will be expressed in relative terms (e.g., \"last Monday\", or \"yesterday\") ",
+          "which must be translated to exact dates. To correctly interpret the dsl in the examples, assume the current time is 2024-10-11T12:00-05:00."
+        ],
+        "examples": [
+          {
+            "natural_language": "Get SP500 order imbalance from Wednesday.",
+            "dsl": "OrderImbalance(symbol: \"ES\", venue: \"CME\", begin: \"2024-10-08T17:00:00-06:00\", end: \"2024-10-09T16:00:00-06:00\", \"PT1M\")"
+          },
+          {
+            "natural_language": "Get SP500 order imbalance from Wednesday for the top 5 price levels.",
+            "dsl": "OrderImbalance(symbol: \"ES\", venue: \"CME\", begin: \"2024-10-08T17:00:00-06:00\", end: \"2024-10-09T16:00:00-06:00\", levels: 5, interval: \"PT1M\")"
+          },
+          {
+            "natural_language": "Get orderflow imbalance for 10 year T-Notes in the first minute after the market opened yesterday.",
+            "dsl": "OrderImbalance(symbol: \"ES\", venue: \"CME\", begin: \"2024-10-10T17:00:00-06:00\", end: \"2024-10-10T17:01:00-06:00\", interval: \"PT1S\")"
+          },
+          {
+            "natural_language": "Retrieve NVIDIA order flow from Oct 3",
+            "dsl": "OrderImbalance(symbol: \"NVDA\", venue: \"NASDAQ\", begin: \"2024-10-02TT18:00:00-05:00\", end: \"2024-10-03T17:00:00-05:00\", interval: \"PT1S\")"
+          },
+          {
+            "natural_language": "Show me order imbalance for Crude oil futures this week",
+            "dsl": "OrderImbalance(symbol: \"CL\", venue: \"CME\", begin: \"2024-10-06:T17:00:00-06:00\", interval: \"PT1H\")"
+          },
+          {
+            "natural_language": "Fetch hourly orderflow for Gold futures this week",
+            "dsl": "OrderImbalance(symbol: \"GC\", venue: \"CME\", begin: \"2024-10-06:T17:00:00-06:00\", interval: \"PT1H\")"
+          },
+          {
+            "natural_language": "What was the minute by minute orderflow for Tesla yesterday?",
+            "dsl": "OrderImbalance(symbol: \"TSLA\", venue: \"NASDAQ\", begin: \"2024-10-09T18:00:00-05:00\", end: \"2024-10-10T17:00:00-05:00\", interval: \"PT1M\")"
+          }
+        ]
+      },
+      {
+        "category": "VWAP",
+        "description": "This is a few shot learning example for an NL to DSL. The DSL function in question is VWAP (symbol_or_contract, datetime1, datetime2). The first argument is a future (like CL, for crude oil, or ES for SP 500) or a specific future contract like crude oil Dec 24 future (CLZ4) or SP 500 Dec 2024 (ESZ4). The second argument is where the VWAP has to be calculated (also the beginning of the time window), and the third argument is datetime specifying the end of the time window. So, for example, a 20-day VWAP on 9/30/2023, the first argument will be 9/30/2023, and the third argument will be 9/10/2023 (20 days prior). This will also have datetimes expressed as Today, Yesterday, Last Friday, Last Sunday, etc. For the purpose of this few-shot prompts, let's assume today is 10/11/2024.",
+        "prompt": "What was the VWAP for SP 500 Future over the past 30 days?",
+        "examples": [
+          {
+            "natural_language": "Find the 20 day VWAP of the SP future on 04/22/2023.",
+            "dsl": "VWAP(\"ES\", 4/22/2023, 4/02/2023)"
+          },
+          {
+            "natural_language": "Find Yesterday's 10 day VWAP of the Crude Oil Future.",
+            "dsl": "VWAP(\"CL\", 10/10/2024, 10/1/2024)"
+          },
+          {
+            "natural_language": "Find today's 30-day VWAP of the 10 year treasury future.",
+            "dsl": "VWAP(\"ZN\", 10/11/2024, 9/11/2024)"
+          },
+          {
+            "natural_language": "Find the 10 hour VWAP of the Natural Gas future from yesterday at 5:00 PM.",
+            "dsl": "VWAP(\"NG\", 9/27/2024 17:00:00, 9/27/2024 7:00:00)"
+          }
+        ]
+      },
+      {
+        "category": "Price Diff",
+        "description": "This is a few shot learning example for an NL to DSL. The DSL function in question is OHLCVPriceDiff (symbol_or_contract, datetime1, datetime2). The first argument is a future (like CL, for crude oil) or a specific future contract like crude oil Dec 24 future (CLZ4). The second and third arguments are dates, that the function finds the difference of prices at. Assume today is 10/15/2024",
+        "prompt": "What is the difference in  SP 500 Future from July 10th to July 14th?",
+        "examples": [
+          {
+            "natural_language": "What is the difference in ZN price from July 10th to July 14th?",
+            "dsl": "OHLCVPriceDiff(\"ZN\", 7/14/2024, 7/10/2024)"
+          },
+          {
+            "natural_language": "What is the price difference in SP Future price from Sept 10th and Sept 5th?",
+            "dsl": "OHLCVPriceDiff(\"ES\", 9/10/2024, 9/5/2024)"
+          },
+          {
+            "natural_language": "What is the price difference in Natural Gas Dec Future price from Aug 30th and Aug 20th?",
+            "dsl": "OHLCVPriceDiff(\"NGZ4\", 8/30/2024, 8/20/2024)"
+          }
+        ]
+      },
+      {
+        "category": "Absolute Volume",
+        "description": "this is a few shot learning example for an NL to DSL. The DSL function in question is AbsVolume(symbol, ContractType, datetime1, datetime2), which computes the highest volume in the date range, specified by datetime1, and datetime2. The first argument is a future symbol (like CL for crude oil, ES for SP 500 etc).  The second argument is the ContractType, which is \"Future\" for futures and \"FutureSpread\" for future spreads. the 3rd argument is the begin date and the 4th argument is the end date, between which the max volume is computed. If no future is specified use \"*\" from the first arugment. For this prompt, assume today is 10/11/2024 ",
+        "prompt": "Which  SP 500 Future traded the most between 7/14/2024 and 7/10/2024",
+        "examples": [
+          {
+            "natural_language": "Which SP Future traded the most between 7/14/2024 and 7/10/2024",
+            "dsl": "AbsVolume(\"ES\", \"Future\", 7/14/2024, 7/10/2024)"
+          },
+          {
+            "natural_language": "Which Gold Spread traded the most between Today and Sept 20th",
+            "dsl": "AbsVolume(\"GC\", \"FutureSpread\", 10/1/2024, 9/20/2024)"
+          },
+          {
+            "natural_language": "Which 10 year Spread traded the highest volume last week?",
+            "dsl": "AbsVolume(\"ZN\", \"FutureSpread\", 9/27/2024, 9/23/2024)"
+          },
+          {
+            "natural_language": "Which Corn Spread traded the most today?",
+            "dsl": "AbsVolume(\"ZC\", \"FutureSpread\", 10/1/2024, 10/1/2024)"
+          },
+          {
+            "natural_language": "Which Gold Future traded the most today?",
+            "dsl": "AbsVolume(\"GC\", \"Future\", 9/30/2024, 9/30/2024)"
+          }
+        ]
+      },
+      {
+        "category": "Roll",
+        "description": "This is a few shot learning example for an NL to DSL. The DSL function in question is FuturesRoll(symbol) which computes the rollover date for a future, ie when the highest volume swtches from one expiration to the next one. ",
+        "prompt": "When does the roll typically occur In  SP 500?",
+        "examples": [
+          {
+            "natural_language": "When does the roll typically occur In CL?",
+            "dsl": "FuturesRoll(\"CL\")"
+          },
+          {
+            "natural_language": "When does the roll typically occur for 10 year treasury?",
+            "dsl": "FuturesRoll(\"ZN\")"
+          },
+          {
+            "natural_language": "When does the roll usually occur for Gold Future",
+            "dsl": "FuturesRoll(\"GC\")"
+          },
+          {
+            "natural_language": "At what date does the future rollover happens for SP500?",
+            "dsl": "FuturesRoll(\"ES\")"
+          },
+          {
+            "natural_language": "When does the future rollover happens for the Corn Future?",
+            "dsl": "FuturesRoll(\"ZC\")"
+          }
+        ]
+      },
+      {
+        "category": "Daily Move",
+        "description": "This is a few shot learning example for an NL to DSL. ",
+        "prompt": "How often did Sp 500 Future settle + or - 10% over the previous day?",
+        "examples": [
+          {
+            "natural_language": "List all the days when the SP 500 closed + or -5% from the previous day's close",
+            "dsl": "ColDiff(filteredOHLCV(\"ES\"),\"ts_event\", -1, \"close\", \"close\", \"close_diff\", 5)"
+          },
+          {
+            "natural_language": "List all the days when the Crude Oil closed + or -2.5% from the previous day's close",
+            "dsl": "ColDiff(filteredOHLCV(\"CL\"),\"ts_event\", -1, \"close\", \"close\", \"close_diff\", 2.5)"
+          },
+          {
+            "natural_language": "Show me all the days when the Ten year Treaury Future opened + or -1% from the previous day's open",
+            "dsl": "ColDiff(filteredOHLCV(\"ZN\"),\"ts_event\", -1, \"open\", \"open\", \"open_diff\", 1.0)"
+          },
+          {
+            "natural_language": "List all the trading days when the Silver Future opened + or -1% from the previous day's close",
+            "dsl": "ColDiff(filteredOHLCV(\"SI\"),\"ts_event\", -1, \"open\", \"open\", \"open_diff\", 1.0)"
+          }
+        ]
+      },
+      {
+        "category": "Average Volume",
+        "description": "This is a few shot learning example for an NL to DSL. For the purpose of this few shot prompts, lets assume today is 10/3/2024 ",
+        "prompt": "When does the most open interest typically shift from one contract to another in SP 500 Future?",
+        "examples": [
+          {
+            "natural_language": "When does the most open interest typically shift from one contract to another in e-minis? ",
+            "dsl": "FuturesRoll(\"ES\")"
+          },
+          {
+            "natural_language": "Usually when does the open interest move from one contract to another in WTI?",
+            "dsl": "FuturesRoll(\"CL\")"
+          }
+        ]
+      },
+      {
+        "category": "Events",
+        "description": "This is a few shot learning example for an NL to DSL. In the NL prompt the user is asking to show the price of a future (treasury future, or SP 500 future etc) around an event type. The event type could be \"FOMC Announcement\" or \"Storage numbers\" etc. The DSL is as follows: EventsPriceAction(future_name, event_type, pre_event_window, post_event_window, number_of_events) first arg is the name of the future, (ex ES or ZN etc), the second argument is the event type, for example, \"FOMC announcement\", the 3rd argument is a time window for how much pre-event data to include in the end result, the 4th argument is how much post-event data to include in the end result. Both are strings of the form \"5d\" for 5 days, \"5h\" for 5 hours and \"120m\" for 120 minutes and so on. Usually, only one of the event periods is provided. Set the other to be half of what the user provided. The last argument is the number of events to consider. So if it’s 7, that means the most recent 7 events. If the user does not specify that, pass a large number.",
+        "prompt": "What did the volume and price of SP500 Future do following the release of the last 10 FOMC rates?",
+        "examples": [
+          {
+            "natural_language": "How did SP 500 Future do in the 5 days following the FOMC announcements?",
+            "dsl": "EventsPriceAction(\"ES\", \"FOMC Announcement\", \"2d\", \"5d\", 10)"
+          },
+          {
+            "natural_language": "How did the price of Treasury Future evolve in the 2 days following the Job Reports?",
+            "dsl": "EventsPriceAction(\"ZN\", \"Job Reports\", \"1d\", \"2d\", 10)"
+          },
+          {
+            "natural_language": "Show me the price movement and volume of Crude Oil futures during the 5-minute window following the last 5 OPEC meetings.",
+            "dsl": "EventsPriceAction(\"CL\", \"OPEC Meeting\", \"2m\", \"5m\", 5)"
+          },
+          {
+            "natural_language": "Show me the price movement and volume of SP 500 futures preceding the 10-minute window of the last 5 OPEC meetings.",
+            "dsl": "EventsPriceAction(\"CL\", \"OPEC Meeting\", \"10m\", \"5m\", 5)"
+          }
+        ]
+      },
+      {
+        "category": "Indicator, Standard Deviation",
+        "MulitLineDescription": [
+          "This is a few shot learning example for an NL to DSL. In the NL prompt the user is asking to show all the days when ",
+          "the price of a future (treasury future, or SP 500 future etc) moved more than m times the N days standard deviation, ",
+          "where both m and N are provided. For example, the DSL for \"Using a 90d standard deviation, give me the dates and ",
+          "amounts of the last 4+ standard deviation moves in WTI futures\" would be: ",
+          "r=standardDeviation(\"ES\", \"90d\")\n",
+          "r=math(r, \"abs_daily_change = Abs(close - open) comp = std_dev * 4\")\n",
+          "r=filter(r,\"abs_daily_change >= comp\"). ",
+          "The first line, creates a stream for StandardDeviation, based on the number of days, for example 90d standard deviation, ",
+          "and assigned it to r, the second line takes r from the first line and passes it to a math stream that creates 2 new columns, ",
+          "abs_daily_change and comp. comp is the factor (4, from 4+ in the prompt) times standard deviation. The third line filters ",
+          "(like a sql where) of rows where abs_daily_change >= comp"
+        ],
+        "prompt": "Using a 90d standard deviation, give me the dates and amounts of the last 1+ standard deviation moves in SP  futures ?",
+        "examples": [
+          {
+            "natural_language": "Using a 90d standard deviation, give me the dates and amounts of the last 1+ standard deviation moves in WTI futures ?",
+            "dsl": "r=standardDeviation(\"CL\", \"90d\")\nr=math(r, \"abs_daily_change = Abs(close - open) comp = std_dev * 1.0\")\nr=filter(r,\"abs_daily_change >= comp\")"
+          },
+          {
+            "natural_language": "Show me the times the SP Futures moved twice the 60 day standard deviation",
+            "dsl": "r=standardDeviation(\"ES\", \"60d\")\nr=math(r, \"abs_daily_change = Abs(close - open) comp = std_dev * 2.0\")\nr=filter(r,\"abs_daily_change >= comp\")"
+          },
+          {
+            "natural_language": "List all the days the Gold Futures moved 1.5 times the 45 day standard deviation",
+            "dsl": "r=standardDeviation(\"GC\", \"45d\")\nr=math(r, \"abs_daily_change = Abs(close - open) comp = std_dev * 1.5\")\nr=filter(r,\"abs_daily_change >= comp\")"
+          },
+          {
+            "natural_language": "List all the days the 10 year treasury future moved more than twice the 1 month standard deviation",
+            "dsl": "r=standardDeviation(\"ZN\", \"30d\")\nr=math(r, \"abs_daily_change = Abs(close - open) comp = std_dev * 2.0\")\nr=filter(r,\"abs_daily_change >= comp\")"
+          }
+        ]
+      }
+    ]
+  },
   "SparkApi": {
     "IsEnabled": false,
     "Credentials": {
@@ -77,7 +435,7 @@
         {
           "TagType": ".*",
           "Assembly": "DataStreamCore",
-         "Class": "bts.PubsubServerGenericDataHandler"
+          "Class": "bts.PubsubServerGenericDataHandler"
         }
       ]
     }

src/DataStreamSuite/Bts.Extensions.Hosting/Bts.Extensions.Hosting.csproj

@@ -1,13 +1,13 @@
 <Project Sdk="Microsoft.NET.Sdk">
 
   <PropertyGroup>
-    <TargetFramework>net7.0</TargetFramework>
+    <TargetFramework>net8.0</TargetFramework>
     <ImplicitUsings>enable</ImplicitUsings>
     <Nullable>enable</Nullable>
   </PropertyGroup>
 
   <ItemGroup>
-    <PackageReference Include="Microsoft.Extensions.Hosting" Version="7.0.1" />
+    <PackageReference Include="Microsoft.Extensions.Hosting" Version="9.0.7" />
   </ItemGroup>
 
 </Project>

src/DataStreamSuite/Bts.Extensions.Hosting/HostBuilderExtensions.cs

@@ -3,15 +3,12 @@
 using Microsoft.Extensions.Configuration;
 using Microsoft.Extensions.Hosting;
 
+using System.Collections.Generic;
 using System.Diagnostics;
 
 public static class HostBuilderExtensions {
    public static IHostBuilder CreateBtsBuilder(string[] args) {
-      // Obtain calling assembly name here, rather than in the lambda where it is actually needed,
-      // because want the name of the assembly that called this function, not the assembly that will eventually invoke
-      // the lambda...that will be the Microsoft.Extensions.Hosting assembly.
-      // Get the name of the assembly that called this function. That will act as the "appsettings"
-      // json file prefix
+      // The calling assembly's name serves as the "appsettings" json file prefix.
       var exeName = GetCallingAssemblyName();
       return Host.CreateDefaultBuilder(args)
          .ConfigureAppConfiguration((context, configBuilder) =>
@@ -56,4 +53,21 @@ public static class HostBuilderExtensions {
          ?.GetName()
          ?.Name
          ?? throw new InvalidOperationException("Failed to determine name of calling assembly (calling method is null)");
+
+   /// <summary>
+   /// Separates command line args from config args, using '--' as the delimiter. Args
+   /// before the delimiter are considered command line args, while those after it are
+   /// config args.
+   /// </summary>
+   public static (string[] Args, string[] ConfigArgs) SplitConfigArgs(string[] args) {
+      int splitIdx = Array.IndexOf(args, "--");
+      if (splitIdx == -1) {
+         return (args, Array.Empty<string>());
+      }
+      var cfgArgs = new string[args.Length - splitIdx - 1];
+      Array.Copy(args, splitIdx + 1, cfgArgs, 0, cfgArgs.Length);
+      var nonCfgArgs = new string[args.Length - cfgArgs.Length - 1];
+      Array.Copy(args, 0, nonCfgArgs, 0, nonCfgArgs.Length);
+      return (nonCfgArgs, cfgArgs);
+   }
 }

src/DataStreamSuite/Bts.Net/Bts.Net.csproj

@@ -1,7 +1,7 @@
 <Project Sdk="Microsoft.NET.Sdk">
 
   <PropertyGroup>
-    <TargetFramework>net7.0</TargetFramework>
+    <TargetFramework>net8.0</TargetFramework>
     <ImplicitUsings>enable</ImplicitUsings>
     <Nullable>enable</Nullable>
   </PropertyGroup>

src/DataStreamSuite/DataStream/BlueOrb/AddColumnRow.cs

@@ -0,0 +1,31 @@
+namespace bts.BlueOrb;
+
+using BCG;
+
+public class AddColumnRow : IBlueRow {
+   readonly IBlueRow inputRow;
+   readonly IBlueTableSchema outputSchema;
+   readonly object? value;
+   readonly int valueIndex;
+
+   public AddColumnRow(IBlueRow inputRow, IBlueTableSchema outputSchema, object? value, int valueIndex) {
+      this.inputRow = inputRow;
+      this.outputSchema = outputSchema;
+      this.value = value;
+      this.valueIndex = valueIndex;
+   }
+
+   public object? this[int index] {
+      get {
+         if (index == valueIndex) {
+            return value;
+         }
+         if (index >= valueIndex) {
+            index--;
+         }
+         return inputRow[index];
+      }
+   }
+
+   IBlueTableSchema IBlueRow.schema => outputSchema;
+}

src/DataStreamSuite/DataStream/BlueOrb/AddColumnTable.cs

@@ -0,0 +1,62 @@
+namespace bts.BlueOrb;
+
+using BCG;
+
+using Bts.DataStreamCore;
+
+using System;
+using System.Collections.Generic;
+
+public class AddColumnTable : IBlueRowCollection {
+   readonly IBlueRowCollection inputRows;
+   readonly Func<IBlueRow, object?> getValue;
+   readonly IBlueTableSchema outputSchema;
+   readonly int valueIndex;
+
+   public AddColumnTable(IBlueRowCollection inputRows, string columnName, Type columnType, Func<IBlueRow, object?> getValue) {
+      this.inputRows = inputRows;
+      this.getValue = getValue;
+      var inputCols = inputRows.schema.columnInfo;
+      var outputCols = new ColumnInfo[inputCols.Length + 1];
+      Array.Copy(inputCols, outputCols, inputCols.Length);
+      outputCols[^1] = new(columnName, inputCols.Length, columnType);
+      outputSchema = new BlueTableSchema(outputCols, "AddColumnTable");
+      valueIndex = inputCols.Length;
+   }
+
+   public IBlueRow this[int id] {
+      get {
+         var r = inputRows[id];
+         return new AddColumnRow(r, outputSchema, getValue(r), valueIndex);
+      }
+   }
+
+   public IBlueTableSchema schema => outputSchema;
+
+   public int Count => inputRows.Count;
+
+   public bool containsID(int id) =>
+      inputRows.containsID(id);
+
+   public IEnumerator<IBlueRow> GetEnumerator() {
+      foreach (var r in inputRows) {
+         yield return new AddColumnRow(r, outputSchema, getValue(r), valueIndex);
+      }
+   }
+
+   public IBlueRow GetRow(int index) {
+      var r = inputRows.GetRow(index);
+      return new AddColumnRow(r, outputSchema, getValue(r), valueIndex);
+   }
+
+   public int IndexOf(IBlueRow item) {
+      int id = item.Id();
+      if (!inputRows.containsID(id)) {
+         return -1;
+      }
+      var r = inputRows[id];
+      return BlueRowEqualityComparer.Equals(new AddColumnRow(r, outputSchema, getValue(r), valueIndex), item)
+         ? inputRows.IndexOf(r)
+         : -1;
+   }
+}

src/DataStreamSuite/DataStream/BlueOrb/ApplyStream.cs

@@ -0,0 +1,93 @@
+#nullable enable
+namespace bts.BlueOrb;
+
+using BCG;
+
+using Bts.DataStream;
+using Bts.DataStreamCore;
+
+using Spark.Core.Disposables;
+
+using System;
+
+public class ApplyStream : ViewStream {
+   readonly ViewStream source;
+   readonly Func<IBlueRow, object?> apply;
+   IBlueTableSchema? outputSchema;
+   BlueTableImmutable? lastFired;
+   IDisposable? sub;
+
+   public ApplyStream(ViewStream source, IFunctionEvaluator applyEvaluator) {
+      this.source = source;
+      apply = Apply;
+
+      object? Apply(IBlueRow r) {
+         var res = applyEvaluator.Evaluate(string.Empty, new StreamArgument[] { (null, 0, r) });
+         return res;
+      }
+   }
+
+   public ApplyStream(ViewStream source, Func<IBlueRow, object?> apply) {
+      this.source = source;
+      this.apply = apply;
+   }
+
+   protected override void myOnFirstSubscriber() {
+      sub = source.Subscribe(OnSourceChanged);
+
+      void OnSourceChanged(IDataPacket dp) {
+         Lazy<BlueTableImmutable> lastFired = new(() => this.lastFired ??= new(Log.notnull(outputSchema)));
+         Lazy<BlueTableImmutable.DataPacketBuilder> bd = new(() => lastFired.Value.ToDataPacketBuilder());
+
+         foreach (var r in dp.deletedRows) {
+            var res = apply(r);
+            if (res is IBlueRow blueRow) {
+               bd.Value.Remove(blueRow.Id());
+            } else if (res is IBlueRowCollection blueRows) {
+               foreach (var br in blueRows) {
+                  bd.Value.Remove(br.Id());
+               }
+            } else {
+               Log.assert(false, $"apply delegate returned obj of unexpected type '{res?.GetType()}'");
+            }
+         }
+         foreach (var r in dp.addedRows) {
+            var res = apply(r);
+            if (res is IBlueRow blueRow) {
+               outputSchema ??= blueRow.schema;
+               bd.Value.AddOrUpdate(blueRow);
+            } else if (res is IBlueRowCollection blueRows) {
+               outputSchema ??= blueRows.schema;
+               foreach (var br in blueRows) {
+                  bd.Value.AddOrUpdate(br);
+               }
+            } else {
+               if (outputSchema is null) {
+                  // We get the outputSchema from the result of the first apply, if it's null, we
+                  // cannot determine the output data type.
+                  // TODO: maybe support specifying return type in DSL?
+                  Log.notnull(res);
+               }
+               var resType = res?.GetType();
+               if (outputSchema is null) {
+                  Log.notnull(resType);
+                  Log.assert(ColumnInfo.isAllowedType(resType));
+                  outputSchema = new BlueTableSchema(new ColumnInfo[] {
+                     new("id", 0, typeof(int)),
+                     new("result", 1, resType),
+                  }, dp.newRows.schema.tableName + "_Apply");
+               }
+               bd.Value.AddOrUpdate(new BlueRow(outputSchema, new object?[] { r.Id(), res }));
+               // Log.assert(false, $"apply delegate returned obj of unexpected type '{res?.GetType()}'");
+            }
+         }
+         if (bd.IsValueCreated) {
+            myFireEvent(bd.Value.ToDataPacket(out this.lastFired));
+         }
+      }
+   }
+
+   protected override void myOnLastSubscriber() {
+      Disposable.dispose(ref sub);
+   }
+}

src/DataStreamSuite/DataStream/BlueOrb/ApplyTable.cs

@@ -0,0 +1,62 @@
+namespace bts.BlueOrb;
+
+using BCG;
+
+using Bts.DataStreamCore;
+
+using System;
+using System.Collections.Generic;
+
+public class ApplyTable : IBlueRowCollection {
+   readonly IBlueRowCollection source;
+   readonly IBlueTableSchema outputSchema;
+   readonly List<IBlueRow> outputRows;
+   readonly Dictionary<int, int> idToIndex = new();
+   readonly Func<IBlueRow, IBlueRow> applyFn;
+
+   public ApplyTable(IBlueRowCollection source, IBlueTableSchema? outputSchema, Func<IBlueRow, IBlueRow> applyFn) {
+      this.source = source;
+      this.applyFn = applyFn;
+      outputRows = new(source.Count);
+      foreach (var r in source) {
+         outputRows.Add(applyFn(r));
+      }
+      if (outputRows.Count > 0) {
+         outputSchema ??= outputRows[0].schema;
+      }
+      this.outputSchema = outputSchema ?? source.schema;
+   }
+
+   public IBlueRow this[int id] => outputRows[idToIndex[id]];
+
+   IBlueRow IList<IBlueRow>.this[int index] {
+      get => GetRow(index);
+      set => throw new NotSupportedException();
+   }
+
+   public IBlueTableSchema schema => outputSchema;
+
+   public int Count => source.Count;
+
+   public bool containsID(int id) =>
+      source.containsID(id);
+
+   public IEnumerator<IBlueRow> GetEnumerator() =>
+      outputRows.GetEnumerator();
+
+   public IBlueRow GetRow(int index) =>
+      outputRows[index];
+
+   public int IndexOf(IBlueRow item) {
+      int id = item.Id();
+      if (!source.containsID(id)) {
+         return -1;
+      }
+      var inputRow = source[id];
+      int idx = source.IndexOf(inputRow);
+      if (BlueRowEqualityComparer.Equals(outputRows[idx], item)) {
+         return idx;
+      }
+      return -1;
+   }
+}

src/DataStreamSuite/DataStream/BlueOrb/AsyncViewStream.cs

@@ -0,0 +1,61 @@
+namespace Bts.DataStream.BlueOrb;
+
+using BCG;
+
+using Bts.DataStream;
+using Bts.DataStreamCore;
+using Bts.PubSub;
+
+using Spark.Core.Disposables;
+
+using System;
+using System.Threading;
+using System.Threading.Tasks;
+
+public static class AsyncViewStream {
+   public static ViewStream Create(IWorkScheduler workScheduler, Func<Task<ViewStream?>> factory) {
+      return ViewStreamFactory.Create(Subscribe);
+
+      IDisposable Subscribe(Action<IDataPacket> fire) {
+         var cts = new CancellationTokenSource();
+         var tcs = new TaskCompletionSource<IDisposable>();
+         var syncRoot = new object();
+         IDisposable? streamSub = null;
+         workScheduler.Post(Init);
+         return Disposable.Create(Dispose);
+
+         async void Init() {
+            lock (syncRoot) {
+               if (cts.IsCancellationRequested) {
+                  return;
+               }
+            }
+            var stream = await factory();
+            if (stream is null) {
+               return;
+            }
+            var tempStreamSub = stream.Subscribe(fire);
+            lock (syncRoot) {
+               if (cts.IsCancellationRequested) {
+                  tempStreamSub.Dispose();
+               } else {
+                  streamSub = tempStreamSub;
+               }
+            }
+         }
+
+         void Dispose() {
+            IDisposable? tempStreamSub;
+            lock (syncRoot) {
+               tempStreamSub = streamSub;
+               if (tempStreamSub is null) {
+                  cts.Cancel();
+                  return;
+               }
+               streamSub = null;
+            }
+            tempStreamSub?.Dispose();
+         }
+      }
+   }
+}

src/DataStreamSuite/DataStream/BlueOrb/BlueAddIndexColumnTable.cs

@@ -0,0 +1,115 @@
+namespace bts.BlueOrb;
+
+using BCG;
+
+using bts.collections;
+using bts.utils;
+
+using Bts.DataStream;
+using Bts.DataStreamCore;
+
+using Spark.Core.Collections;
+
+using System;
+using System.Collections.Generic;
+using System.Linq;
+
+public class BlueAddIndexColumnTable : IBlueRowCollection {
+   readonly IBlueTableSchema schema;
+   readonly IList<object?>[] columnValues;
+   readonly int count;
+
+   BlueAddIndexColumnTable(IBlueTableSchema schema, IList<object?>[] columnValues, int? count = null) {
+      this.schema = schema;
+      this.columnValues = columnValues;
+      this.count = count ?? columnValues[0].Count;
+   }
+
+   IBlueTableSchema IBlueRowCollection.schema => schema;
+
+   public int Count => count;
+
+   public IBlueRow this[int id] =>
+      id > 0 && id <= count
+         ? GetRow(id - 1)
+         : throw new KeyNotFoundException();
+
+   public static BlueAddIndexColumnTable Create(string tableName, ColumnInfo[] columnInfos, IList<object?>[] columnValues) {
+      if (columnInfos.Any(c => c.colName == "id")) {
+         throw new ArgumentException("'id' column already exists", nameof(columnInfos));
+      }
+      if (columnInfos.Length != columnValues.Length) {
+         throw new ArgumentException("columnValues length differs from columnInfos length", nameof(columnValues));
+      }
+      int count = columnValues.FirstOrDefault()?.Count ?? 0;
+      var origCols = columnInfos;
+      var origColValues = columnValues;
+      columnInfos = new ColumnInfo[origCols.Length + 1];
+      columnInfos[0] = new ColumnInfo("id", 0, typeof(int));
+      columnValues = new IList<object?>[origCols.Length + 1];
+      columnValues[0] = new StepList<int>(1, 1, count).WrapSelector(id => (object?)id, null);
+      for (int i = 1; i < columnInfos.Length; i++) {
+         columnInfos[i] = new ColumnInfo(origCols[i - 1].colName, i, origCols[i - 1].colType);
+         columnValues[i] = origColValues[i - 1];
+      }
+      var schema = new BlueTableSchema(columnInfos, tableName);
+      return new(schema, columnValues, count);
+   }
+
+   public static BlueAddIndexColumnTable Create(IBlueTableSchema schema, IList<object?>[] columnValues) {
+      int count = columnValues.FirstOrDefault()?.Count ?? 0;
+      int? idColIdx = schema.TryResolveIndices("id");
+      var idColValues = new StepList<int>(1, 1, count).WrapSelector(id => (object?)id, null);
+      var columnInfos = schema.columnInfo;
+      if (idColIdx.HasValue) {
+         // Replace existing id column. columnValues may or may not contain the old ids (if its
+         // length is one less column than that of columnInfos, it doesn't contain them).
+         switch (columnValues.Length - schema.columnInfo.Length) {
+            case 0:
+               columnValues[idColIdx.Value] = idColValues;
+               break;
+            case -1:
+               columnValues = ArrayUtils.InsertExact(columnValues, idColIdx.Value, true, idColValues);
+               break;
+            default:
+               throw new ArgumentException("ColumnInfos length disagrees with columnValues length");
+         }
+      } else {
+         if (columnValues.Length != columnInfos.Length) {
+            throw new ArgumentException("ColumnInfos length disagrees with columnValues length");
+         }
+         var origColInfos = columnInfos;
+         columnInfos = new ColumnInfo[columnInfos.Length + 1];
+         columnInfos[0] = new("id", 0, typeof(int));
+         for (int i = 1; i < columnInfos.Length; i++) {
+            var c = origColInfos[i - 1];
+            columnInfos[i] = new(c.colName, c.colNo + 1, c.colType);
+         }
+         columnValues = ArrayUtils.InsertExact(columnValues, 0, true, idColValues);
+      }
+      return new(new BlueTableSchema(schema.tableName, columnInfos), columnValues);
+   }
+
+   public bool containsID(int id) =>
+      id > 0 && id <= count;
+
+   public IList<object?> GetColumn(int index) =>
+      columnValues[index];
+
+   public IBlueRow GetRow(int index) =>
+      new BlueRowColumnWrapper(schema, index, columnValues);
+
+   public int IndexOf(IBlueRow item) {
+      int id = item.Id();
+      return id > 0 && id <= count && BlueRowEqualityComparer.Equals(GetRow(id - 1), item)
+         ? id - 1
+         : -1;
+   }
+
+   public IEnumerator<IBlueRow> GetEnumerator() {
+      int count = Count;
+      for (int i = 0; i < count; i++) {
+         yield return GetRow(i);
+      }
+   }
+}

src/DataStreamSuite/DataStream/BlueOrb/BlueColumnRowWrapper.cs

@@ -0,0 +1,21 @@
+namespace bts.BlueOrb;
+
+using BCG;
+
+using System.Collections.Generic;
+
+public class BlueRowColumnWrapper : IBlueRow {
+   readonly int index;
+   readonly IList<object?>[] columns;
+
+   public BlueRowColumnWrapper(IBlueTableSchema schema, int index, IList<object?>[] columns) {
+      this.schema = schema;
+      this.index = index;
+      this.columns = columns;
+   }
+
+   public object? this[int index] =>
+      columns[index][this.index];
+
+   public IBlueTableSchema schema { get; }
+}

src/DataStreamSuite/DataStream/BlueOrb/BlueIndexedColumnTable.cs

@@ -0,0 +1,52 @@
+namespace bts.BlueOrb;
+
+using BCG;
+
+using Bts.DataStreamCore;
+
+using mbts.utils;
+
+using System.Collections.Generic;
+using System.Linq;
+
+public class BlueIndexedColumnTable : IBlueRowCollection {
+   readonly IBlueTableSchema schema;
+   readonly IList<object?>[] columnValues;
+   readonly Dictionary<int, int> rowIdToIndex;
+
+   public BlueIndexedColumnTable(IBlueTableSchema schema, IList<object?>[] columnValues) {
+      this.schema = schema;
+      this.columnValues = columnValues;
+      int idColumnIndex = schema.ResolveIndices("id");
+      rowIdToIndex = columnValues[idColumnIndex]
+         .Select((idObj, idx) => new KeyValuePair<int, int>(ExceptionAssert.Argument.Is<int>(idObj), idx))
+         .ToDictionary();
+   }
+
+   public IBlueRow this[int id] =>
+      rowIdToIndex.TryGetValue(id, out var idx)
+         ? GetRow(idx)
+         : throw new KeyNotFoundException();
+
+   IBlueTableSchema IBlueRowCollection.schema => schema;
+
+   public int Count => rowIdToIndex.Count;
+
+   public bool containsID(int id) =>
+      rowIdToIndex.ContainsKey(id);
+
+   public IEnumerator<IBlueRow> GetEnumerator() {
+      int count = Count;
+      for (int i = 0; i < count; i++) {
+         yield return GetRow(i);
+      }
+   }
+
+   public IBlueRow GetRow(int index) =>
+      new BlueRowColumnWrapper(schema, index, columnValues);
+
+   public int IndexOf(IBlueRow item) =>
+      rowIdToIndex.TryGetValue(item.Id(), out var idx) && BlueRowEqualityComparer.Equals(GetRow(idx), item)
+         ? idx
+         : -1;
+}

src/DataStreamSuite/DataStream/BlueOrb/CSVReaderStream.cs

@@ -0,0 +1,18 @@
+namespace bts.BlueOrb {
+   using System;
+
+   using BCG;
+
+   [DataStreamUserRespresentation("CSVReaderStream")]
+   public class CSVReaderStream : ViewStream {
+      public CSVReaderStream(string file_name)
+         : this(file_name, false) { }
+      public CSVReaderStream(string file_name, bool guess_schema) {
+         ViewStream rs = new BCG.CSVReaderStream(file_name);
+         if (guess_schema) {
+            rs = new SchemaGuesserStream(rs);
+         }
+         mySetRealSource(rs);
+      }
+   }
+}

src/DataStreamSuite/DataStream/BlueOrb/CSVSaverStream.cs

@@ -0,0 +1,13 @@
+namespace bts.BlueOrb {
+   using BCG;
+
+   using System;
+
+   [DataStreamUserRespresentation("CSVSaverStream")]
+   public class CSVSaverStream: ViewStream {
+      public CSVSaverStream(ViewStream parent_str, string file_name) {
+         BCG.CSVSaverStream rs = new BCG.CSVSaverStream(parent_str, file_name, true, Int32.MaxValue, false);
+         mySetRealSource(rs);
+      }
+   }
+}

src/DataStreamSuite/DataStream/BlueOrb/CastStream.cs

@@ -0,0 +1,43 @@
+namespace bts.BlueOrb;
+
+using BCG;
+
+using Bts.DataStream;
+
+using Spark.Core.Disposables;
+
+using System;
+
+public class CastStream : ViewStream, IDisposable {
+   readonly ViewStream inputStream;
+   readonly (string, Type)[] casts;
+   IDisposable? sub;
+
+   public CastStream(ViewStream inputStream, params (string, Type)[] casts) {
+      this.inputStream = inputStream;
+      this.casts = casts;
+   }
+
+   ~CastStream() =>
+      Dispose();
+
+   public void Dispose() {
+      GC.SuppressFinalize(this);
+      Disposable.dispose(ref sub);
+   }
+
+   protected override void myOnFirstSubscriber() {
+      inputStream.Subscribe(OnInputChanged);
+
+      void OnInputChanged(IDataPacket dp) {
+         var dpOut = new DataPacket(
+            new CastTable(dp.newRows, casts),
+            new CastTable(dp.addedRows, casts),
+            new CastTable(dp.deletedRows, casts));
+         myFireEvent(dpOut);
+      }
+   }
+
+   protected override void myOnLastSubscriber() =>
+      Disposable.dispose(ref sub);
+}

src/DataStreamSuite/DataStream/BlueOrb/CastTable.cs

@@ -0,0 +1,123 @@
+namespace bts.BlueOrb;
+
+using BCG;
+
+using bts.collections;
+
+using Bts.DataStream;
+using Bts.DataStreamCore;
+
+using Spark.Core.Collections;
+
+using System;
+using System.Collections.Generic;
+
+public class CastTable : IBlueRowCollection {
+   readonly IBlueRowCollection rows;
+   readonly IBlueTableSchema outputSchema;
+   readonly (int Index, Func<IBlueRow, object?> GetValue)[] casts;
+   readonly IComparer<(int Index, Func<IBlueRow, object?> GetValue)> castComparer =
+      TupleComparer.Comparer<int, Func<IBlueRow, object?>>(item1Ordinal: 0);
+
+   public CastTable(IBlueRowCollection rows, params (string, Type)[] casts) {
+      this.rows = rows;
+      var inputSchema = rows.schema;
+      var inputCols = inputSchema.columnInfo;
+      var castIndices = inputSchema.ResolveIndices(Array.ConvertAll(casts, c => c.Item1));
+      this.casts = new (int Index, Func<IBlueRow, object?> GetValue)[casts.Length];
+      var outputCols = inputCols.ToArray();
+      for (int i = 0; i < casts.Length; i++) {
+         var colIdx = castIndices[i];
+         this.casts[i] = (colIdx, GetCaster(casts[i].Item2));
+         var inputCol = inputCols[colIdx];
+         Log.assert(inputCol.colNo == colIdx);
+         outputCols[colIdx] = new(inputCol.colName, inputCol.colNo, casts[i].Item2);
+
+         Func<IBlueRow, object?> GetCaster(Type to) {
+            if (to == typeof(short)) {
+               return r => r.GetInt16(colIdx);
+            }
+            if (to == typeof(int)) {
+               return r => r.GetInt32(colIdx);
+            }
+            if (to == typeof(long)) {
+               return r => r.GetInt64(colIdx);
+            }
+            if (to == typeof(ushort)) {
+               return r => r.GetUInt16(colIdx);
+            }
+            if (to == typeof(uint)) {
+               return r => r.GetUInt32(colIdx);
+            }
+            if (to == typeof(ulong)) {
+               return r => r.GetUInt64(colIdx);
+            }
+            if (to == typeof(double)) {
+               return r => r.GetFloat64(colIdx);
+            }
+            if (to == typeof(float)) {
+               return r => r.GetFloat32(colIdx);
+            }
+            if (to == typeof(DateTime)) {
+               return r => r.GetDateTime(colIdx);
+            }
+            if (to == typeof(TimeSpan)) {
+               return r => r.GetTimeSpan(colIdx);
+            }
+            throw new NotImplementedException($"CastTable cast not implemented for (out) type {to}");
+         }
+      }
+      outputSchema = new BlueTableSchema(inputSchema.tableName + "_Cast", outputCols);
+   }
+
+   public IBlueRow this[int id] =>
+      new Row(rows[id], outputSchema, casts, castComparer);
+
+   public IBlueTableSchema schema => outputSchema;
+
+   public int Count => rows.Count;
+
+   public bool containsID(int id) =>
+      rows.containsID(id);
+
+   public IEnumerator<IBlueRow> GetEnumerator() {
+      foreach (var r in rows) {
+         yield return new Row(r, outputSchema, casts, castComparer);
+      }
+   }
+
+   public IBlueRow GetRow(int index) =>
+      new Row(rows.GetRow(index), outputSchema, casts, castComparer);
+
+   public int IndexOf(IBlueRow item) {
+      var inputRow = rows[item.Id()];
+      return BlueRowEqualityComparer.Equals(item, new Row(inputRow, outputSchema, casts, castComparer))
+         ? rows.IndexOf(inputRow)
+         : -1;
+   }
+
+   class Row : IBlueRow {
+      readonly IBlueRow source;
+      readonly (int Index, Func<IBlueRow, object?> GetValue)[] casts;
+      readonly IComparer<(int Index, Func<IBlueRow, object?> GetValue)> castComparer;
+      readonly IBlueTableSchema outputSchema;
+
+      public Row(IBlueRow source, IBlueTableSchema outputSchema, (int Index, Func<IBlueRow, object?> GetValue)[] casts, IComparer<(int Index, Func<IBlueRow, object?> GetValue)> castComparer) {
+         this.source = source;
+         this.casts = casts;
+         this.castComparer = castComparer;
+         this.outputSchema = outputSchema;
+      }
+
+      public object? this[int index] {
+         get {
+            var castIdx = casts.BinarySearch((index, null!), castComparer);
+            return castIdx > -1
+               ? casts[castIdx].GetValue(source)
+               : source[index];
+         }
+      }
+
+      IBlueTableSchema IBlueRow.schema => outputSchema;
+   }
+}

src/DataStreamSuite/DataStream/BlueOrb/ColumnInjectionStream.cs

@@ -0,0 +1,22 @@
+namespace bts.BlueOrb;
+
+using BCG;
+
+using Bts.DataStream;
+
+using System.Reactive.Linq;
+
+public class ColumnInjectionStream : ViewStream {
+   public ColumnInjectionStream(ViewStream inputStream, IInjectedColumn[] outputColumns) {
+      IBlueTableSchema? outputSchema = null;
+      mySetRealSource(inputStream.AsObservable().Select(Transform).AsViewStream());
+
+      IDataPacket Transform(IDataPacket dp) {
+         outputSchema ??= ColumnInjectionTable.GetOutputSchema(dp.Schema, outputColumns);
+         var newRows = new ColumnInjectionTable(dp.newRows, outputSchema, outputColumns);
+         var addedRows = new ColumnInjectionTable(dp.addedRows, outputSchema, outputColumns);
+         var deletedRows = new ColumnInjectionTable(dp.deletedRows, outputSchema, outputColumns);
+         return new DataPacket(newRows, addedRows, deletedRows);
+      }
+   }
+}

src/DataStreamSuite/DataStream/BlueOrb/ColumnInjectionTable.cs

@@ -0,0 +1,156 @@
+namespace bts.BlueOrb;
+
+using BCG;
+
+using Bts.DataStreamCore;
+
+using System;
+using System.Collections.Generic;
+
+public interface IInjectedColumn {
+   string Name { get; }
+   Type Type { get; }
+   int Ordinal { get; }
+   ColumnInfo ColumnInfo => new(Name, Ordinal, Type);
+   object? GetValue(IBlueRow row);
+}
+
+public interface IInjectedColumn<T> : IInjectedColumn {
+   Type IInjectedColumn.Type => typeof(T);
+
+   new T? GetValue(IBlueRow row);
+
+   object? IInjectedColumn.GetValue(IBlueRow row) =>
+      GetValue(row);
+}
+
+public class InjectedColumn : IInjectedColumn {
+   readonly Func<IBlueRow, object?> getValue;
+
+   public InjectedColumn(string name, int index, Type type, Func<IBlueRow, object?> getValue) {
+      Name = name;
+      Ordinal = index;
+      Type = type;
+      this.getValue = getValue;
+   }
+
+   public string Name { get; }
+   public int Ordinal { get; }
+   public Type Type { get; }
+
+   public object? GetValue(IBlueRow row) =>
+      getValue(row);
+}
+
+public class InjectedColumn<T> : IInjectedColumn<T> {
+   readonly Func<IBlueRow, T?> getValue;
+
+   public InjectedColumn(string name, int index, Func<IBlueRow, T?> getValue) {
+      Name = name;
+      Ordinal = index;
+      this.getValue = getValue;
+   }
+
+   public string Name { get; }
+   public int Ordinal { get; }
+
+   public T? GetValue(IBlueRow row) =>
+      getValue(row);
+}
+
+public class ColumnInjectionTable : IBlueRowCollection {
+   readonly IBlueTableSchema outputSchema;
+   readonly IInjectedColumn[] outputColumns;
+   readonly IBlueRowCollection source;
+
+   public ColumnInjectionTable(IBlueRowCollection source, IBlueTableSchema outputSchema, IInjectedColumn[] outputColumns) {
+      this.source = source;
+      this.outputSchema = outputSchema;
+      this.outputColumns = outputColumns;
+   }
+
+   public static ColumnInjectionTable Create(IBlueRowCollection source, IEnumerable<IInjectedColumn> injectedColumns) {
+      var inputCols = source.schema.columnInfo;
+      var outputColumns = new List<IInjectedColumn>(inputCols.Length + 1);
+      var outputColInfos = new List<ColumnInfo>(inputCols.Length + 1);
+      int injectionIdx = 0;
+      using var en = injectedColumns.GetEnumerator();
+      bool hasMoreInjections = en.MoveNext();
+      for (int i = 0; i < inputCols.Length;) {
+         if (hasMoreInjections && en.Current.Ordinal <= i) {
+            outputColInfos.Add(new(en.Current.Name, outputColumns.Count, en.Current.Type));
+            outputColumns.Add(en.Current);
+            injectionIdx++;
+            continue;
+         }
+         var ci = inputCols[i];
+         outputColInfos.Add(new(ci.colName, outputColumns.Count, ci.colType));
+         outputColumns.Add(new InjectedColumn(ci.colName, outputColumns.Count, ci.colType, r => r[ci.colNo]));
+      }
+      var outputSchema = new BlueTableSchema(outputColInfos.ToArray(), $"ColumnInjection({source.schema.tableName})");
+      return new(source, outputSchema, outputColumns.ToArray());
+   }
+
+   public static IBlueTableSchema GetOutputSchema(IBlueTableSchema inputSchema, IEnumerable<IInjectedColumn> injectedColumns) {
+      var inputCols = inputSchema.columnInfo;
+      var outputColumns = new List<IInjectedColumn>(inputCols.Length + 1);
+      var outputColInfos = new List<ColumnInfo>(inputCols.Length + 1);
+      using var en = injectedColumns.GetEnumerator();
+      bool hasMoreInjections = en.MoveNext();
+      for (int i = 0; i < inputCols.Length;) {
+         if (hasMoreInjections && en.Current.Ordinal <= i) {
+            outputColInfos.Add(new(en.Current.Name, outputColumns.Count, en.Current.Type));
+            outputColumns.Add(en.Current);
+            hasMoreInjections = en.MoveNext();
+            continue;
+         }
+         var ci = inputCols[i];
+         outputColInfos.Add(new(ci.colName, outputColumns.Count, ci.colType));
+         outputColumns.Add(new InjectedColumn(ci.colName, outputColumns.Count, ci.colType, r => r[ci.colNo]));
+         i++;
+      }
+      return new BlueTableSchema(outputColInfos.ToArray(), $"ColumnInjection({inputSchema.tableName})");
+   }
+
+   public IBlueRow this[int id] =>
+      new ColumnInjectionRow(source[id], outputSchema, outputColumns);
+
+   public IBlueTableSchema schema => throw new System.NotImplementedException();
+
+   public int Count => source.Count;
+
+   public bool containsID(int id) =>
+      source.containsID(id);
+
+   public IEnumerator<IBlueRow> GetEnumerator() {
+      foreach (var r in source) {
+         yield return new ColumnInjectionRow(r, outputSchema, outputColumns);
+      }
+   }
+
+   public IBlueRow GetRow(int index) =>
+      new ColumnInjectionRow(source.GetRow(index), outputSchema, outputColumns);
+
+   public int IndexOf(IBlueRow item) {
+      int id = item.Id();
+      return source.containsID(id) && BlueRowEqualityComparer.Equals(item, this[id])
+         ? source.IndexOf(source[id])
+         : -1;
+   }
+
+   class ColumnInjectionRow : IBlueRow {
+      readonly IBlueRow row;
+      readonly IInjectedColumn[] columns;
+
+      public ColumnInjectionRow(IBlueRow row, IBlueTableSchema schema, IInjectedColumn[] columns) {
+         this.row = row;
+         this.schema = schema;
+         this.columns = columns;
+      }
+
+      public IBlueTableSchema schema { get; }
+
+      public object? this[int index] =>
+         columns[index].GetValue(row);
+   }
+}

src/DataStreamSuite/DataStream/BlueOrb/ColumnMaskRow.cs

@@ -0,0 +1,19 @@
+namespace bts.BlueOrb;
+
+using BCG;
+
+public class ColumnMaskRow : IBlueRow {
+   readonly IBlueRow inputRow;
+   readonly (string? Name, int Index)[] columnMap;
+
+   public ColumnMaskRow(IBlueRow inputRow, (string? Name, int Index)[] columnMap, IBlueTableSchema schema) {
+      this.inputRow = inputRow;
+      this.columnMap = columnMap;
+      this.schema = schema;
+   }
+
+   public IBlueTableSchema schema { get; }
+
+   public object? this[int index] =>
+      inputRow[columnMap[index].Index];
+}

src/DataStreamSuite/DataStream/BlueOrb/ColumnMaskStream.cs

@@ -0,0 +1,68 @@
+namespace bts.BlueOrb;
+
+using BCG;
+
+using Bts.DataStream;
+
+using Spark.Core.Disposables;
+
+using System;
+using System.Diagnostics.CodeAnalysis;
+
+public class ColumnMaskStream : ViewStream {
+   readonly ViewStream source;
+   readonly string? columnMaskExpression;
+
+   IDisposable? sub;
+   IBlueTableSchema? outputSchema;
+   // Each item in this array represents an output column as a tuple of
+   // new name, and corresponding index in the input schema.
+   (string? OutputName, int InputIndex)[]? columnMappings;
+   (string From, string To)[]? columnNameMappings;
+   readonly bool includeUnmappedColumns;
+
+   public ColumnMaskStream(ViewStream source, string columnMaskExpression) {
+      this.source = source;
+      this.columnMaskExpression = columnMaskExpression;
+   }
+
+   public ColumnMaskStream(ViewStream source, bool includeUnmappedColumns, params (string From, string To)[] columnNameMappings) {
+      this.source = source;
+      this.columnNameMappings = columnNameMappings;
+      this.includeUnmappedColumns = includeUnmappedColumns;
+   }
+
+   ~ColumnMaskStream() =>
+      Disposable.dispose(ref sub);
+
+   protected override void myOnFirstSubscriber() {
+      Log.isnull(sub);
+      sub = source.Subscribe(OnSourceChanged);
+
+      void OnSourceChanged(IDataPacket dp) {
+         ensureOutputSchema(dp.Schema);
+         var addedRows = new ColumnMaskTable(dp.addedRows, columnMappings, outputSchema);
+         var deletedRows = new ColumnMaskTable(dp.deletedRows, columnMappings, outputSchema);
+         var allRows = new ColumnMaskTable(dp.newRows, columnMappings, outputSchema);
+         myFireEvent(new DataPacket(allRows, addedRows, deletedRows));
+      }
+   }
+
+   protected override void myOnLastSubscriber() =>
+      Disposable.dispose(ref sub);
+
+   [MemberNotNull(nameof(outputSchema), nameof(columnMappings))]
+   void ensureOutputSchema(IBlueTableSchema inputSchema) {
+      if (outputSchema is not null) {
+         Log.notnull(columnMappings);
+         return;
+      }
+      if (columnMaskExpression is not null) {
+         outputSchema = ColumnMaskTable.GetOutputSchema(inputSchema, columnMaskExpression, out columnMappings);
+      } else if (columnNameMappings is not null) {
+         outputSchema = ColumnMaskTable.GetOutputSchema(inputSchema, includeUnmappedColumns, columnNameMappings, out columnMappings);
+      } else {
+         throw new InvalidOperationException("Cannot build output schema");
+      }
+   }
+}

src/DataStreamSuite/DataStream/BlueOrb/ColumnMaskTable.cs

@@ -0,0 +1,120 @@
+namespace bts.BlueOrb;
+
+using BCG;
+
+using bts.collections;
+using bts.utils;
+
+using Bts.DataStreamCore;
+
+using System;
+using System.Collections.Generic;
+using System.Linq;
+
+public class ColumnMaskTable : IBlueRowCollection {
+   readonly IBlueRowCollection inputRows;
+   readonly (string? Name, int Index)[] columnMap;
+
+   public ColumnMaskTable(IBlueRowCollection inputRows, (string? Name, int Index)[] columnMap, IBlueTableSchema schema) {
+      this.inputRows = inputRows;
+      this.columnMap = columnMap;
+      this.schema = schema;
+   }
+
+   public IBlueTableSchema schema { get; }
+
+   public int Count => inputRows.Count;
+
+   public IBlueRow this[int id] =>
+      new ColumnMaskRow(inputRows[id], columnMap, schema);
+
+   public static ColumnMaskTable Create(IBlueRowCollection table, string columnMaskExpression) {
+      var outputSchema = GetOutputSchema(table.schema, columnMaskExpression, out var columnMappings);
+      return new ColumnMaskTable(table, columnMappings, outputSchema);
+   }
+
+   public static IBlueTableSchema GetOutputSchema(IBlueTableSchema inputSchema, string columnMaskExpression, out (string? Name, int Index)[] columnMappings) {
+      bool includeOthers;
+      if (columnMaskExpression.StartsWith('!')) {
+         columnMaskExpression = columnMaskExpression[1..].TrimStart();
+         includeOthers = false;
+      } else {
+         includeOthers = true;
+      }
+      var map = columnMaskExpression.Split(' ').Select(MapColumn).ToOrderedDictionary(tup => tup.Index, tup => tup.Name);
+      var inputCols = inputSchema.columnInfo;
+      if (includeOthers) {
+         var implicitCols = inputCols.Where(c => !map.ContainsKey(c.colNo)).ToList();
+         map.AddRange(implicitCols.Select(c => new KeyValuePair<int, string?>(c.colNo, null)));
+      } else if (!map.Any(c => inputCols[c.Key].colName == "id") && inputCols.TryGetFirst(c => c.colName == "id", out var idColInfo)) {
+         // Add 'id' col if missing
+         map.Insert(0, new(idColInfo.colNo, "id"));
+      }
+      var outputCols = new ColumnInfo[map.Count];
+      for (int i = 0; i < outputCols.Length; i++) {
+         var inputColInfo = inputCols[map.Keys[i]];
+         outputCols[i] = new(map.Values[i] ?? inputColInfo.colName, i, inputColInfo.colType);
+      }
+      columnMappings = map.WrapSelector(kv => (kv.Value, kv.Key), null).ToArray();
+      return new BlueTableSchema(outputCols, inputSchema.tableName + "_ColumnMask");
+
+      (string? Name, int Index) MapColumn(string columnStr) {
+         var parts = columnStr.Split(':');
+         return parts.Length switch {
+            1 => ((string?)null, inputSchema[parts[0]].colNo),
+            2 => (parts[1], inputSchema[parts[0]].colNo),
+            _ => throw new InvalidOperationException("Unable to get output schema due to invalid column expression"),
+         };
+      }
+   }
+
+   public static IBlueTableSchema GetOutputSchema(IBlueTableSchema inputSchema, bool includeUnmapped, (string From, string To)[] columnNameMappings, out (string? OutputName, int InputIndex)[] columnMappings) {
+      var inputCols = inputSchema.columnInfo;
+      ColumnInfo[]? outputCols;
+      if (includeUnmapped) {
+         columnMappings = Array.ConvertAll(inputCols, c => ((string?)c.colName, c.colNo));
+         outputCols = ArrayUtils.Clone(inputCols);
+         foreach (var (from, to) in columnNameMappings) {
+            var idx = inputCols.ResolveIndices(from);
+            var c = outputCols[idx];
+            Log.assert(c.colNo == idx);
+            outputCols[idx] = new(to, c.colNo, c.colType);
+            columnMappings[idx] = (to, idx);
+         }
+      } else {
+         outputCols = new ColumnInfo[columnNameMappings.Length];
+         columnMappings = Array.ConvertAll(columnNameMappings, m => ((string?)m.To, inputCols.ResolveIndices(m.From)));
+         Array.Copy(inputCols, outputCols, inputCols.Length);
+         for (int i = 0; i < columnNameMappings.Length; i++) {
+            var (from, to) = columnNameMappings[i];
+            var idx = inputCols.ResolveIndices(from);
+            outputCols[idx] = new(to, i, inputCols[columnMappings[i].InputIndex].colType);
+         }
+      }
+      return new BlueTableSchema(outputCols, inputSchema.tableName + "_ColumnMask");
+   }
+
+   public bool containsID(int id) =>
+      inputRows.containsID(id);
+
+   public IEnumerator<IBlueRow> GetEnumerator() {
+      foreach (var r in inputRows) {
+         yield return new ColumnMaskRow(r, columnMap, schema);
+      }
+   }
+
+   public IBlueRow GetRow(int index) =>
+      new ColumnMaskRow(inputRows.GetRow(index), columnMap, schema);
+
+   public int IndexOf(IBlueRow item) {
+      int id = item.Id();
+      if (inputRows.containsID(id)) {
+         var inputRow = inputRows[id];
+         var outputRow = new ColumnMaskRow(inputRow, columnMap, schema);
+         if (BlueRowEqualityComparer.Equals(item, outputRow)) {
+            return inputRows.IndexOf(inputRow);
+         }
+      }
+      return -1;
+   }
+}

src/DataStreamSuite/DataStream/BlueOrb/ColumnSelectorRow.cs

@@ -0,0 +1,61 @@
+// namespace bts.BlueOrb;
+
+// using BCG;
+
+// using System;
+// using System.Linq;
+
+// /// <summary>
+// /// Represents a row transformation table (all columns except 'id' can be changed).
+// /// </summary>
+// public class ColumnSelectorRow : IBlueRow {
+//    readonly IBlueRow source;
+//    readonly IBlueTableSchema outputSchema;
+//    readonly Func<object?, object?>[] columnSelectors;
+
+//    public ColumnSelectorRow(IBlueRow source, params (int Index, Type Type, Func<object?, object?> GetValue)[] columnSelectors) {
+//       this.source = source;
+//       this.columnSelectors = Array.ConvertAll(columnSelectors, c => (c.Index, c.GetValue));
+//       outputSchema = getSchema(source.schema, columnSelectors);
+//    }
+
+//    public ColumnSelectorRow(IBlueRow source, IBlueTableSchema outputSchema, params (int Index, Func<object?, object?> GetValue)[] columnSelectors) {
+//       this.source = source;
+//       this.outputSchema = outputSchema;
+//       this.columnSelectors = columnSelectors;
+//    }
+
+//    public object? this[int index] {
+//       get {
+//          for (int i = 0; i < columnSelectors.Length; i++) {
+//             if (columnSelectors[i].Index == index) {
+//                return columnSelectors[i].GetValue(source[index]);
+//             }
+//          }
+//          return source[index];
+//       }
+//    }
+
+//    public IBlueTableSchema schema => outputSchema;
+
+//    public static Func<object?, object?>[] BuildColumnSelectors(IBlueTableSchema schema, params (int Index, Func<object?, object?> GetValue)[] columnSelectors) {
+//       var cols = schema.columnInfo;
+//       var rv = Array.ConvertAll(schema.columnInfo, c => new Func<object?, object?>(_ => _));
+//       for (int i = 0; i < columnSelectors.Length; i++) {
+//          var cs = columnSelectors[i];
+//          rv[cs.Index] = cs.GetValue;
+//       }
+//       return rv;
+//    }
+
+//    static IBlueTableSchema getSchema(IBlueTableSchema inputSchema, (int Index, Type Type, Func<object?, object?> GetValue)[] columnSelectors) {
+//       var inputCols = inputSchema.columnInfo;
+//       var outputCols = inputCols.ToArray();
+//       for (int i = 0; i < columnSelectors.Length; i++) {
+//          var (idx, typ, _) = columnSelectors[i];
+//          var col = inputCols[idx];
+//          outputCols[idx] = new(col.colName, col.colNo, typ);
+//       }
+//       return new BlueTableSchema(outputCols, "ColumnSelector");
+//    }
+// }

src/DataStreamSuite/DataStream/BlueOrb/ColumnSelectorRowFactory.cs

@@ -0,0 +1,107 @@
+namespace bts.BlueOrb;
+
+using BCG;
+
+using bts.collections;
+
+using Bts.DataStreamCore;
+
+using System;
+using System.Collections.Generic;
+using System.Linq;
+
+public class ColumnSelectorRowFactory {
+   readonly IBlueTableSchema outputSchema;
+   readonly Func<object?, object?>[] columnSelectors;
+
+   public ColumnSelectorRowFactory(IBlueTableSchema outputSchema, Func<object?, object?>[] columnSelectors) {
+      this.outputSchema = outputSchema;
+      this.columnSelectors = columnSelectors;
+   }
+
+   public IBlueTableSchema OutputSchema => outputSchema;
+
+   public static ColumnSelectorRowFactory Create(IBlueTableSchema inputSchema, params (string Name, Type Type, Func<object?, object?> GetValue)[] columnOverrides) =>
+      Create(inputSchema, columnOverrides.AsEnumerable());
+
+   public static ColumnSelectorRowFactory Create(IBlueTableSchema inputSchema, params (int Index, Type Type, Func<object?, object?> GetValue)[] columnOverrides) =>
+      Create(inputSchema, columnOverrides.AsEnumerable());
+
+   public static ColumnSelectorRowFactory Create(IBlueTableSchema inputSchema, IEnumerable<(string Name, Type Type, Func<object?, object?> GetValue)> columnOverrides) =>
+      Create(inputSchema, columnOverrides.Select(c => (inputSchema.ResolveIndices(c.Name), c.Type, c.GetValue)));
+
+   public static ColumnSelectorRowFactory Create(IBlueTableSchema inputSchema, IEnumerable<(int Index, Type Type, Func<object?, object?> GetValue)> columnOverrides) {
+      var bd = new Builder(inputSchema);
+      foreach (var c in columnOverrides) {
+         bd.AddOrUpdate(c);
+      }
+      return bd.ToImmutable();
+   }
+
+   public IBlueRow Create(IBlueRow row) =>
+      new Row(row, outputSchema, columnSelectors);
+
+   public Builder ToBuilder() =>
+      new(outputSchema);
+
+   class Row : IBlueRow {
+      readonly IBlueRow source;
+      readonly IBlueTableSchema schema;
+      readonly Func<object?, object?>[] columnSelectors;
+
+      public Row(IBlueRow source, IBlueTableSchema schema, Func<object?, object?>[] columnSelectors) {
+         this.source = source;
+         this.schema = schema;
+         this.columnSelectors = columnSelectors;
+      }
+
+      public object? this[int index] =>
+         columnSelectors[index](source[index]);
+
+      IBlueTableSchema IBlueRow.schema => schema;
+   }
+
+   public class Builder {
+      readonly IBlueTableSchema inputSchema;
+      readonly List<(int Index, Type Type, Func<object?, object?> GetValue)> columnOverrides = new();
+
+      public Builder(IBlueTableSchema inputSchema) {
+         this.inputSchema = inputSchema;
+      }
+
+      public IBlueTableSchema ToSchema() =>
+         ToSchema(inputSchema.tableName + "_ColumnSelector");
+
+      public IBlueTableSchema ToSchema(string tableName) {
+         var inputCols = inputSchema.columnInfo;
+         var outputCols = inputCols.ToArray();
+         foreach (var c in columnOverrides) {
+            var inputCol = inputCols[c.Index];
+            outputCols[c.Index] = new(inputCol.colName, c.Index, c.Type);
+         }
+         return new BlueTableSchema(outputCols, tableName);
+      }
+
+      public ColumnSelectorRowFactory ToImmutable() {
+         var outputSchema = ToSchema();
+         var columnSelectors = Array.ConvertAll(outputSchema.columnInfo, c => new Func<object?, object?>(_ => _));
+         foreach (var cs in columnOverrides) {
+            columnSelectors[cs.Index] = cs.GetValue;
+         }
+         return new(outputSchema, columnSelectors);
+      }
+
+      public void AddOrUpdate((int Index, Type Type, Func<object?, object?> GetValue) item) =>
+         columnOverrides.Add(item);
+
+      public void AddOrUpdate(int index, Type type, Func<object?, object?> getValue) =>
+         columnOverrides.Add((index, type, getValue));
+
+      public void AddOrUpdate<T>(int index, Func<object?, T> getValue) {
+         columnOverrides.Add((index, typeof(T), GetValueWrapper));
+
+         object? GetValueWrapper(object? inputVal) =>
+            getValue(inputVal);
+      }
+   }
+}

src/DataStreamSuite/DataStream/BlueOrb/CountStream.cs

@@ -14,9 +14,6 @@ namespace BCG {
          myVS = vs;
       }
 
-      public CountStream(string tableName)
-         : this(new TableStream(tableName)) { }
-
       protected override void myOnFirstSubscriber() {
          myVS.onChange += myHandleChanges;
       }

src/DataStreamSuite/DataStream/BlueOrb/CovarianceTable.cs

@@ -0,0 +1,117 @@
+namespace bts.BlueOrb;
+
+using BCG;
+
+using Bts.DataStreamCore;
+
+using MathNet.Numerics.Statistics;
+
+using System;
+using System.Collections.Generic;
+using System.Linq;
+
+public class CovarianceTable : IBlueRowCollection {
+   readonly IBlueTableSchema schema;
+   readonly Lazy<IBlueRow> row;
+
+   public CovarianceTable(IBlueRowCollection rows1, IBlueRowCollection rows2) {
+      var inputSchema = rows1.schema;
+      var inputCols = inputSchema.columnInfo;
+      var outputCols = Array.ConvertAll(rows1.schema.columnInfo, ColumnFactory);
+      schema = new BlueTableSchema(outputCols, inputSchema.tableName + "_" + rows2.schema.tableName + "_Covariance");
+      row = new(RowFactory);
+
+      static ColumnInfo ColumnFactory(ColumnInfo inputCol) {
+         var t = inputCol.colType;
+         if (inputCol.colName != "id") {
+            if (t == typeof(short) || t == typeof(int) || t == typeof(long)
+               || t == typeof(ushort) || t == typeof(uint) || t == typeof(ulong)
+               || t == typeof(double) || t == typeof(float)) {
+               t = typeof(double);
+            } else if (t == typeof(DateTime)) {
+               t = typeof(TimeSpan);
+            }
+         }
+         return new(inputCol.colName, inputCol.colNo, t);
+      }
+
+      IBlueRow RowFactory() {
+         var rowVals = new object[inputCols.Length];
+         for (int i = 0; i < inputCols.Length; i++) {
+            var col = inputCols[i];
+            var colType = col.colType;
+            if (col.colName == "id") {
+               rowVals[i] = 1;
+            } else if (colType == typeof(short)) {
+               rowVals[i] = GetValue(v => v as short?);
+            } else if (colType == typeof(int)) {
+               rowVals[i] = GetValue(v => v as int?);
+            } else if (colType == typeof(long)) {
+               rowVals[i] = GetValue(v => v as long?);
+            } else if (colType == typeof(ushort)) {
+               rowVals[i] = GetValue(v => v as ushort?);
+            } else if (colType == typeof(uint)) {
+               rowVals[i] = GetValue(v => v as uint?);
+            } else if (colType == typeof(ulong)) {
+               rowVals[i] = GetValue(v => v as ulong?);
+            } else if (colType == typeof(double)) {
+               rowVals[i] = GetValue(v => v as double?);
+            } else if (colType == typeof(float)) {
+               rowVals[i] = GetValue(v => v as float?);
+            } else if (colType == typeof(DateTime)) {
+               rowVals[i] = TimeSpan.FromSeconds(GetValue(v => (v as DateTime?)?.Ticks / 10_000_000));
+            }
+
+            double GetValue(Func<object?, double?> convert) {
+               var vals1 = rows1.GetColumn(i)
+                  .Select(convert)
+                  .SelectWhereValues();
+               var vals2 = rows2.GetColumn(i)
+                  .Select(convert)
+                  .SelectWhereValues();
+               return vals1.Covariance(vals2);
+            }
+         }
+         return new Row(schema, rowVals);
+      }
+   }
+
+   public IBlueRow this[int id] =>
+      id == 1 ? row.Value : throw new KeyNotFoundException();
+
+   IBlueTableSchema IBlueRowCollection.schema => schema;
+
+   public int Count => 1;
+
+   IBlueRow IList<IBlueRow>.this[int index] {
+      get => GetRow(index);
+      set => throw new NotSupportedException();
+   }
+
+   public bool containsID(int id) =>
+      id == 1;
+
+   public IEnumerator<IBlueRow> GetEnumerator() {
+      yield return row.Value;
+   }
+
+   public IBlueRow GetRow(int index) =>
+      index == 0 ? row.Value : throw new IndexOutOfRangeException();
+
+   public int IndexOf(IBlueRow item) =>
+      BlueRowEqualityComparer.Equals(item, row.Value) ? 0 : -1;
+
+   class Row : IBlueRow {
+      readonly IBlueTableSchema schema;
+      readonly object?[] values;
+
+      public Row(IBlueTableSchema schema, object?[] values) {
+         this.schema = schema;
+         this.values = values;
+      }
+
+      public object? this[int index] => values[index];
+
+      IBlueTableSchema IBlueRow.schema => schema;
+   }
+}

src/DataStreamSuite/DataStream/BlueOrb/CummulativeSumStream.cs

@@ -0,0 +1,93 @@
+namespace bts.BlueOrb;
+
+using BCG;
+
+using bts.collections;
+
+using Bts.DataStream;
+using Bts.DataStreamCore;
+
+using Spark.Core.Disposables;
+
+using System;
+using System.Diagnostics.CodeAnalysis;
+using System.Linq;
+
+public class CummulativeSumStream : ViewStream, IDisposable {
+   readonly ViewStream inputStream;
+   readonly CummulativeSumTable.StringBehavior stringBehavior;
+   IBlueTableSchema? schema;
+   IDisposable? sub;
+   SingleRowTable? lastFired;
+   object?[]? lastFiredValues;
+   Func<object?, object?, object?>[]? columnOperators;
+   int idColumnIdx = -1;
+
+   public CummulativeSumStream(ViewStream inputStream, CummulativeSumTable.StringBehavior stringBehavior = CummulativeSumTable.StringBehavior.LastValue) {
+      this.inputStream = inputStream;
+      this.stringBehavior = stringBehavior;
+   }
+
+   ~CummulativeSumStream() =>
+      Disposable.dispose(ref sub);
+
+   public void Dispose() {
+      GC.SuppressFinalize(this);
+      Disposable.dispose(ref sub);
+   }
+
+   protected override void myOnFirstSubscriber() {
+      Disposable.dispose(ref sub);
+      sub = inputStream.Subscribe(OnInputStreamChanged);
+
+      void OnInputStreamChanged(IDataPacket dp) {
+         ensureInit(dp);
+         var values = lastFiredValues?.ToArray() ?? new object?[schema.columnInfo.Length];
+         foreach (var r in dp.addedRows) {
+            for (int i = 0; i < values.Length; i++) {
+               values[i] = columnOperators[i](values[i], r[i]);
+            }
+         }
+         var table = new SingleRowTable(new BlueRow(schema, values));
+         DataPacket dpOut = new(table, table, lastFired as IBlueRowCollection ?? new BlueTable(schema));
+         lastFired = table;
+         lastFiredValues = values;
+         myFireEvent(dpOut);
+      }
+   }
+
+   protected override void myOnLastSubscriber() =>
+      Disposable.dispose(ref sub);
+
+   [MemberNotNull(nameof(schema), nameof(columnOperators))]
+   void ensureInit(IDataPacket dp) {
+      if (schema is not null) {
+         Log.notnull(columnOperators);
+         return;
+      }
+      schema = dp.Schema;
+      idColumnIdx = schema.ResolveIndices("id");
+      var colInfos = schema.columnInfo;
+      columnOperators = new Func<object?, object?, object?>[schema.columnInfo.Length];
+      for (int i = 0; i < colInfos.Length; i++) {
+         if (i == idColumnIdx) {
+            columnOperators[i] = (x, y) => 1;
+            continue;
+         }
+         columnOperators[i] = schema.columnInfo[i].type_code() switch {
+            TypeCode.Int32 => (x, y) => (x as int? ?? 0) + (BlueDataUtility.GetInt32(y) ?? 0),
+            TypeCode.Int64 => (x, y) => (x as long? ?? 0) + (BlueDataUtility.GetInt64(y) ?? 0),
+            TypeCode.UInt32 => (x, y) => (x as uint? ?? 0) + (BlueDataUtility.GetUInt32(y) ?? 0),
+            TypeCode.UInt64 => (x, y) => (x as ulong? ?? 0) + (BlueDataUtility.GetUInt64(y) ?? 0),
+            TypeCode.Single => (x, y) => (x as float? ?? 0) + (BlueDataUtility.GetFloat32(y) ?? 0),
+            TypeCode.Double => (x, y) => (x as double? ?? 0) + (BlueDataUtility.GetFloat64(y) ?? 0),
+            TypeCode.String => stringBehavior switch {
+               CummulativeSumTable.StringBehavior.Concatenate => (x, y) => (x as string ?? string.Empty) + (y as string ?? string.Empty),
+               _ => (accum, next) => next,
+            },
+            TypeCode.DateTime => (x, y) => y,
+            _ => throw new InvalidOperationException($"Failed to add row values for col '{schema.columnInfo[i]}'"),
+         };
+      }
+   }
+}

src/DataStreamSuite/DataStream/BlueOrb/CummulativeSumTable.cs

@@ -0,0 +1,119 @@
+namespace bts.BlueOrb;
+
+using BCG;
+
+using bts.collections;
+
+using Bts.DataStreamCore;
+
+using System;
+using System.Collections.Generic;
+using System.Linq;
+
+public class CummulativeSumTable : IBlueRowCollection {
+   readonly IBlueRowCollection input;
+   readonly IList<object?>[] columns;
+   readonly StringBehavior stringBehavior;
+
+   public CummulativeSumTable(IBlueRowCollection input, StringBehavior stringBehavior = StringBehavior.LastValue) {
+      this.input = input;
+      var colInfos = input.schema.columnInfo.OrderBy(c => c.colNo).ToArray();
+      if (colInfos.Length > 0) {
+         Log.assert(colInfos[0].colNo == 0 && colInfos[^1].colNo == colInfos.Length - 1);
+      }
+      columns = colInfos.WrapSelector(c => GetColumn(c.colNo), null).ToArray();
+   }
+
+   /// <summary>
+   /// By default, we take the last value for strings intstead of adding (concatenating) them.
+   /// Otherwise it would be too easy to create massive concatenated strings by mistake.
+   /// </summary>
+   public enum StringBehavior {
+      LastValue,
+      Concatenate
+   }
+
+   public IBlueRow this[int id] =>
+      id == 1 ? GetRow(0) : throw new KeyNotFoundException();
+
+   public IBlueTableSchema schema => input.schema;
+
+   public int Count => 1;
+
+   public bool containsID(int id) =>
+      id == 1;
+
+   public IList<object?> GetColumn(int colIndex) {
+      var count = input.Count;
+      var colInfo = input.schema.columnInfo[colIndex];
+      if (colInfo.colName == "id") {
+         return new object?[] { 1 };
+      }
+      object? res = null;
+      var colVals = input.GetColumn(colIndex);
+      if (colInfo.colType == typeof(TimeSpan)) {
+         // TimeSpan is a special case because it doesn't have a TypeCode.
+         res = TimeSpanImpl(colVals);
+      } else {
+         var typeCode = colInfo.type_code();
+         const int MAX_STRING_LENGTH = 1024;
+         res = typeCode switch {
+            TypeCode.Int32 => colVals.Cast<int>().Sum(),
+            TypeCode.Int64 => colVals.Cast<long>().Sum(),
+            TypeCode.UInt32 => colVals.Cast<uint>().Sum(),
+            TypeCode.UInt64 => colVals.Cast<ulong>().Aggregate(0UL, (s, v) => s + v),
+            TypeCode.Single => colVals.Cast<float>().Sum(),
+            TypeCode.Double => colVals.Cast<double>().Sum(),
+            TypeCode.DateTime => colVals.Cast<DateTime>().Aggregate(DateTime.MinValue, (s, v) => new DateTime(s.Ticks + v.Ticks)),
+            TypeCode.String => stringBehavior switch {
+               StringBehavior.Concatenate => colVals.Cast<string>().Aggregate(string.Empty, (s, v) => {
+                  if (s.Length + v.Length > MAX_STRING_LENGTH) {
+                     throw new InvalidOperationException("Cannot sum strings because the result is too long");
+                  }
+                  return s + v;
+               }),
+               _ => count == 0 ? null : colVals[^1] as string,
+            },
+            _ => null,
+         };
+      }
+      return new object?[] { res };
+
+      static object? TimeSpanImpl(IList<object?> colVals) {
+         var ticks = colVals.Cast<TimeSpan>().Select(ts => ts.Ticks).Sum();
+         return TimeSpan.FromTicks(ticks);
+      }
+   }
+
+   public IEnumerator<IBlueRow> GetEnumerator() {
+      yield return GetRow(0);
+   }
+
+   public IBlueRow GetRow(int index) =>
+      new BlueRowColumnWrapper(input.schema, index, columns);
+
+   public int IndexOf(IBlueRow item) {
+      int id = item.Id();
+      var inputRow = input[id];
+      int idx = input.IndexOf(inputRow);
+      if (idx != -1) {
+         var row = GetRow(idx);
+         if (BlueRowEqualityComparer.Equals(item, row)) {
+            return idx;
+         }
+      }
+      return -1;
+   }
+}
+
+public class CummulativeSumTableEvaluator : IFunctionEvaluator {
+   public object? Evaluate(string functionId, IEnumerable<StreamArgument> args) {
+      if (args.TryGetFirst(out var a)) {
+         var table = a.Value as IBlueRowCollection;
+         if (table is not null) {
+            return new CummulativeSumTable(table);
+         }
+      }
+      return null;
+   }
+}

src/DataStreamSuite/DataStream/BlueOrb/DiffStream.cs

@@ -0,0 +1,163 @@
+namespace bts.BlueOrb;
+
+using BCG;
+
+using Bts.DataStream;
+using Bts.DataStreamCore;
+
+using Spark.Core.Collections;
+using Spark.Core.Disposables;
+
+using System;
+using System.Collections.Generic;
+using System.Linq.Expressions;
+
+public class DiffStream : ViewStream, IDisposable {
+   readonly ViewStream source;
+   readonly int period;
+   IBlueTableSchema? outputSchema;
+   IDisposable? sub;
+   BlueTableImmutable? lastFired;
+
+   public DiffStream(ViewStream source, int period) {
+      this.source = source;
+      this.period = period;
+   }
+
+   ~DiffStream() =>
+      Dispose();
+
+   public void Dispose() =>
+      Disposable.dispose(ref sub);
+
+   protected override void myOnFirstSubscriber() {
+      Log.isnull(sub);
+      sub = source.Subscribe(OnSourceChange);
+
+      void OnSourceChange(IDataPacket dp) {
+         if (outputSchema is null) {
+            var inputSchema = dp.Schema;
+            var outputCols = new ColumnInfo[inputSchema.columnInfo.Length];
+            for (int i = 0; i < inputSchema.columnInfo.Length; i++) {
+               outputCols[i] = GetOutputColumn(inputSchema.columnInfo[i]);
+            }
+            outputSchema = new BlueTableSchema($"{inputSchema.tableName}_Diff{period}", outputCols);
+         }
+         lastFired ??= new(outputSchema);
+         var newTable = new DiffTable(dp.newRows, period);
+         var allRows = new List<object?[]>();
+         foreach (IBlueRow r in newTable) {
+            var rvals = new object?[r.schema.columnInfo.Length];
+            int x = 0;
+            foreach (var col in r.schema.columnInfo) {
+               var val = r[col.colNo];
+               rvals[x++] = val;
+            }
+            allRows.Add(rvals);
+         }
+         var bd = lastFired.ToDataPacketBuilder();
+         var removedIndices = new List<int>();
+         var addedIndices = new List<int>();
+         var ops = new List<(int Index, bool IsAdd)>();
+         foreach (var r in dp.deletedRows) {
+            int id = r.Id();
+            var rOut = lastFired[id];
+            var idx = lastFired.IndexOf(rOut);
+            Log.assert(idx >= 0 && idx < lastFired.Count);
+            removedIndices.Add(idx);
+            ops.Add((idx, false));
+         }
+         foreach (var r in dp.addedRows) {
+            var idx = dp.newRows.IndexOf(r);
+            Log.assert(idx >= 0 && idx < dp.newRows.Count);
+            addedIndices.Add(idx);
+            ops.Add((idx, true));
+         }
+         ops.Sort(Comparer<(int Index, bool IsAdd)>.Create((a, b) => a.Index.CompareTo(b.Index)));
+         int rmCount = 0;
+         var removedRows = new Dictionary<int, IBlueRow>();
+         for (int i = 0; i < ops.Count; i++) {
+            var op = ops[i];
+            if (op.IsAdd) {
+               continue;
+            }
+            rmCount++;
+            int j = op.Index/* - rmCount*/;
+            var stop = Math.Min(j + period, lastFired.Count/* - rmCount*/);
+            // Remove the row that was removed in the incoming DataPacket.
+            bd.Remove(lastFired.GetRow(j).Id());
+            for (j++; j < stop; j++) {
+               // Remove rows that changed as a result. We will add the updated versions 
+               // of these rows in the next pass.
+               var r = lastFired.GetRow(j);
+               bd.Remove(r.Id());
+            }
+         }
+         int addCount = ops.Count - rmCount;
+         for (int i = ops.Count - 1; i >= 0; i--) {
+            var op = ops[i];
+            if (op.IsAdd) {
+               addCount--;
+               int j = op.Index;
+               var stop = Math.Min(j + period, newTable.Count);
+               for (; j < stop; j++) {
+                  var r = newTable.GetRow(j);
+                  bd.AddOrUpdate(r);
+               }
+            } else {
+               rmCount--;
+               int j = op.Index - rmCount + addCount;
+               var stop = Math.Min(j + period, newTable.Count);
+               for (; j < stop; j++) {
+                  // Add rows that changed due to the removed row (the row with which they compute
+                  // the diff has changed).
+                  var r = newTable.GetRow(j);
+                  bd.AddOrUpdate(r);
+               }
+            }
+         }
+         var dpOut = bd.ToDataPacket(out lastFired, false);
+         myFireEvent(dpOut);
+      }
+
+      ColumnInfo GetOutputColumn(ColumnInfo inputColumn) {
+         return inputColumn.type_code() switch {
+            TypeCode.Boolean => inputColumn.Clone(),
+            TypeCode.Byte or TypeCode.SByte or TypeCode.Int16 => WithType(TypeCode.Int32),
+            TypeCode.UInt32 or TypeCode.UInt64 => WithType(TypeCode.Int64),
+            _ => inputColumn.Clone(),
+         };
+
+         ColumnInfo WithType(TypeCode typeCode) =>
+            new(inputColumn.colName, inputColumn.colNo, typeCode);
+      }
+   }
+
+   protected override void myOnLastSubscriber() =>
+      Disposable.dispose(ref sub);
+
+   static class ExpressionBuilder {
+      public static Func<IBlueRow, IBlueRow, int?> Int32DiffExpression(int colIndex) {
+         var colIndexParam = Expression.Constant(colIndex, typeof(int));
+         var rowParam = Expression.Parameter(typeof(IBlueRow), "row");
+         var getValMethod = typeof(ExpressionBuilder).GetMethod(nameof(getInt32Value))
+            ?? throw new InvalidOperationException("Failed to find int32 value getter via reflection");
+         var getVal = Expression.Lambda<Func<IBlueRow, int?>>(Expression.Call(null, getValMethod, rowParam, colIndexParam), rowParam).Compile();
+         return ComputeDiff;
+
+         int? ComputeDiff(IBlueRow leftRow, IBlueRow rightRow) {
+            var leftVal = getVal(leftRow);
+            var rightVal = getVal(rightRow);
+            if (leftVal.HasValue && rightVal.HasValue) {
+               return leftVal.Value - rightVal.Value;
+            }
+            return null;
+         }
+      }
+
+      static int? getInt32Value(IBlueRow row, int colIndex) {
+         var obj = row[colIndex];
+         return obj as int?;
+      }
+   }
+}

src/DataStreamSuite/DataStream/BlueOrb/DiffTable.cs

@@ -0,0 +1,101 @@
+namespace bts.BlueOrb;
+
+using BCG;
+
+using bts.collections;
+
+using Bts.DataStreamCore;
+
+using Spark.Core.Collections;
+
+using System;
+using System.Collections.Generic;
+using System.Linq;
+
+public class DiffTable : IBlueRowCollection {
+   readonly IBlueRowCollection input;
+   readonly int period;
+   readonly IList<object?>[] columns;
+
+   public DiffTable(IBlueRowCollection input, int period) {
+      this.input = input;
+      this.period = period;
+      var colInfos = input.schema.columnInfo.OrderBy(c => c.colNo).ToArray();
+      if (colInfos.Length > 0) {
+         Log.assert(colInfos[0].colNo == 0 && colInfos[^1].colNo == colInfos.Length - 1);
+      }
+      columns = colInfos.WrapSelector(c => GetColumn(c.colNo), null).ToArray();
+   }
+
+   public IBlueRow this[int id] =>
+      GetRow(input.IndexOf(input[id]));
+
+   public IBlueTableSchema schema => input.schema;
+
+   public int Count => input.Count;
+
+   public bool containsID(int id) =>
+      input.containsID(id);
+
+   public IList<object?> GetColumn(int colIndex) {
+      var count = input.Count;
+      var colInfo = input.schema.columnInfo[colIndex];
+      if (colInfo.colName == "id") {
+         return new StepList<int>(0, 1, count).as_mutable().WrapSelector(GetId, null);
+      }
+      var typeCode = colInfo.type_code();
+      return new StepList<int>(0, 1, count).as_mutable().WrapSelector(GetValue, null);
+
+      object? GetId(int rowIndex) {
+         var row = input.GetRow(rowIndex);
+         return row.Id();
+      }
+
+      object? GetValue(int rowIndex) {
+         var rightIndex = rowIndex - period;
+         if (rowIndex < 0 || rowIndex >= count || rightIndex < 0 || rightIndex >= count) {
+            return null;
+         }
+         var leftRow = input.GetRow(rowIndex);
+         var rightRow = input.GetRow(rightIndex);
+         return GetDiff(leftRow[colIndex], rightRow[colIndex], typeCode);
+      }
+   }
+
+   public IEnumerator<IBlueRow> GetEnumerator() {
+      var count = input.Count;
+      for (int i = 0; i < count; i++) {
+         yield return GetRow(i);
+      }
+   }
+
+   public IBlueRow GetRow(int index) =>
+      new BlueRowColumnWrapper(input.schema, index, columns);
+
+   public int IndexOf(IBlueRow item) {
+      int id = item.Id();
+      var inputRow = input[id];
+      int idx = input.IndexOf(inputRow);
+      if (idx != -1) {
+         var row = GetRow(idx);
+         if (BlueRowEqualityComparer.Equals(item, row)) {
+            return idx;
+         }
+      }
+      return -1;
+   }
+
+   static object? GetDiff(object? leftVal, object? rightVal, TypeCode colType) {
+      if (leftVal is null || rightVal is null) {
+         return null;
+      }
+      return colType switch {
+         TypeCode.Int32 => (int)leftVal - (int)rightVal,
+         TypeCode.Int64 => (long)leftVal - (long)rightVal,
+         TypeCode.Double => (double)leftVal - (double)rightVal,
+         TypeCode.String => ((string)leftVal).Replace((string)rightVal, string.Empty),
+         TypeCode.DateTime => new DateTime(((DateTime)leftVal).Ticks - ((DateTime)rightVal).Ticks),
+         _ => null,
+      };
+   }
+}

src/DataStreamSuite/DataStream/BlueOrb/FeatherReaderStream.cs

@@ -0,0 +1,506 @@
+namespace bts.BlueOrb;
+
+using Apache.Arrow;
+using Apache.Arrow.Ipc;
+using Apache.Arrow.Types;
+
+using BCG;
+
+using bts.collections;
+
+using Bts.DataStreamCore;
+
+using Spark.Core.Disposables;
+using Spark.Core.Threading;
+
+using System;
+using System.Collections;
+using System.Collections.Generic;
+using System.Collections.Immutable;
+using System.IO;
+using System.Linq;
+using System.Threading.Tasks;
+
+[DataStreamUserRespresentation("featherReader")]
+public class FeatherReaderStream : ViewStream {
+   readonly string file;
+   FileStream fileStream;
+   // Let the table leak in case some dangling references linger around after
+   // myOnLastSubscriber is called. This should be fine since it disposes itself
+   // in its finalizer.
+   FeatherTable? table;
+
+   public FeatherReaderStream(string file) =>
+      this.file = file;
+
+   protected override void myOnFirstSubscriber() {
+      if (!Path.Exists(file)) {
+         throw new InvalidOperationException("Feather file does not exist");
+      }
+      fileStream = File.OpenRead(file);
+      table = new(new ArrowStreamReader(fileStream));
+      myFireEvent(new DataPacket(table, table, new BlueTableImmutable(table.schema)));
+   }
+
+   interface IFeatherColumn {
+      int Count { get; }
+
+      object? GetValue(int i);
+   }
+
+   interface IFeatherColumn<T> : IFeatherColumn {
+      new T? GetValue(int i);
+
+      object? IFeatherColumn.GetValue(int i) =>
+         GetValue(i);
+   }
+
+   class FeatherColumn : IFeatherColumn {
+      readonly IArrowArray array;
+
+      public FeatherColumn(IArrowArray array) {
+         this.array = array;
+      }
+
+      public int Count => array.Length;
+
+      public static IFeatherColumn Create(IArrowArray array) =>
+         array switch {
+            Int32Array a => new FeatherInt32Column(a),
+            Int64Array a => new FeatherInt64Column(a),
+            DoubleArray a => new FeatherFloat64Column(a),
+            StringArray a => new FeatherStringColumn(a),
+            BooleanArray a => new FeatherBooleanColumn(a),
+            Date64Array a => new FeatherDateTimeColumn(a),
+            Time64Array a => new FeatherTimeStampColumn(a),
+            _ => throw new NotSupportedException($"Unsupported type: {array.GetType()}")
+         };
+
+      public object? GetValue(int i) =>
+         array switch {
+            Int32Array a => a.GetValue(i),
+            Int64Array a => a.GetValue(i),
+            DoubleArray a => a.GetValue(i),
+            StringArray a => a.GetString(i),
+            BooleanArray a => a.GetValue(i),
+            Date64Array a => a.GetValue(i),
+            Time64Array a => a.GetValue(i),
+            _ => throw new NotSupportedException($"Unsupported type: {array.GetType()}")
+         };
+   }
+
+   class FeatherInt32Column : IFeatherColumn<int?> {
+      readonly Int32Array array;
+
+      public FeatherInt32Column(Int32Array array) =>
+         this.array = array;
+
+      public int Count => array.Length;
+
+      public int? GetValue(int i) => array.GetValue(i);
+   }
+
+   class FeatherInt64Column : IFeatherColumn<long?> {
+      readonly Int64Array array;
+
+      public FeatherInt64Column(Int64Array array) =>
+         this.array = array;
+
+      public int Count => array.Length;
+
+      public long? GetValue(int i) => array.GetValue(i);
+   }
+
+   class FeatherFloat64Column : IFeatherColumn<double?> {
+      readonly DoubleArray array;
+
+      public FeatherFloat64Column(DoubleArray array) =>
+         this.array = array;
+
+      public int Count => array.Length;
+
+      public double? GetValue(int i) => array.GetValue(i);
+   }
+
+   class FeatherFloat32Column : IFeatherColumn<float?> {
+      readonly FloatArray array;
+
+      public FeatherFloat32Column(FloatArray array) =>
+         this.array = array;
+
+      public int Count => array.Length;
+
+      public float? GetValue(int i) => array.GetValue(i);
+   }
+
+   class FeatherStringColumn : IFeatherColumn<string?> {
+      readonly StringArray array;
+
+      public FeatherStringColumn(StringArray array) =>
+         this.array = array;
+
+      public int Count => array.Length;
+
+      public string? GetValue(int i) => array.GetString(i);
+   }
+
+   class FeatherBooleanColumn : IFeatherColumn<bool?> {
+      readonly BooleanArray array;
+
+      public FeatherBooleanColumn(BooleanArray array) =>
+         this.array = array;
+
+      public int Count => array.Length;
+
+      public bool? GetValue(int i) => array.GetValue(i);
+   }
+
+   class FeatherDateTimeColumn : IFeatherColumn<DateTime?> {
+      readonly Date64Array array;
+
+      public FeatherDateTimeColumn(Date64Array array) =>
+         this.array = array;
+
+      public int Count => array.Length;
+
+      public DateTime? GetValue(int i) {
+         var nanos = array.GetValue(i);
+         if (!nanos.HasValue) {
+            return null;
+         }
+         return new(nanos.Value / 100);
+      }
+   }
+
+   class FeatherTimeStampColumn : IFeatherColumn<DateTime?> {
+      readonly Time64Array array;
+
+      public FeatherTimeStampColumn(Time64Array array) =>
+         this.array = array;
+
+      public int Count => array.Length;
+
+      public DateTime? GetValue(int i) {
+         var nanos = array.GetValue(i);
+         if (!nanos.HasValue) {
+            return null;
+         }
+         return new(nanos.Value / 100);
+      }
+   }
+
+   class FeatherBlueRow : IBlueRow {
+      readonly int id;
+      readonly int rowIndex;
+      readonly IFeatherColumn[] data;
+
+      public FeatherBlueRow(int id, int rowIndex, IFeatherColumn[] data, IBlueTableSchema schema) {
+         this.id = id;
+         this.rowIndex = rowIndex;
+         this.data = data;
+         Schema = schema;
+      }
+
+      public object? this[int index] =>
+         index == 0
+            ? id
+            : data[index - 1].GetValue(rowIndex);
+
+      public IBlueTableSchema Schema { get; }
+      IBlueTableSchema IBlueRow.schema => Schema;
+   }
+
+   class FeatherColumnDataList : IReadOnlyList<object?> {
+      // Represents 1 column split into batches (hence it being an array)
+      readonly IReadOnlyList<IFeatherColumn> columns;
+      int? count;
+
+      public FeatherColumnDataList(IReadOnlyList<IFeatherColumn> columns) {
+         this.columns = columns;
+      }
+
+      public object? this[int index] {
+         get {
+            for (int i = 0; i < columns.Count; i++) {
+               var col = columns[i];
+               if (index < col.Count) {
+                  return col.GetValue(index);
+               }
+               index -= col.Count;
+            }
+            throw new IndexOutOfRangeException();
+         }
+      }
+
+      public int Count =>
+         count ??= columns.Sum(c => c.Count);
+
+      public IEnumerator<object?> GetEnumerator() {
+         for (int i = 0; i < columns.Count; i++) {
+            var col = columns[i];
+            for (int j = 0; j < col.Count; j++) {
+               yield return col.GetValue(i);
+            }
+         }
+      }
+
+      IEnumerator IEnumerable.GetEnumerator() =>
+         GetEnumerator();
+   }
+
+   class FeatherTable : IImmutableRowCollection, IDisposable {
+      ImmutableList<(RecordBatch Records, IFeatherColumn[] Columns)> recordBatches = ImmutableList<(RecordBatch Records, IFeatherColumn[] Columns)>.Empty;
+      readonly IReadOnlyDictionary<string, Field> fields;
+      readonly AtomicGate gate = new();
+      int count;
+      ArrowStreamReader? reader;
+
+      public FeatherTable(ArrowStreamReader reader) {
+         this.reader = reader;
+         if (reader.Schema is null) {
+            var batch = reader.ReadNextRecordBatch();
+            Log.notnull(batch);
+            Log.notnull(reader.Schema);
+            var fcols = new IFeatherColumn[batch.ColumnCount];
+            for (int i = 0; i < fcols.Length; i++) {
+               fcols[i] = FeatherColumn.Create(batch.Column(i));
+            }
+            recordBatches = recordBatches.Add((batch, fcols));
+            count = batch.Length;
+         }
+         fields = reader.Schema.Fields;
+         var columns = new ColumnInfo[fields.Count + 1];
+         columns[0] = new("id", 0, typeof(int));
+         for (int i = 0; i < fields.Count; i++) {
+            var f = reader.Schema.GetFieldByIndex(i);
+            Log.assert(f.Name != "id");
+            columns[i + 1] = new(f.Name, i + 1, getFieldType(f));
+         }
+         schema = new BlueTableSchema("FeatherReader", columns);
+      }
+
+      ~FeatherTable() =>
+         Disposable.DisposeThreadSafe(ref reader);
+
+      public IBlueTableSchema schema { get; }
+
+      public int Count {
+         get {
+            if (reader is not null) {
+               readBatches().Wait();
+               Log.isnull(reader);
+            }
+            return count;
+         }
+      }
+
+      public IBlueRow this[int id] => GetRow(id - 1);
+
+      public bool containsID(int id) {
+         while (id > count && reader is not null) {
+            readBatches(1).Wait();
+         }
+         return id <= count;
+      }
+
+      public int IndexOf(IBlueRow item) {
+         int id = item.Id();
+         while (id > count && reader is not null) {
+            readBatches(1).Wait();
+         }
+         return id <= count && BlueRowEqualityComparer.Equals(GetRow(id - 1), item)
+            ? id - 1
+            : -1;
+      }
+
+      public IBlueRow GetRow(int index) {
+         if (index < 0) {
+            throw new IndexOutOfRangeException();
+         }
+         if (count <= index) {
+            readRows(index + 1).Wait();
+         }
+         int id = index + 1;
+         foreach (var (records, columns) in recordBatches) {
+            if (index < records.Length) {
+               return new FeatherBlueRow(id, index, columns, schema);
+            }
+            index -= records.Length;
+         }
+         throw new IndexOutOfRangeException();
+      }
+
+      public IList<object?> GetColumn(int index) =>
+         new FeatherColumnDataList(new MyColList(this, index)).as_mutable();
+
+      public IEnumerator<IBlueRow> GetEnumerator() {
+         int i = 0;
+         foreach (var (records, columns) in recordBatches) {
+            yield return new FeatherBlueRow(i + 1, i, columns, schema);
+            i++;
+         }
+         while (reader is not null) {
+            var task = readBatches(1);
+            task.Wait();
+            if (!task.Result) {
+               break;
+            }
+            yield return new FeatherBlueRow(i + 1, i, recordBatches[i].Columns, schema);
+            i++;
+         }
+      }
+
+      public void Dispose() =>
+         Disposable.DisposeThreadSafe(ref reader);
+
+      static Type getFieldType(Field f) =>
+         f.DataType.TypeId switch {
+            ArrowTypeId.Null => throw new NotImplementedException(),
+            ArrowTypeId.Boolean => typeof(bool),
+            ArrowTypeId.UInt8 => typeof(byte),
+            ArrowTypeId.Int8 => typeof(sbyte),
+            ArrowTypeId.UInt16 => typeof(ushort),
+            ArrowTypeId.Int16 => typeof(short),
+            ArrowTypeId.UInt32 => typeof(uint),
+            ArrowTypeId.Int32 => typeof(int),
+            ArrowTypeId.UInt64 => typeof(ulong),
+            ArrowTypeId.Int64 => typeof(long),
+            ArrowTypeId.HalfFloat => throw new NotImplementedException(),
+            ArrowTypeId.Float => typeof(float),
+            ArrowTypeId.Double => typeof(double),
+            ArrowTypeId.String => typeof(string),
+            ArrowTypeId.Binary => throw new NotImplementedException(),
+            ArrowTypeId.FixedSizedBinary => throw new NotImplementedException(),
+            ArrowTypeId.Date32 => throw new NotImplementedException(),
+            ArrowTypeId.Date64 => throw new NotImplementedException(),
+            ArrowTypeId.Timestamp => throw new NotImplementedException(),
+            ArrowTypeId.Time32 => throw new NotImplementedException(),
+            ArrowTypeId.Time64 => throw new NotImplementedException(),
+            ArrowTypeId.Interval => throw new NotImplementedException(),
+            ArrowTypeId.Decimal128 => throw new NotImplementedException(),
+            ArrowTypeId.Decimal256 => throw new NotImplementedException(),
+            ArrowTypeId.List => throw new NotImplementedException(),
+            ArrowTypeId.Struct => throw new NotImplementedException(),
+            ArrowTypeId.Union => throw new NotImplementedException(),
+            ArrowTypeId.Dictionary => throw new NotImplementedException(),
+            ArrowTypeId.Map => throw new NotImplementedException(),
+            _ => throw new ArgumentException("Unsupported field type", nameof(f)),
+         };
+
+      async Task<bool> readRows(int minCount) {
+         if (reader is null) {
+            return count >= minCount;
+         }
+         int n = 0;
+         using (gate.BeginTransaction()) {
+            if (reader is not null) {
+               int origCount = count;
+               while ((count + n) < minCount) {
+                  Log.assert(count == origCount);
+                  var batch = await reader.ReadNextRecordBatchAsync();
+                  if (batch is null) {
+                     Disposable.dispose(ref reader);
+                     break;
+                  }
+                  var columns = new IFeatherColumn[batch.ColumnCount];
+                  for (int i = 0; i < columns.Length; i++) {
+                     columns[i] = FeatherColumn.Create(batch.Column(i));
+                  }
+                  recordBatches.Add((batch, columns));
+                  n += batch.Length;
+               }
+               count += n;
+            }
+         }
+         return count >= minCount;
+      }
+
+      async Task<bool> readBatches(int? count = null) {
+         if (reader is null) {
+            return !count.HasValue || count.Value == 0;
+         }
+         if (count.HasValue && count.Value <= recordBatches.Count) {
+            return true;
+         }
+         int n = 0;
+         using (gate.BeginTransaction()) {
+            if (reader is not null) {
+               int origCount = this.count;
+               var recordBatchesBd = recordBatches.ToBuilder();
+               while (!count.HasValue || count-- > recordBatchesBd.Count) {
+                  Log.assert(count == origCount);
+                  var batch = await reader.ReadNextRecordBatchAsync();
+                  if (batch is null) {
+                     Disposable.dispose(ref reader);
+                     break;
+                  }
+                  var columns = new IFeatherColumn[batch.ColumnCount];
+                  for (int i = 0; i < columns.Length; i++) {
+                     columns[i] = FeatherColumn.Create(batch.Column(i));
+                  }
+                  recordBatchesBd.Add((batch, columns));
+                  n += batch.Length;
+               }
+               recordBatches = recordBatchesBd.ToImmutable();
+               this.count += n;
+            }
+         }
+         return !count.HasValue || count.Value <= recordBatches.Count;
+      }
+
+      class MyColList : IReadOnlyList<IFeatherColumn> {
+         readonly FeatherTable owner;
+         readonly int columnIndex;
+
+         public MyColList(FeatherTable owner, int columnIndex) {
+            this.owner = owner;
+            this.columnIndex = columnIndex;
+         }
+
+         public IFeatherColumn this[int index] {
+            get {
+               if (index < 0) {
+                  throw new IndexOutOfRangeException();
+               }
+               if (index >= owner.recordBatches.Count) {
+                  var task = owner.readBatches(index + 1);
+                  task.Wait();
+                  if (!task.Result) {
+                     throw new IndexOutOfRangeException();
+                  }
+               }
+               return owner.recordBatches[index].Columns[columnIndex];
+            }
+         }
+
+         public int Count {
+            get {
+               if (owner.reader is not null) {
+                  owner.readBatches().Wait();
+               }
+               return owner.recordBatches.Count;
+            }
+         }
+
+         public IEnumerator<IFeatherColumn> GetEnumerator() {
+            int i = 0;
+            foreach (var batch in owner.recordBatches) {
+               yield return batch.Columns[columnIndex];
+               i++;
+            }
+            while (owner.reader is not null) {
+               var task = owner.readBatches(1);
+               task.Wait();
+               if (!task.Result) {
+                  break;
+               }
+               yield return owner.recordBatches[i].Columns[columnIndex];
+               i++;
+            }
+         }
+
+         IEnumerator IEnumerable.GetEnumerator() =>
+            GetEnumerator();
+      }
+   }
+}

src/DataStreamSuite/DataStream/BlueOrb/FilterStream.cs

@@ -1,5 +1,10 @@
 namespace bts.BlueOrb {
    using BCG;
+
+   using Bts.DataStreamCore;
+
+   using mbts.utils;
+
    using System;
    using System.Text.RegularExpressions;
 
@@ -23,7 +28,7 @@ namespace bts.BlueOrb {
          public bool filter(IBlueRow dr) {
             bool rv = true;
             for (int i = 0; i < myCols.Length; i++) {
-               rv &= myRegExps[i].IsMatch(dr[myCols[i]] is DBNull ? "" : dr[myCols[i]].ToString());
+               rv &= myRegExps[i].IsMatch(dr[myCols[i]] is DBNull ? "" : dr[myCols[i]]?.ToString() ?? string.Empty);
                if (!rv)
                   return rv;
             }
@@ -31,8 +36,8 @@ namespace bts.BlueOrb {
          }
          #endregion
       }
-      private BlueTable myLastFiredRows;
-      private IBlueRowCollection myLastReceivedRows;
+      BlueTableImmutable? myLastFiredRows;
+      IBlueRowCollection? myLastReceivedRows;
       private IStreamFilter myFilter;
       private ViewStream myParentStream;
       private string myFilterString;
@@ -52,12 +57,11 @@ namespace bts.BlueOrb {
 
 
       public FilterStream(ViewStream vs, IStreamFilter filter) {
-         {
-            myParentStream = vs;
-            myFilter = filter;
-            myFilterString = filter.ToString();
-            if (filter is IDynamicStreamFilter)
-               (filter as IDynamicStreamFilter).onFilterChange += new FilterChangeDelegate(myHandleFilterChange);
+         myParentStream = vs;
+         myFilter = filter;
+         myFilterString = filter.ToString() ?? string.Empty;
+         if (filter is IDynamicStreamFilter dynFilter) {
+            dynFilter.onFilterChange += new FilterChangeDelegate(myHandleFilterChange);
          }
       }
 
@@ -87,37 +91,28 @@ namespace bts.BlueOrb {
       }
 
       protected override void myFireEvent(object o) {
-         DataPacket dp = o as DataPacket;
+         var dp = ExceptionAssert.Argument.Is<IDataPacket>(o);
          if (myLastValue == null || dp.addedRows.Count + dp.deletedRows.Count > 0)
             base.myFireEvent(o);
       }
       #endregion
 
       private void myHandleCallbacks(object newValue) {
-         DataPacket retVal;
-         DataPacket arrivedPacket = newValue as DataPacket;
+         var arrivedPacket = ExceptionAssert.Argument.Is<IDataPacket>(newValue);
          myLastReceivedRows = arrivedPacket.newRows;
-         if (myLastFiredRows != null) {
-            retVal = new DataPacket(myLastFiredRows, new BlueTable(myLastFiredRows.schema),
-               new BlueTable(myLastFiredRows.schema));
-            foreach (IBlueRow r in arrivedPacket.deletedRows)
-               if (myLastFiredRows.containsID((int)r["id"])) {
-                  myLastFiredRows.deleteRow(r);
-                  (retVal.deletedRows as BlueTable).addRow(r);
-               }
-            foreach (IBlueRow r in arrivedPacket.addedRows)
-               if (myFilter.filter(r)) {
-                  myLastFiredRows.addRow(r);
-                  (retVal.addedRows as BlueTable).addRow(r);
-               }
-         } else {
-            myLastFiredRows = new BlueTable(arrivedPacket.schema);
-            retVal = new DataPacket(myLastFiredRows, myLastFiredRows, new BlueTable(myLastFiredRows.schema));
-            foreach (IBlueRow r in arrivedPacket.newRows)
-               if (myFilter.filter(r))
-                  myLastFiredRows.addRow(r);
+         var bd = (myLastFiredRows ?? new BlueTableImmutable(arrivedPacket.Schema)).ToDataPacketBuilder();
+         foreach (IBlueRow r in arrivedPacket.deletedRows) {
+            // Don't bother evaluating the filter, just try to remove the row. The latter is
+            // cheaper than the former (it's a noop if this row wasn't in the filtered table).
+            bd.Remove(r.Id());
          }
-         myFireEvent(retVal);
+         foreach (IBlueRow r in arrivedPacket.newRows) {
+            if (myFilter.filter(r)) {
+               bd.AddOrUpdate(r);
+            }
+         }
+         var dpOut = bd.ToDataPacket(out myLastFiredRows);
+         myFireEvent(dpOut);
       }
 
       public IStreamFilter filter {
@@ -126,15 +121,22 @@ namespace bts.BlueOrb {
                myFilter = value;
                if (numberOfSubscribers > 0) {
                   if (myLastReceivedRows != null) {
-                     BlueTable nRows = new BlueTable(myLastReceivedRows.schema);
-                     foreach (IBlueRow r in myLastReceivedRows)
-                        if (myFilter.filter(r))
-                           nRows.addRow(r);
-                     IBlueRowCollection dRows = new BlueTable(nRows.schema),
-                        aRows = new BlueTable(nRows.schema);
-                     BlueDataUtility.computeDifferences(nRows, myLastFiredRows, out dRows, out aRows);
-                     myLastFiredRows = nRows;
-                     myFireEvent(new DataPacket(nRows, aRows, dRows));
+                     var bd = (myLastFiredRows ?? new BlueTableImmutable(myLastReceivedRows.schema)).ToDataPacketBuilder();
+                     foreach (IBlueRow r in myLastReceivedRows) {
+                        int id = r.Id();
+                        bool includeNow = myFilter.filter(r);
+                        bool includeBefore = myLastFiredRows?.containsID(id) ?? false;
+                        if (includeNow == includeBefore) {
+                           continue;
+                        }
+                        if (includeNow) {
+                           bd.AddOrUpdate(r);
+                        } else {
+                           bd.Remove(id);
+                        }
+                     }
+                     var dpOut = bd.ToDataPacket(out myLastFiredRows);
+                     myFireEvent(dpOut);
                   }
                }
             }

src/DataStreamSuite/DataStream/BlueOrb/FilterTable.cs

@@ -0,0 +1,72 @@
+namespace bts.BlueOrb;
+
+using BCG;
+
+using Bts.DataStreamCore;
+
+using System;
+using System.Collections.Generic;
+
+public class FilterTable : IBlueRowCollection {
+   readonly IBlueRowCollection source;
+   readonly HashSet<int> excludedIndices = new();
+   readonly Dictionary<int, (int Index, int NumSkips, bool Included, IBlueRow Row)> rowMap = new();
+   readonly List<int> filteredIndexToId = new();
+
+   public FilterTable(IBlueRowCollection source, Func<IBlueRow, bool> filter) {
+      this.source = source;
+      int idx = 0;
+      int numSkips = 0;
+      foreach (var r in source) {
+         int id = r.Id();
+         bool included = filter(r);
+         rowMap.Add(id, (idx, numSkips, included, r));
+         if (!included) {
+            excludedIndices.Add(idx);
+            numSkips++;
+         } else {
+            filteredIndexToId.Add(id);
+         }
+         idx++;
+      }
+   }
+
+   public IBlueRow this[int id] {
+      get {
+         var r = rowMap[id];
+         return r.Included ? r.Row : throw new KeyNotFoundException();
+      }
+   }
+
+   public IBlueTableSchema schema => source.schema;
+
+   public int Count => filteredIndexToId.Count;
+
+   public bool containsID(int id) =>
+      rowMap.TryGetValue(id, out var r) && r.Included;
+
+   public IEnumerator<IBlueRow> GetEnumerator() {
+      int idx = 0;
+      foreach (var r in source) {
+         if (!excludedIndices.Contains(idx)) {
+            yield return r;
+         }
+         idx++;
+      }
+   }
+
+   public IBlueRow GetRow(int index) {
+      if (index < 0 || index >= source.Count) {
+         throw new IndexOutOfRangeException();
+      }
+      var id = filteredIndexToId[index];
+      return rowMap[id].Row;
+   }
+
+   public int IndexOf(IBlueRow item) =>
+      rowMap.TryGetValue(item.Id(), out var r)
+         && r.Included
+         && BlueRowEqualityComparer.Equals(r.Row, item)
+            ? r.Index - r.NumSkips
+            : -1;
+}

src/DataStreamSuite/DataStream/BlueOrb/GroupByApplyStream.cs

@@ -0,0 +1,222 @@
+namespace Bts.SharedStreams.StreamFactories;
+
+using BCG;
+
+using bts;
+using bts.api.core;
+using bts.BlueOrb;
+
+using Bts.DataStream;
+using Bts.DataStreamCore;
+
+using System;
+using System.Collections.Generic;
+using System.Collections.Immutable;
+using System.Linq;
+using System.Reactive.Linq;
+using System.Threading;
+
+public class GroupByApplyStream : ViewStream {
+   readonly ViewStream inputStream;
+   readonly IFunctionEvaluator groupBy;
+   readonly IFunctionEvaluator apply;
+
+   public GroupByApplyStream(ViewStream inputStream, IFunctionEvaluator groupBy, IFunctionEvaluator apply) {
+      this.inputStream = inputStream;
+      this.groupBy = groupBy;
+      this.apply = apply;
+   }
+
+   public static IObservable<TableUpdate<IBlueRow, int>> CreateGroupByApplyTable(ViewStream inputStream, IFunctionEvaluator groupBy, IFunctionEvaluator apply) =>
+      CreateApplyTable(CreateGroupByTable(inputStream, groupBy), apply);
+
+   public static IObservable<TableUpdate<IBlueRow, int>> CreateApplyTable<TGroupKey>(IObservable<TableUpdate<TableUpdateGrouping<IBlueRow, int, TGroupKey>, TGroupKey>> inputStream, Func<TGroupKey, IBlueRowCollection, IBlueRowCollection?> apply) where TGroupKey : IEquatable<TGroupKey> {
+      var groupTables = ImmutableDictionary<TGroupKey, GroupTableBuilder>.Empty;
+      var inputToOutputIds = ImmutableDictionary<int, int>.Empty;
+      var outputToInputIds = ImmutableDictionary<int, int>.Empty;
+      int idCounter = 0;
+
+      return Observable.Create<TableUpdate<IBlueRow, int>>(Subscribe);
+
+      int GetNextId() =>
+         Interlocked.Increment(ref idCounter);
+
+      IDisposable Subscribe(IObserver<TableUpdate<IBlueRow, int>> observer) {
+         int hashCounter = 0;
+         var hashIndex = new Dictionary<TGroupKey, int>();
+         var tableOut = TableUpdate<IBlueRow, int>.mk_empty(r => r.Id());
+         return inputStream.Subscribe(OnNext, OnError, OnCompleted);
+
+         void OnNext(TableUpdate<TableUpdateGrouping<IBlueRow, int, TGroupKey>, TGroupKey> upd) {
+            var bd = tableOut.to_builder();
+            foreach (var group in upd.removed) {
+               var hash = hashIndex[group.group_key];
+               bd.remove(hash);
+            }
+            var groupTablesBd = groupTables.ToBuilder();
+            foreach (var group in upd.added.Concat(upd.changed)) {
+               Log.assert(group.table.state.Count > 0);
+               var hash = hashIndex.GetOrAdd(group.group_key, _ => hashCounter++);
+               var groupRowCollection = new TableUpdateGroupingRowCollection<TGroupKey>(group, group.table.state.Values.First().schema);
+               var applyArgs = new (string?, int?, object?)[] { ("rows", 0, groupRowCollection) };
+               var resRows = apply(group.group_key, groupRowCollection);
+               if (resRows is null) {
+                  bd.remove(hash);
+                  continue;
+               }
+               UpdateGroupTable(group.group_key, resRows);
+            }
+            groupTables = groupTablesBd.ToImmutable();
+            tableOut = bd.to_table();
+            observer.OnNext(tableOut);
+
+            void UpdateGroupTable(TGroupKey groupKey, IBlueRowCollection groupRows) {
+               var indexedRows = groupTablesBd.GetOrAdd(groupKey, _ => new(GetNextId)).Reset(groupRows);
+               foreach (var r in indexedRows) {
+                  bd.add_or_update(r);
+               }
+            }
+         }
+      }
+
+      void OnError(Exception ex) {
+         Log.DataFlow.error($$"""[stream_err.json]{ "typ": "{{nameof(GroupByApplyStream)}}", "inpTyp": {{inputStream.GetType()}}, "err": "{{ex}}" }""");
+      }
+
+      void OnCompleted() {
+         Log.DataFlow.warn($$"""[stream_fin.json]{ "typ": "{{nameof(GroupByApplyStream)}}", "inpTyp": "{{inputStream.GetType()}}" }""");
+      }
+   }
+
+   public static IObservable<TableUpdate<IBlueRow, int>> CreateApplyTable<TGroupKey>(IObservable<TableUpdate<TableUpdateGrouping<IBlueRow, int, TGroupKey>, TGroupKey>> inputStream, IFunctionEvaluator apply) where TGroupKey : IEquatable<TGroupKey> {
+      var groupTables = ImmutableDictionary<TGroupKey, GroupTableBuilder>.Empty;
+      var inputToOutputIds = ImmutableDictionary<int, int>.Empty;
+      var outputToInputIds = ImmutableDictionary<int, int>.Empty;
+      int idCounter = 0;
+
+      return Observable.Create<TableUpdate<IBlueRow, int>>(Subscribe);
+
+      int GetNextId() =>
+         Interlocked.Increment(ref idCounter);
+
+      IDisposable Subscribe(IObserver<TableUpdate<IBlueRow, int>> observer) {
+         int hashCounter = 0;
+         var hashIndex = new Dictionary<TGroupKey, int>();
+         var tableOut = TableUpdate<IBlueRow, int>.mk_empty(r => r.Id());
+         return inputStream.Subscribe(OnNext, OnError, OnCompleted);
+
+         void OnNext(TableUpdate<TableUpdateGrouping<IBlueRow, int, TGroupKey>, TGroupKey> upd) {
+            var bd = tableOut.to_builder();
+            foreach (var group in upd.removed) {
+               var hash = hashIndex[group.group_key];
+               bd.remove(hash);
+            }
+            var groupTablesBd = groupTables.ToBuilder();
+            foreach (var group in upd.added.Concat(upd.changed)) {
+               Log.assert(group.table.state.Count > 0);
+               var hash = hashIndex.GetOrAdd(group.group_key, _ => hashCounter++);
+               var groupRowCollection = new TableUpdateGroupingRowCollection<TGroupKey>(group, group.table.state.Values.First().schema);
+               var applyArgs = new StreamArgument[] { ("rows", 0, groupRowCollection) };
+               var res = apply.Evaluate(string.Empty, applyArgs);
+               if (res is IBlueRow resSingleRow) {
+                  UpdateGroupTable(group.group_key, new SingleRowTable(resSingleRow));
+               } else if (res is IBlueRowCollection resRows) {
+                  UpdateGroupTable(group.group_key, resRows);
+               } else {
+                  throw new InvalidOperationException("Invalid result returned by 'Apply' fn");
+               }
+            }
+            groupTables = groupTablesBd.ToImmutable();
+            tableOut = bd.to_table();
+            observer.OnNext(tableOut);
+
+            void UpdateGroupTable(TGroupKey groupKey, IBlueRowCollection groupRows) {
+               var indexedRows = groupTablesBd.GetOrAdd(groupKey, _ => new(GetNextId)).Reset(groupRows);
+               foreach (var r in indexedRows) {
+                  bd.add_or_update(r);
+               }
+            }
+         }
+      }
+
+      void OnError(Exception ex) {
+         Log.DataFlow.error($$"""[stream_err.json]{ "typ": "{{nameof(GroupByApplyStream)}}", "inpTyp": {{inputStream.GetType()}}, "err": "{{ex}}" }""");
+      }
+
+      void OnCompleted() {
+         Log.DataFlow.warn($$"""[stream_fin.json]{ "typ": "{{nameof(GroupByApplyStream)}}", "inpTyp": "{{inputStream.GetType()}}" }""");
+      }
+   }
+
+   public static IObservable<TableUpdate<TableUpdateGrouping<IBlueRow, int, EquatableKeyWrapper>, EquatableKeyWrapper>> CreateGroupByTable(ViewStream inputStream, IFunctionEvaluator groupBy) {
+      return Observable.Create<TableUpdate<TableUpdateGrouping<IBlueRow, int, EquatableKeyWrapper>, EquatableKeyWrapper>>(Subscribe);
+
+      IDisposable Subscribe(IObserver<TableUpdate<TableUpdateGrouping<IBlueRow, int, EquatableKeyWrapper>, EquatableKeyWrapper>> observer) {
+         TableUpdate<TableUpdateGrouping<IBlueRow, int, EquatableKeyWrapper>, EquatableKeyWrapper> tableOut = TableUpdate<TableUpdateGrouping<IBlueRow, int, EquatableKeyWrapper>, EquatableKeyWrapper>.mk_empty(r => r.group_key);
+         return inputStream.AsObservable().AsTables().GroupTable(r => r.Id(), GroupKey).Subscribe(observer);
+
+         EquatableKeyWrapper GroupKey(IBlueRow r) {
+            var groupByArgs = new StreamArgument[] { ("row", 0, r) };
+            var groupKey = new EquatableKeyWrapper(groupBy.Evaluate(string.Empty, groupByArgs) ?? DBNull.Value);
+            return groupKey;
+         }
+      }
+   }
+
+   public static IObservable<TableUpdate<TableUpdateGrouping<IBlueRow, int, EquatableKeyArrayWrapper>, EquatableKeyArrayWrapper>> CreateGroupByTable(ViewStream inputStream, string[] columns, bool useColumnIndices = true) {
+      Func<IBlueRow, EquatableKeyArrayWrapper> groupBy;
+      if (!useColumnIndices) {
+         groupBy = GroupKeyByName;
+
+         EquatableKeyArrayWrapper GroupKeyByName(IBlueRow r) =>
+            new(Array.ConvertAll(columns, c => r[c]));
+      } else {
+         int[]? columnIndices = null;
+         groupBy = GroupKeyByIndex;
+
+         EquatableKeyArrayWrapper GroupKeyByIndex(IBlueRow r) {
+            columnIndices ??= r.schema.ResolveIndices(columns);
+            return new(Array.ConvertAll(columnIndices, c => r[c]));
+         }
+      }
+      return CreateGroupByTable(inputStream, groupBy);
+   }
+
+   public static IObservable<TableUpdate<TableUpdateGrouping<IBlueRow, int, TKey>, TKey>> CreateGroupByTable<TKey>(ViewStream inputStream, Func<IBlueRow, TKey> groupBy) where TKey : IEquatable<TKey> {
+      return Observable.Create<TableUpdate<TableUpdateGrouping<IBlueRow, int, TKey>, TKey>>(Subscribe);
+
+      IDisposable Subscribe(IObserver<TableUpdate<TableUpdateGrouping<IBlueRow, int, TKey>, TKey>> observer) =>
+         inputStream.AsObservable().AsTables().GroupTable(r => r.Id(), groupBy).Subscribe(observer);
+   }
+
+   class GroupTableBuilder {
+      readonly Func<int> getUniqueId;
+      ImmutableDictionary<int, int> inputToOutputIds = ImmutableDictionary<int, int>.Empty;
+      ImmutableDictionary<int, int> outputToInputIds = ImmutableDictionary<int, int>.Empty;
+      IndexTable? current;
+
+      public GroupTableBuilder(Func<int> getUniqueId) =>
+         this.getUniqueId = getUniqueId;
+
+      public IndexTable? Current => current;
+
+      public IndexTable Reset(IBlueRowCollection rows) {
+         var inputToOutputIdsBd = inputToOutputIds.ToBuilder();
+         var outputToInputIdsBd = outputToInputIds.ToBuilder();
+         foreach (var r in rows) {
+            int inputId = r.Id();
+            if (!inputToOutputIdsBd.TryGetValue(inputId, out var outputId)) {
+               outputId = getUniqueId();
+               inputToOutputIdsBd.Add(inputId, outputId);
+               outputToInputIdsBd.Add(outputId, inputId);
+            }
+            Log.assert(outputToInputIdsBd.TryGetValue(outputId, out var oinputId) && oinputId == inputId);
+            Log.assert(inputToOutputIdsBd.TryGetValue(inputId, out var ioutputId) && ioutputId == outputId);
+         }
+         inputToOutputIds = inputToOutputIdsBd.ToImmutable();
+         outputToInputIds = outputToInputIdsBd.ToImmutable();
+         current = new(rows, outputToInputIds, inputToOutputIds);
+         return current;
+      }
+   }
+}

src/DataStreamSuite/DataStream/BlueOrb/GroupByStream.cs

@@ -7,6 +7,7 @@ using System.Text.Json;
 using bts;
 //using bts.SilverPine;
 using BCG;
+using mbts.utils;
 
 namespace bts.BlueOrb {
    [DataStreamUserRespresentation("GroupByStream")]
@@ -14,41 +15,42 @@ namespace bts.BlueOrb {
 
       public interface INonIncGroupByEvaluator {
          // grouped_rows
-         IBlueRowCollection evaluate(IDictionary<int, List<IBlueRow>> grouped_rows);
-         string[] get_group_by_cols();
+         IBlueRowCollection? evaluate(IDictionary<int, List<IBlueRow>> grouped_rows);
+         string[]? get_group_by_cols();
       }
 
       private class DelNonIncGroupByEvaluator : INonIncGroupByEvaluator {
-         Func<string[]> _groupby_cols_func;
+         Func<string[]?> _groupby_cols_func;
          Func<List<IBlueRow>, List<IBlueRow>> _evaluate_func;
-         public DelNonIncGroupByEvaluator(Func<string[]> groupby_cols_func,
-                                          Func<List<IBlueRow>, List<IBlueRow>> evaluate_func) {
+         public DelNonIncGroupByEvaluator(Func<string[]?> groupby_cols_func, Func<List<IBlueRow>, List<IBlueRow>> evaluate_func) {
             _groupby_cols_func = groupby_cols_func;
             _evaluate_func = evaluate_func;
          }
-         public IBlueRowCollection evaluate(IDictionary<int, List<IBlueRow>> grouped_rows) {
-            BlueTable rv = null;
+         public IBlueRowCollection? evaluate(IDictionary<int, List<IBlueRow>> grouped_rows) {
+            BlueTable? rv = null;
             foreach (var kv in grouped_rows) {
                List<IBlueRow> g_rows = kv.Value;
                List<IBlueRow> out_rows = _evaluate_func.Invoke(g_rows);
-               if (out_rows != null && out_rows.Count > 0) {
-                  rv = rv ?? new BlueTable(out_rows.ElementAt(0).schema);
+               if (out_rows is null || out_rows.Count == 0) {
+                  continue;
                }
+               rv ??= new BlueTable(out_rows.ElementAt(0).schema);
                rv.addRows(out_rows);
             }
             return rv;
          }
-         public string[] get_group_by_cols() {
+         public string[]? get_group_by_cols() {
             return _groupby_cols_func();
          }
-
       }
-      private ViewStream _main_stream;
-      private INonIncGroupByEvaluator _groupby_func;
-      private BlueRowHasher _row_hasher;
-      private string[] _groupby_cols;
-      private Stopwatch _timer = new();
-      private int _n_total_fires = 0, _n_expensive_fires = 0;
+      readonly ViewStream? _main_stream;
+      readonly INonIncGroupByEvaluator? _groupby_func;
+      readonly string[]? _groupby_cols;
+      readonly Stopwatch _timer = new();
+
+      BlueRowHasher? _row_hasher;
+      int _n_total_fires = 0;
+      int _n_expensive_fires = 0;
 
       public GroupByStream(ViewStream main_stream, INonIncGroupByEvaluator groupby_func) {
          _main_stream = main_stream;
@@ -57,24 +59,27 @@ namespace bts.BlueOrb {
          _row_hasher = new BlueRowHasher(_groupby_cols == null ? 0 : _groupby_cols.Length);
       }
 
-      public GroupByStream(ViewStream main_stream, Func<string[]> groupby_cols_func,
-                                 Func<List<IBlueRow>, List<IBlueRow>> evaluate_func)
-                                 : this(main_stream, new DelNonIncGroupByEvaluator(groupby_cols_func, evaluate_func)) { }
+      public GroupByStream(ViewStream main_stream, Func<string[]?> groupby_cols_func, Func<List<IBlueRow>, List<IBlueRow>> evaluate_func)
+         : this(main_stream, new DelNonIncGroupByEvaluator(groupby_cols_func, evaluate_func)) { }
 
-      public GroupByStream(ViewStream parent_str, string groupby_col, string standard_group_by_text)
-      : this(parent_str, groupby_col, standard_group_by_text, String.Empty) { }
+      public GroupByStream(ViewStream parent_str, string groupby_cols, string standard_group_by_text)
+      : this(parent_str, groupby_cols, standard_group_by_text, String.Empty) { }
 
       // this is the main constructor that does everything
-      public GroupByStream(ViewStream parent_str, string groupby_col, string standard_group_by_ops, string optional_sort_col) {
+      public GroupByStream(ViewStream parent_str, string groupby_cols, string standard_group_by_ops, string optional_sort_col) {
          MultiRowOpHelper std_group_by = MultiRowOpHelper.parse_ops(standard_group_by_ops);
-         Func<string[]> groupby_cols_func = () => {
-            return String.IsNullOrEmpty(groupby_col) ? null : new string[] { groupby_col };
+         Func<string[]?> groupby_cols_func = () => {
+            if (String.IsNullOrEmpty(groupby_cols))
+               return null;
+            return groupby_cols.Split(',').Select(x => x.Trim()).ToArray();
          };
          Func<List<IBlueRow>, List<IBlueRow>> evaluate_func = (group_rows) => {
             if (!String.IsNullOrEmpty(optional_sort_col))
                group_rows.Sort((r1, r2) => BlueDataUtility.compare_rows(r2, r1, optional_sort_col));
             // this si where the group by aggregation happens
             IBlueRow rv = MultiRowOpHelper.group_by_compute(group_rows, std_group_by);
+            if (rv == null)
+               return new List<IBlueRow>();
             return new IBlueRow[] { rv }.ToList();
          };
          ViewStream rs = new GroupByStream(parent_str, groupby_cols_func, evaluate_func);
@@ -85,12 +90,12 @@ namespace bts.BlueOrb {
       }
 
 
-      protected override void myOnFirstSubscriber() { _main_stream.onChange += _main_stream_onchange; }
+      protected override void myOnFirstSubscriber() { Log.notnull(_main_stream).onChange += _main_stream_onchange; }
 
-      protected override void myOnLastSubscriber() { _main_stream.onChange -= _main_stream_onchange; }
+      protected override void myOnLastSubscriber() { Log.notnull(_main_stream).onChange -= _main_stream_onchange; }
 
       private void _main_stream_onchange(object nv) {
-         DataPacket dp = nv as DataPacket;
+         var dp = ExceptionAssert.Argument.Is<IDataPacket>(nv);
          ///grouping together
          _timer.Restart();
          IDictionary<int, List<IBlueRow>> grouped_rows = (_groupby_cols == null || _groupby_cols.Length == 0) ?
@@ -104,7 +109,10 @@ namespace bts.BlueOrb {
          // Log.log(bts.Log.Level.Info, bts.log.Subsystem.gen, String.Format("Time to group {0} rows into {1} groups = {2}", dp.newRows.Count, grouped_rows.Count, _timer.lastIntervalTime));
          /// reducing
          _timer.Restart();
-         IBlueRowCollection total_state = _groupby_func.evaluate(grouped_rows);
+         var total_state = Log.notnull(_groupby_func).evaluate(grouped_rows);
+         if (total_state == null)
+            // mayeb returna n empty table ?
+            return;
          _timer.Stop();
          var applyElapsedMs = _timer.ElapsedMilliseconds;
          _n_total_fires++;
@@ -113,9 +121,7 @@ namespace bts.BlueOrb {
             Log.DataFlow.info(
                $"Time to evaluate {grouped_rows.Count} groups {_timer.ElapsedMilliseconds}ms, ratio {_n_expensive_fires}/{_n_total_fires}");
          }
-         // TODO,
-         if (total_state != null)
-            myFireEvent(new DataPacket(total_state, total_state, myLastValue == null ? new BlueTable(total_state.schema) : (myLastValue as DataPacket).newRows));
+         myFireEvent(new DataPacket(total_state, total_state, myLastValue == null ? new BlueTable(total_state.schema) : (myLastValue as DataPacket).newRows));
       }
    }
 }

src/DataStreamSuite/DataStream/BlueOrb/GroupByTable.cs

@@ -0,0 +1,83 @@
+namespace bts.BlueOrb;
+
+using BCG;
+
+using Bts.DataStreamCore;
+
+using System;
+using System.Collections.Generic;
+using System.Linq;
+
+public class GroupByTable<TKey> : IBlueRowCollection {
+   readonly IBlueTableSchema outputSchema;
+   readonly IBlueRowCollection inputRows;
+   readonly List<IBlueRow> rows;
+   readonly Dictionary<int, int> idToIndex;
+
+   public GroupByTable(IBlueRowCollection inputRows, Func<IBlueRow, TKey> getKey, IEqualityComparer<TKey> keyEqualityComparer, IComparer<TKey> keyComparer, string? groupKeyCol = "group") {
+      this.inputRows = inputRows;
+      var cols = inputRows.schema.columnInfo.ToList();
+      rows = new(inputRows.Count);
+      int groupKeyColIdx = -1;
+      if (!string.IsNullOrEmpty(groupKeyCol)) {
+         groupKeyColIdx = cols.Count;
+         cols.Add(new(groupKeyCol, groupKeyColIdx, typeof(TKey)));
+      }
+      for (int i = 0; i < cols.Count; i++) {
+         Log.assert(cols[i].colNo == i);
+      }
+      outputSchema = new BlueTableSchema(cols.ToArray(), "GroupBy");
+      rows.AddRange(inputRows
+         .GroupBy(getKey, keyEqualityComparer)
+         .OrderBy(g => g.Key, keyComparer)
+         .SelectMany(rg => rg.Select(r => new Row(rg.Key, r, outputSchema, groupKeyColIdx))));
+      idToIndex = rows.Select((r, i) => (r, i)).ToDictionary(t => t.r.Id(), t => t.i);
+   }
+
+   public IBlueRow this[int id] =>
+      rows[idToIndex[id]];
+
+   public IBlueTableSchema schema => outputSchema;
+
+   public int Count => rows.Count;
+
+   public bool containsID(int id) =>
+      idToIndex.ContainsKey(id);
+
+   public IEnumerator<IBlueRow> GetEnumerator() =>
+      rows.GetEnumerator();
+
+   public IBlueRow GetRow(int index) =>
+      rows[index];
+
+   public int IndexOf(IBlueRow item) =>
+      idToIndex.TryGetValue(item.Id(), out var idx) && BlueRowEqualityComparer.Equals(item, rows[idx])
+         ? idx
+         : -1;
+
+   public class Row : IBlueRow {
+      readonly IBlueRow inputRow;
+      readonly IBlueTableSchema outputSchema;
+      readonly int keyColIdx;
+
+      public Row(TKey key, IBlueRow inputRow, IBlueTableSchema outputSchema, int keyColIdx = -1) {
+         Key = key;
+         this.inputRow = inputRow;
+         this.outputSchema = outputSchema;
+         this.keyColIdx = keyColIdx;
+      }
+
+      public object? this[int index] {
+         get {
+            if (index == keyColIdx) {
+               return Key;
+            }
+            return inputRow[index];
+         }
+      }
+
+      public TKey Key { get; }
+
+      public IBlueTableSchema schema => outputSchema;
+   }
+}

src/DataStreamSuite/DataStream/BlueOrb/ImmutableOrderedTable.cs

@@ -0,0 +1,229 @@
+namespace bts.BlueOrb;
+
+using BCG;
+
+using Bts.DataStreamCore;
+
+using Spark.Core.Collections;
+
+using System;
+using System.Collections.Generic;
+using System.Collections.Immutable;
+using System.Linq;
+
+public class ImmutableOrderedTable : IBlueRowCollection {
+   readonly IBlueTableSchema schema;
+   readonly ImmutableSortedSet<KeyValuePair<long, IBlueRow>> rows;
+   readonly IReadOnlyDictionary<long, IBlueRow> ordinalToRow;
+   readonly ImmutableDictionary<int, long> idToOrdinal;
+   long newRowOrdinal = 0;
+
+   public ImmutableOrderedTable(IBlueTableSchema schema)
+      : this(
+         schema,
+         ImmutableSortedSet.Create(KeyValuePairComparer.KeyComparer<long, IBlueRow>()),
+         ImmutableDictionary<int, long>.Empty, 0) { }
+
+   ImmutableOrderedTable(IBlueTableSchema schema, ImmutableSortedSet<KeyValuePair<long, IBlueRow>> rows, ImmutableDictionary<int, long> idToOrdinal, long newRowOrdinal) {
+      this.schema = schema;
+      this.rows = rows;
+      this.idToOrdinal = idToOrdinal;
+      this.newRowOrdinal = newRowOrdinal;
+      ordinalToRow = ImmutableSortedSetDictionary.Create(rows);
+   }
+
+   public static OrderedTable FromRowCollection(IBlueRowCollection rowCollection) {
+      var rowList = new List<IBlueRow>(rowCollection.Count);
+      var idToIndex = new Dictionary<int, int>();
+      foreach (var r in rowCollection) {
+         idToIndex[r.Id()] = rowList.Count;
+         rowList.Add(r);
+      }
+      return new(rowCollection.schema, rowList, idToIndex);
+   }
+
+   public IBlueTableSchema Schema => schema;
+
+   public int Count => rows.Count;
+
+   public IBlueRow this[int id] =>
+      idToOrdinal.TryGetValue(id, out var ord) ? ordinalToRow[ord] : throw new KeyNotFoundException();
+
+   IBlueTableSchema IBlueRowCollection.schema => schema;
+
+   public bool containsID(int id) =>
+      idToOrdinal.ContainsKey(id);
+
+   public IEnumerator<IBlueRow> GetEnumerator() {
+      foreach (var (_, row) in rows) {
+         yield return row;
+      }
+   }
+
+   public IBlueRow GetRow(int index) =>
+      rows[index].Value;
+
+   public int IndexOf(IBlueRow item) =>
+      idToOrdinal.TryGetValue(item.Id(), out var ord) ? rows.IndexOf(new(ord, null!)) : -1;
+
+   public Builder ToBuilder() =>
+      new(schema, rows, idToOrdinal, newRowOrdinal);
+
+   public class Builder : IBlueRowCollection {
+      readonly IBlueTableSchema schema;
+
+      ImmutableSortedSet<KeyValuePair<long, IBlueRow>> originalRows;
+      ImmutableSortedSet<KeyValuePair<long, IBlueRow>>.Builder rows;
+      ImmutableDictionary<int, long> originalIdToOrdinal;
+      ImmutableDictionary<int, long>.Builder idToOrdinal;
+      IReadOnlyDictionary<long, IBlueRow> originalOrdinalToRow;
+      IReadOnlyDictionary<long, IBlueRow> ordinalToRow;
+
+      readonly HashSet<int> removes = new();
+      readonly HashSet<int> adds = new();
+      long newRowOrdinal;
+
+      public Builder(IBlueTableSchema schema, ImmutableSortedSet<KeyValuePair<long, IBlueRow>> rows, ImmutableDictionary<int, long> idToOrdinal, long newRowOrdinal) {
+         this.schema = schema;
+         originalRows = rows;
+         this.rows = rows.ToBuilder();
+         originalIdToOrdinal = idToOrdinal;
+         this.idToOrdinal = idToOrdinal.ToBuilder();
+         originalOrdinalToRow = ImmutableSortedSetDictionary.Create(originalRows);
+         ordinalToRow = ImmutableSortedSetDictionary.CreateBuilder(this.rows);
+         this.newRowOrdinal = newRowOrdinal;
+      }
+
+      public IBlueRow this[int id] {
+         get {
+            if (idToOrdinal.TryGetValue(id, out var ord) && !removes.Contains(id)) {
+               return ordinalToRow[ord];
+            }
+            throw new KeyNotFoundException();
+         }
+      }
+
+      public int Count => rows.Count + adds.Count - removes.Count;
+
+      IBlueTableSchema IBlueRowCollection.schema => schema;
+
+      public void Add(IBlueRow row) =>
+         addOrUpdate(row, false);
+
+      public void AddOrUpdate(IBlueRow row) =>
+         addOrUpdate(row, true);
+
+      public bool containsID(int id) =>
+         (idToOrdinal.ContainsKey(id) && !removes.Contains(id)) || adds.Contains(id);
+
+      public IEnumerator<IBlueRow> GetEnumerator() {
+         foreach (var row in rows) {
+            if (!removes.Contains(row.Value.Id())) {
+               yield return row.Value;
+            }
+         }
+      }
+
+      public IBlueRow GetRow(int index) =>
+         rows[index].Value;
+
+      public int IndexOf(IBlueRow item) {
+         if (idToOrdinal.TryGetValue(item.Id(), out var ord)) {
+            return rows.IndexOf(new(ord, item));
+         }
+         return -1;
+      }
+
+      public bool Remove(int id) {
+         if (!idToOrdinal.TryGetValue(id, out var ord)) {
+            return false;
+         }
+         idToOrdinal.Remove(id);
+         Log.assert(rows.Remove(new(ord, null!)));
+         Log.assert(ordinalToRow.Count == rows.Count);
+         adds.Remove(id);
+         if (originalIdToOrdinal.ContainsKey(id)) {
+            removes.Add(id);
+         }
+         return true;
+      }
+
+      bool ICollection<IBlueRow>.Remove(IBlueRow item) {
+         int id = item.Id();
+         if (!idToOrdinal.TryGetValue(id, out var ord) || !BlueRowEqualityComparer.Equals(item, ordinalToRow[ord])) {
+            return false;
+         }
+         rows.Remove(new(ord, null!));
+         adds.Remove(id);
+         if (originalIdToOrdinal.ContainsKey(id)) {
+            removes.Add(id);
+         }
+         return true;
+      }
+
+      public ImmutableOrderedTable ToImmutable(out IDataPacket dp) {
+         var newRows = new ImmutableOrderedTable(schema, rows.ToImmutable(), idToOrdinal.ToImmutable(), newRowOrdinal);
+         dp = new DataPacket(
+            newRows,
+            new BlueTableListWrapper(schema, adds.Select(id => ordinalToRow[idToOrdinal[id]]).ToList()),
+            new BlueTableListWrapper(schema, removes.Select(id => originalOrdinalToRow[originalIdToOrdinal[id]]).ToList()));
+         return newRows;
+      }
+
+      void addOrUpdate(IBlueRow row, bool canUpdate) {
+         if (newRowOrdinal == long.MaxValue) {
+            resetOrdinal();
+         }
+         int id = row.Id();
+         if (idToOrdinal.TryGetValue(id, out var ord)) {
+            if (!canUpdate) {
+               throw new ArgumentException("Row id already present", nameof(row));
+            }
+            rows.Remove(new(ord, row));
+         }
+         ord = newRowOrdinal++;
+         idToOrdinal[id] = ord;
+         if (originalIdToOrdinal.ContainsKey(id)) {
+            removes.Add(id);
+         }
+         rows.Add(new(ord, row));
+         adds.Add(id);
+      }
+
+      void resetOrdinal() {
+         Log.DataFlow.warn($$"""[immutable_ordered_table_reset_ordinal.json]{ "tableName": "{{schema.tableName}}", "origNumRows": {{originalRows.Count}}, "numRows": {{rows.Count}}, "ts": {{DateTime.Now}} }""");
+         var originalRowsBd = originalRows.Clear().ToBuilder();
+         var originalIdToOrdinalBd = originalIdToOrdinal.Clear().ToBuilder();
+         var newRowsBd = ImmutableSortedSet.CreateBuilder(rows.KeyComparer);
+         var newIdToOrdinalBd = ImmutableDictionary.CreateBuilder<int, long>(idToOrdinal.KeyComparer);
+         long ord = 0;
+         long? prevRowOrd = null;
+         foreach (var kv in originalRows) {
+            if (prevRowOrd.HasValue) {
+               Log.assert(prevRowOrd.Value < kv.Key);
+            }
+            prevRowOrd = kv.Key;
+            originalRowsBd.Add(new(ord, kv.Value));
+            originalIdToOrdinalBd.Add(new(kv.Value.Id(), ord));
+            ord++;
+         }
+         prevRowOrd = null;
+         foreach (var kv in rows) {
+            if (prevRowOrd.HasValue) {
+               Log.assert(prevRowOrd.Value < kv.Key);
+            }
+            prevRowOrd = kv.Key;
+            newRowsBd.Add(new(ord, kv.Value));
+            newIdToOrdinalBd.Add(new(kv.Value.Id(), ord));
+            ord++;
+         }
+         rows = newRowsBd;
+         originalRows = originalRowsBd.ToImmutable();
+         originalIdToOrdinal = originalIdToOrdinalBd.ToImmutable();
+         idToOrdinal = newIdToOrdinalBd;
+         originalOrdinalToRow = ImmutableSortedSetDictionary.Create(originalRows);
+         ordinalToRow = ImmutableSortedSetDictionary.CreateBuilder(rows);
+         newRowOrdinal = ord;
+      }
+   }
+}

src/DataStreamSuite/DataStream/BlueOrb/IndexRow.cs

@@ -0,0 +1,23 @@
+namespace bts.BlueOrb;
+
+using BCG;
+
+public class IndexRow : IBlueRow {
+   readonly IBlueRow inputRow;
+   readonly int id;
+   readonly int idIndex;
+
+   public IndexRow(IBlueRow inputRow, int id, int idIndex, IBlueTableSchema schema) {
+      this.inputRow = inputRow;
+      this.id = id;
+      this.idIndex = idIndex;
+      this.schema = schema;
+   }
+
+   public IBlueRow InputRow => inputRow;
+
+   public IBlueTableSchema schema { get; }
+
+   public object? this[int index] =>
+      index == idIndex ? id : inputRow[index];
+}

src/DataStreamSuite/DataStream/BlueOrb/IndexStream.cs

@@ -0,0 +1,53 @@
+namespace bts.BlueOrb;
+
+using BCG;
+
+using Bts.DataStream;
+using Bts.DataStreamCore;
+
+using Spark.Core.Disposables;
+
+using System;
+using System.Collections.Immutable;
+
+public class IndexStream : ViewStream {
+   readonly ViewStream inputStream;
+
+   ImmutableDictionary<int, int> inputToOutputIds = ImmutableDictionary<int, int>.Empty;
+   ImmutableDictionary<int, int> outputToInputIds = ImmutableDictionary<int, int>.Empty;
+   int idCounter = 0;
+   IDisposable? sub;
+
+   public IndexStream(ViewStream inputStream) =>
+      this.inputStream = inputStream;
+
+   ~IndexStream() =>
+      Disposable.dispose(ref sub);
+
+   protected override void myOnFirstSubscriber() {
+      base.myOnFirstSubscriber();
+      sub = inputStream.Subscribe(OnInputChanged);
+
+      void OnInputChanged(IDataPacket dp) {
+         var inputToOutputIdsBd = inputToOutputIds.ToBuilder();
+         var outputToInputIdsBd = outputToInputIds.ToBuilder();
+         foreach (var r in dp.addedRows) {
+            int inputId = r.Id();
+            if (!inputToOutputIds.TryGetValue(inputId, out int outputId)) {
+               outputId = ++idCounter;
+               inputToOutputIdsBd[inputId] = outputId;
+               outputToInputIdsBd[outputId] = inputId;
+            }
+         }
+         inputToOutputIds = inputToOutputIdsBd.ToImmutable();
+         outputToInputIds = outputToInputIdsBd.ToImmutable();
+         myFireEvent(new DataPacket(
+            new IndexTable(dp.newRows, outputToInputIds, inputToOutputIds),
+            new IndexTable(dp.addedRows, outputToInputIds, inputToOutputIds),
+            new IndexTable(dp.deletedRows, outputToInputIds, inputToOutputIds)));
+      }
+   }
+
+   protected override void myOnLastSubscriber() =>
+      Disposable.dispose(ref sub);
+}

src/DataStreamSuite/DataStream/BlueOrb/IndexTable.cs

@@ -0,0 +1,93 @@
+namespace bts.BlueOrb;
+
+using BCG;
+
+using Bts.DataStreamCore;
+
+using Spark.Core.Collections;
+
+using System;
+using System.Collections.Generic;
+using System.Collections.Immutable;
+
+public class IndexTable : IBlueRowCollection {
+   readonly IBlueRowCollection inputRows;
+   readonly IDictionary<int, int> outputToInputIdMap;
+   readonly IDictionary<int, int> inputToOutputIdMap;
+   readonly int idColIndex;
+
+   public IndexTable(IBlueRowCollection inputRows, IDictionary<int, int> outputToInputIdMap, IDictionary<int, int> inputToOutputIdMap) {
+      this.inputRows = inputRows;
+      this.outputToInputIdMap = outputToInputIdMap;
+      this.inputToOutputIdMap = inputToOutputIdMap;
+      if (!inputRows.schema.columnInfo.TryGetFirst(c => c.colName == "id", out var idColInfo)) {
+         throw new ArgumentException("No 'id' column", nameof(inputRows));
+      }
+      idColIndex = idColInfo.colNo;
+   }
+
+   public IBlueTableSchema schema => inputRows.schema;
+
+   public int Count => inputRows.Count;
+
+   public IBlueRow this[int id] =>
+      outputToInputIdMap.TryGetValue(id, out var inputId)
+         ? new IndexRow(inputRows[inputId], id, idColIndex, schema)
+         : throw new KeyNotFoundException();
+
+   IBlueRow IList<IBlueRow>.this[int index] {
+      get => GetRow(index);
+      set => throw new NotSupportedException();
+   }
+
+   public static IndexTable Create(IBlueRowCollection inputRows) {
+      var outputToInputIdMap = new Dictionary<int, int>();
+      var inputToOutputIdMap = new Dictionary<int, int>();
+      foreach (var r in inputRows) {
+         int inputId = r.Id();
+         int outputId = outputToInputIdMap.Count + 1;
+         outputToInputIdMap[outputId] = inputId;
+         inputToOutputIdMap[inputId] = outputId;
+      }
+      return new(inputRows, outputToInputIdMap, inputToOutputIdMap);
+   }
+
+   public static BlueTableImmutable CreateImmutable(IBlueTableSchema schema, IEnumerable<IBlueRow> rows) {
+      int idIndex = schema.ResolveIndices("id");
+      var rowSetBd = ImmutableSortedSet.Create(KeyValuePairComparer.KeyComparer<int, IBlueRow>()).ToBuilder();
+      foreach (var r in rows) {
+         int idx = rowSetBd.Count + 1;
+         var inputRow = (r as IndexRow)?.InputRow ?? r;
+         rowSetBd.Add(new(idx, new IndexRow(inputRow, idx, idIndex, schema)));
+      }
+      return new(schema, rowSetBd.ToImmutable());
+   }
+
+   public bool containsID(int id) =>
+      inputRows.containsID(id);
+
+   public IEnumerator<IBlueRow> GetEnumerator() {
+      foreach (var r in inputRows) {
+         yield return new IndexRow(r, inputToOutputIdMap[r.Id()], idColIndex, schema);
+      }
+   }
+
+   public IBlueRow GetRow(int index) {
+      var r = inputRows.GetRow(index);
+      return new IndexRow(r, inputToOutputIdMap[r.Id()], idColIndex, schema);
+   }
+
+   public int IndexOf(IBlueRow item) {
+      int id = item.Id();
+      if (outputToInputIdMap.TryGetValue(id, out var inputId)) {
+         var inputRow = inputRows[inputId];
+         var r = new IndexRow(inputRow, id, idColIndex, schema);
+         if (BlueRowEqualityComparer.Equals(r, item)) {
+            var idx = inputRows.IndexOf(inputRow);
+            Log.assert(idx >= 0);
+            return idx;
+         }
+      }
+      return -1;
+   }
+}

src/DataStreamSuite/DataStream/BlueOrb/Join/SimpleJoinStream.cs

@@ -20,9 +20,9 @@
          private IBlueTableSchema _schema;
          private int[] _join_cols;
          private IDictionary<RowHash, List<IBlueRow>> _hash_row_map = new Dictionary<RowHash, List<IBlueRow>>();
-         
+
          public JoinBookkeepingData(ViewStream vs, List<string> join_cols, bool is_left, BlueRowHasher row_hasher) {
-            _join_col_names = join_cols.Where (c => !String.IsNullOrEmpty(c)).ToList ();
+            _join_col_names = join_cols.Where(c => !String.IsNullOrEmpty(c)).ToList();
             _is_left = is_left;
             _hasher = row_hasher;
          }
@@ -31,23 +31,23 @@
 
          public IDictionary<RowHash, List<IBlueRow>> rows => _hash_row_map;
 
-         public List<IBlueRow> get_matching_rows (int hash) {
+         public List<IBlueRow> get_matching_rows(int hash) {
             List<IBlueRow> rv = null;
             _hash_row_map.TryGetValue(hash, out rv);
             return rv;
          }
 
          private RowHash _get_hash(IBlueRow r) {
-            if (_join_cols == null || _join_cols.Length == 0) { 
+            if (_join_cols == null || _join_cols.Length == 0) {
                return 0;
             }
             return _hasher.get_hash(r, _join_cols);
          }
 
          public void update_data(DataPacket dp) {
-            if (!ready) {               
+            if (!ready) {
                _schema = dp.newRows.schema;
-               _join_cols = _join_col_names.Count == 0 ? new int[] { }: _join_col_names.Select(n => dp.schema[n].colNo).ToArray();
+               _join_cols = _join_col_names.Count == 0 ? new int[] { } : _join_col_names.Select(n => dp.schema[n].colNo).ToArray();
             }
             _hash_row_map.Clear();
             foreach (IBlueRow r in dp.newRows) {
@@ -82,7 +82,7 @@
          _rs = rs;
          List<string> lj_col_list = ljc.Split(',').Where(x => !String.IsNullOrEmpty(x)).ToList();
          List<string> rj_col_list = rjc.Split(',').Where(x => !String.IsNullOrEmpty(x)).ToList();
-         Log.assert(lj_col_list.Count ==  rj_col_list.Count);
+         Log.assert(lj_col_list.Count == rj_col_list.Count);
          int n_cols = lj_col_list.Count;
          BlueRowHasher hasher = new BlueRowHasher(n_cols);
          _ldata = new JoinBookkeepingData(ls, lj_col_list, true, hasher);
@@ -101,7 +101,7 @@
       }
 
       private void _rs_onchange(object dp_obj) {
-         DataPacket dp = dp_obj as DataPacket ;
+         DataPacket dp = dp_obj as DataPacket;
          _rdata.update_data(dp);
          _form_and_fire();
       }
@@ -128,11 +128,11 @@
       }
 
       private void _form_and_fire() {
-         if (!_ldata.ready||!_rdata.ready)
+         if (!_ldata.ready || !_rdata.ready)
             return;
          _build_schema();
          List<object[]> res_rows = new List<object[]>();
-         
+
          Action<object[], IBlueRow, bool> copy_row = (rval, r, is_left) => {
             int offset = 1;
             JoinBookkeepingData sdata = _ldata;
@@ -148,7 +148,7 @@
             }
          };
          Func<int, IBlueRow, object[]> build_row_array = (id, l_row) => {
-            object[] rv = new object[_schema.columnNames.Length];            
+            object[] rv = new object[_schema.columnNames.Length];
             for (int i = 0; i < _schema.columnNames.Length; i++) {
                rv[i] = DBNull.Value;
             }
@@ -162,7 +162,7 @@
             List<IBlueRow> l_rows = kv.Value;
             List<IBlueRow> r_rows = _rdata.get_matching_rows(hash);
 
-            foreach (IBlueRow lr in l_rows) {                              
+            foreach (IBlueRow lr in l_rows) {
                if (r_rows != null) {
                   foreach (IBlueRow r in r_rows) {
                      object[] rowvals = build_row_array(res_rows.Count, lr);
@@ -182,6 +182,15 @@
          var dp = BlueDataUtility.build_dp(rv, myLastValue as DataPacket);
          myFireEvent(dp);
       }
-
    }
+
+   [DataStreamUserRespresentation("FullJoinStream")]
+   public class FullJoinStream : ViewStream {
+      public FullJoinStream(ViewStream l_str, string ljcs, ViewStream r_str, string rjcs) {
+         ViewStream js = new SimpleJoinStream(l_str, ljcs, r_str, rjcs);
+         ViewStream rs = new bts.BlueOrb.FilterStream(js, r => !DBNull.Value.Equals(r["right_id"]));
+         mySetRealSource(rs);
+      }
+   }
+
 }

src/DataStreamSuite/DataStream/BlueOrb/MeanTable.cs

@@ -0,0 +1,96 @@
+namespace bts.BlueOrb;
+
+using BCG;
+
+using bts.collections;
+
+using Bts.DataStreamCore;
+
+using System;
+using System.Collections.Generic;
+using System.Linq;
+
+public class MeanTable : IBlueRowCollection {
+   readonly IBlueRowCollection input;
+   readonly IList<object?>[] columns;
+
+   public MeanTable(IBlueRowCollection input) {
+      this.input = input;
+      var colInfos = input.schema.columnInfo.OrderBy(c => c.colNo).ToArray();
+      if (colInfos.Length > 0) {
+         Log.assert(colInfos[0].colNo == 0 && colInfos[^1].colNo == colInfos.Length - 1);
+      }
+      columns = colInfos.WrapSelector(c => GetColumn(c.colNo), null).ToArray();
+      for (int i = 0; i < colInfos.Length; i++) {
+         if (colInfos[i].colName == "id") {
+            continue;
+         }
+         var ci = colInfos[i];
+         switch (ci.type_code()) {
+            case TypeCode.Int16:
+            case TypeCode.Int32:
+            case TypeCode.Int64:
+            case TypeCode.UInt16:
+            case TypeCode.UInt32:
+            case TypeCode.UInt64:
+            case TypeCode.Single:
+               colInfos[i] = new(ci.colName, ci.colNo, typeof(double));
+               break;
+         }
+      }
+      schema = new BlueTableSchema(colInfos, input.schema.tableName + "_Mean");
+   }
+
+   public IBlueRow this[int id] =>
+      id == 1 ? GetRow(0) : throw new KeyNotFoundException();
+
+   public IBlueTableSchema schema { get; }
+
+   public int Count => 1;
+
+   public bool containsID(int id) =>
+      id == 1;
+
+   public IList<object?> GetColumn(int colIndex) {
+      var count = input.Count;
+      var colInfo = input.schema.columnInfo[colIndex];
+      if (colInfo.colName == "id") {
+         return new object?[] { 1 };
+      }
+      var typeCode = colInfo.type_code();
+      var colVals = input.GetColumn(colIndex);
+      object? res = typeCode switch {
+         TypeCode.Int16 => colVals.Select(x => x is short v ? (double?)v : null).SelectWhereValues().Average(),
+         TypeCode.Int32 => colVals.Select(x => x is int v ? (double?)v : null).SelectWhereValues().Average(),
+         TypeCode.Int64 => colVals.Select(x => x is long v ? (double?)v : null).SelectWhereValues().Average(),
+         TypeCode.UInt16 => colVals.Select(x => x is ushort v ? (double?)v : null).SelectWhereValues().Average(),
+         TypeCode.UInt32 => colVals.Select(x => x is uint v ? (double?)v : null).SelectWhereValues().Average(),
+         TypeCode.UInt64 => colVals.Select(x => x is ulong v ? (double?)v : null).SelectWhereValues().Average(),
+         TypeCode.Single => colVals.Select(x => x is float v ? (double?)v : null).SelectWhereValues().Average(),
+         TypeCode.Double => colVals.Select(x => x as double?).SelectWhereValues().Average(),
+         TypeCode.DateTime => new DateTime((long)colVals.Cast<DateTime>().Select(dt => (double)dt.Ticks).Average()),
+         _ => null,
+      };
+      return new object?[] { res };
+   }
+
+   public IEnumerator<IBlueRow> GetEnumerator() {
+      yield return GetRow(0);
+   }
+
+   public IBlueRow GetRow(int index) =>
+      new BlueRowColumnWrapper(input.schema, index, columns);
+
+   public int IndexOf(IBlueRow item) {
+      int id = item.Id();
+      var inputRow = input[id];
+      int idx = input.IndexOf(inputRow);
+      if (idx != -1) {
+         var row = GetRow(idx);
+         if (BlueRowEqualityComparer.Equals(item, row)) {
+            return idx;
+         }
+      }
+      return -1;
+   }
+}

src/DataStreamSuite/DataStream/BlueOrb/Merge/MergeAsofStream.cs

@@ -0,0 +1,50 @@
+namespace bts.BlueOrb;
+
+using BCG;
+
+using Bts.DataStream;
+using Bts.DataStreamCore;
+
+using System.Reactive.Linq;
+
+public class MergeAsofStream : ViewStream {
+   public MergeAsofStream(ViewStream left, int[] leftJoinIndices, ViewStream right, int[] rightJoinIndices, MergeAsofTable.MergeType mergeType = MergeAsofTable.MergeType.LessOrEqual) {
+      mySetRealSource(
+         left.AsObservable()
+         .CombineLatest(right.AsObservable(), GetMergeTable)
+         .Scan((IDataPacket)null!, Update)
+         .AsViewStream());
+
+      MergeAsofTable GetMergeTable(IDataPacket l, IDataPacket r) =>
+         new(l.newRows, leftJoinIndices, r.newRows, rightJoinIndices, mergeType);
+
+      IDataPacket Update(IDataPacket? lastDp, MergeAsofTable table) {
+         if (lastDp is null) {
+            return new DataPacket(table, table, new BlueTable(table.Schema));
+         }
+         BlueDataUtility.computeDifferences(table, lastDp.newRows, out var removes, out var adds);
+         return new DataPacket(table, adds, removes);
+      }
+   }
+
+   public MergeAsofStream(ViewStream left, string[] leftJoinNames, ViewStream right, string[] rightJoinNames, MergeAsofTable.MergeType mergeType = MergeAsofTable.MergeType.LessOrEqual) {
+      int[]? leftJoinIndices = null;
+      int[]? rightJoinIndices = null;
+      mySetRealSource(
+         left.AsObservable()
+         .CombineLatest(right.AsObservable(), GetMergeTable)
+         .Scan((IDataPacket)null!, Update)
+         .AsViewStream());
+
+      MergeAsofTable GetMergeTable(IDataPacket l, IDataPacket r) =>
+         new(l.newRows, leftJoinIndices ??= l.Schema.ResolveIndices(leftJoinNames), r.newRows, rightJoinIndices ??= r.Schema.ResolveIndices(rightJoinNames), mergeType);
+
+      IDataPacket Update(IDataPacket? lastDp, MergeAsofTable table) {
+         if (lastDp is null) {
+            return new DataPacket(table, table, new BlueTable(table.Schema));
+         }
+         BlueDataUtility.computeDifferences(table, lastDp.newRows, out var removes, out var adds);
+         return new DataPacket(table, adds, removes);
+      }
+   }
+}

src/DataStreamSuite/DataStream/BlueOrb/Merge/MergeAsofTable.cs

@@ -0,0 +1,236 @@
+namespace bts.BlueOrb;
+
+using BCG;
+
+using bts.collections;
+
+using Bts.DataStreamCore;
+
+using mbts.utils;
+
+using System;
+using System.Collections.Generic;
+using System.Collections.Immutable;
+using System.Linq;
+
+public class MergeAsofTable : IBlueRowCollection {
+   const int ID_COLUMN_SENTINEL = int.MinValue;
+   readonly ImmutableDictionary<int, int> rowMap;
+   readonly ImmutableList<MergeRow> rowList;
+   readonly ImmutableList<(IBlueRow Row, bool IsLeft)> mergedRows;
+   readonly IComparer<(IBlueRow Row, bool IsLeft)> mergedComparer;
+   readonly IBlueTableSchema schema;
+   readonly MergeType mergeType;
+
+   public MergeAsofTable(IBlueRowCollection left, int[] leftJoinIndices, IBlueRowCollection right, int[] rightJoinIndices, MergeType mergeType = MergeType.LessOrEqual) {
+      var leftSchema = left.schema;
+      var rightSchema = right.schema;
+      var leftCols = leftSchema.columnInfo;
+      var rightCols = rightSchema.columnInfo;
+      var colInfos = left.schema.columnInfo.OrderBy(c => c.colNo).ToList();
+      this.mergeType = mergeType;
+      if (colInfos.Count > 0) {
+         Log.assert(colInfos[0].colNo == 0 && colInfos[^1].colNo == colInfos.Count - 1);
+      }
+      foreach (var c in rightCols.OrderBy(c => c.colNo)) {
+         if (colInfos.Any(c => c.colName == c.colName)) {
+            continue;
+         }
+         colInfos.Add(new(c.colName, colInfos.Count, c.colType));
+      }
+      schema = buildSchema(leftCols, rightCols, out var mergedColIndices);
+      rowList = buildRowList(left, leftJoinIndices, right, rightJoinIndices, schema, mergedColIndices, mergeType, out mergedRows, out mergedComparer);
+      rowMap = Enumerable.Range(0, rowList.Count).ToImmutableDictionary(i => rowList[i].Id());
+   }
+
+   public enum MergeType {
+      Less,
+      LessOrEqual,
+      Greater,
+      GreaterOrEqual
+   }
+
+   public IBlueRow this[int id] =>
+      rowList[rowMap[id]];
+
+   IBlueRow IList<IBlueRow>.this[int index] {
+      get => GetRow(index);
+      set => throw new NotSupportedException();
+   }
+
+   public IBlueTableSchema Schema => schema;
+
+   IBlueTableSchema IBlueRowCollection.schema => schema;
+
+   public int Count => rowList.Count;
+
+   public bool containsID(int id) =>
+      rowMap.ContainsKey(id);
+
+   public IEnumerator<IBlueRow> GetEnumerator() =>
+      rowList.GetEnumerator();
+
+   public IBlueRow GetRow(int index) =>
+      rowList[index];
+
+   public int IndexOf(IBlueRow item) {
+      int id = item.Id();
+      return rowMap.TryGetValue(id, out var idx) && BlueRowEqualityComparer.Equals(item, rowList[idx])
+         ? idx
+         : -1;
+   }
+
+   static ImmutableList<MergeRow> buildRowList(
+      IBlueRowCollection left,
+      int[] leftJoinIndices,
+      IBlueRowCollection right,
+      int[] rightJoinIndices,
+      IBlueTableSchema schema,
+      int[] mergedColIndices,
+      MergeType mergeType,
+      out ImmutableList<(IBlueRow Row, bool IsLeft)> mergedRows,
+      out IComparer<(IBlueRow Row, bool IsLeft)> mergedComparer
+   ) {
+      var leftCols = left.schema.columnInfo;
+      var rightCols = right.schema.columnInfo;
+      var comp = buildComparer(leftCols, leftJoinIndices, rightCols, rightJoinIndices);
+      mergedComparer = comp;
+      var sortedRows = ImmutableList.CreateBuilder<(IBlueRow Row, bool IsLeft)>();
+      foreach (var r in left) {
+         sortedRows.InsertSorted((r, true), comp);
+      }
+      foreach (var r in right) {
+         sortedRows.InsertSorted((r, false), comp);
+      }
+      mergedRows = sortedRows.ToImmutable();
+      return mergeType switch {
+         MergeType.LessOrEqual => LessOrEqualImpl(),
+         _ => throw new NotImplementedException(),
+      };
+
+      ImmutableList<MergeRow> LessOrEqualImpl() {
+         var rowList = ImmutableList.CreateBuilder<MergeRow>();
+         IBlueRow? currLeft = null;
+         IBlueRow? currRight = null;
+         foreach (var (row, isLeft) in sortedRows) {
+            object?[] groupKeys;
+            if (isLeft) {
+               currLeft = row;
+               groupKeys = Array.ConvertAll(leftJoinIndices, i => row[i]);
+            } else {
+               currRight = row;
+               groupKeys = Array.ConvertAll(rightJoinIndices, i => row[i]);
+            }
+            rowList.Add(new(groupKeys, currLeft, currRight, schema, mergedColIndices, 0, rowList.Count + 1));
+         }
+         return rowList.ToImmutable();
+      }
+   }
+
+   static IBlueTableSchema buildSchema(ColumnInfo[] leftCols, ColumnInfo[] rightCols, out int[] mergedColIndices) {
+      var colInfos = new List<ColumnInfo>() {
+         new("id", 0, typeof(int))
+      };
+      var mergedColIndexList = new List<int>();
+      int colNo = 0;
+      foreach (var lc in leftCols) {
+         Log.assert(lc.colNo == colNo);
+         colNo++;
+         if (lc.colName == "id") {
+            continue;
+         }
+         colInfos.Add(new(lc.colName, colInfos.Count, lc.colType));
+         mergedColIndexList.Add(lc.colNo);
+      }
+      colNo = 0;
+      foreach (var rc in rightCols) {
+         Log.assert(rc.colNo == colNo);
+         colNo++;
+         if (colInfos.Any(c => c.colName == rc.colName)) {
+            continue;
+         }
+         colInfos.Add(new(rc.colName, colInfos.Count, rc.colType));
+         mergedColIndexList.Add(-rc.colNo);
+      }
+      mergedColIndices = mergedColIndexList.ToArray();
+      return new BlueTableSchema(colInfos.ToArray(), "MergeAsof");
+   }
+
+   static IComparer<(IBlueRow Row, bool IsLeft)> buildComparer(ColumnInfo[] leftCols, int[] leftJoinIndices, ColumnInfo[] rightCols, int[] rightJoinIndices) {
+      var compBd = new ComparerBuilder<(IBlueRow Row, bool IsLeft)>();
+      for (int i = 0; i < leftJoinIndices.Length; i++) {
+         var li = leftJoinIndices[i];
+         var ri = rightJoinIndices[i];
+         var lc = leftCols[li];
+         var rc = rightCols[ri];
+         Log.assert(lc.colNo == li && rc.colNo == ri);
+         var colType = lc.colType;
+         if (!lc.colType.IsAssignableFrom(rc.colType)) {
+            Log.assert(rc.colType.IsAssignableFrom(lc.colType));
+            colType = rc.colType;
+         }
+         if (colType == typeof(DateTime)) {
+            compBd.Add(t => t.Row.GetDateTime(t.IsLeft ? li : ri));
+         } else if (colType == typeof(TimeSpan)) {
+            compBd.Add(t => t.Row.GetTimeSpan(t.IsLeft ? li : ri));
+         } else if (colType == typeof(short)) {
+            compBd.Add(t => t.Row.GetInt16(t.IsLeft ? li : ri));
+         } else if (colType == typeof(int)) {
+            compBd.Add(t => t.Row.GetInt32(t.IsLeft ? li : ri));
+         } else if (colType == typeof(long)) {
+            compBd.Add(t => t.Row.GetInt64(t.IsLeft ? li : ri));
+         } else if (colType == typeof(ushort)) {
+            compBd.Add(t => t.Row.GetUInt16(t.IsLeft ? li : ri));
+         } else if (colType == typeof(uint)) {
+            compBd.Add(t => t.Row.GetUInt32(t.IsLeft ? li : ri));
+         } else if (colType == typeof(ulong)) {
+            compBd.Add(t => t.Row.GetUInt64(t.IsLeft ? li : ri));
+         } else if (colType == typeof(double)) {
+            compBd.Add(t => t.Row.GetFloat64(t.IsLeft ? li : ri));
+         }
+      }
+      return compBd.ToDualComparer();
+   }
+
+   // public class Builder {
+   //    readonly ImmutableDictionary<int, int> rowMap;
+   //    readonly ImmutableList<MergeRow> rowList;
+   //    readonly ImmutableList<(IBlueRow Row, bool IsLeft)> mergedRows;
+   //    readonly IComparer<(IBlueRow Row, bool IsLeft)> mergedComparer;
+   //    readonly IBlueTableSchema schema;
+   //    readonly MergeType mergeType;
+   //    readonly Dictionary<int, IBlueRow> leftAdds = new();
+   //    readonly HashSet<int> leftRemoves = new();
+   //    readonly Dictionary<int, IBlueRow> rightAdds = new();
+   //    readonly HashSet<int> rightRemoves = new();
+
+   //    public Builder(ImmutableDictionary<int, int> rowMap, ImmutableList<MergeRow> rowList, ImmutableList<(IBlueRow Row, bool IsLeft)> mergedRows, IComparer<(IBlueRow Row, bool IsLeft)> mergedComparer, IBlueTableSchema schema, MergeType mergeType) {
+   //       this.rowMap = rowMap;
+   //       this.rowList = rowList;
+   //       this.mergedRows = mergedRows;
+   //       this.mergedComparer = mergedComparer;
+   //       this.schema = schema;
+   //       this.mergeType = mergeType;
+   //    }
+
+   //    public void AddOrUpdateLeft(IBlueRow row) {
+   //       leftAdds[row.Id()] = row;
+   //    }
+
+   //    public void AddOrUpdateRight(IBlueRow row) {
+   //       rightAdds[row.Id()] = row;
+   //    }
+
+   //    public void RemoveLeft(int id) {
+   //       leftRemoves.Add(id);
+   //    }
+
+   //    public void RemoveRight(int id) {
+   //       rightRemoves.Add(id);
+   //    }
+
+   //    public MergeAsofTable ToImmutable(out ImmutableList<MergeRow> removes, out ImmutableList<MergeRow> adds) {
+
+   //    }
+   // }
+}

src/DataStreamSuite/DataStream/BlueOrb/Merge/MergeRow.cs

@@ -0,0 +1,51 @@
+namespace bts.BlueOrb;
+
+using BCG;
+
+using System;
+
+public class MergeRow : IBlueRow {
+   readonly IBlueRow? left;
+   readonly IBlueRow? right;
+   readonly IBlueTableSchema schema;
+   readonly int[] colIndices;
+   readonly int idColIndex;
+   readonly int id;
+
+   public MergeRow(object?[] groupKeys, IBlueRow? left, IBlueRow? right, IBlueTableSchema schema, int[] colIndices, int idColIndex, int id) {
+      GroupKeys = groupKeys;
+      this.left = left;
+      this.right = right;
+      this.schema = schema;
+      this.colIndices = colIndices;
+      this.idColIndex = idColIndex;
+      this.id = id;
+   }
+
+   public object?[] GroupKeys { get; }
+   public IBlueRow? Left => left;
+   public IBlueRow? Right => right;
+
+   public object? this[int index] {
+      get {
+         // The colIndices array doesn't include the 'id' column, so index can
+         // be equal to colIndices.Length.
+         if (index < 0 || index > colIndices.Length) {
+            throw new IndexOutOfRangeException();
+         }
+         if (index == idColIndex) {
+            return id;
+         }
+         var x = colIndices[index > idColIndex ? index -1 : index];
+         return x < 0 ? right?[-x] : left?[x];
+      }
+   }
+
+   IBlueTableSchema IBlueRow.schema => schema;
+
+   public MergeRow WithLeft(IBlueRow? left) =>
+      new(GroupKeys, left, right, schema, colIndices, idColIndex, id);
+
+   public MergeRow WithRight(IBlueRow? right) =>
+      new(GroupKeys, left, right, schema, colIndices, idColIndex, id);
+}

src/DataStreamSuite/DataStream/BlueOrb/MillisecondStream.cs

@@ -43,7 +43,7 @@ namespace bts.BlueOrb {
             _last_fire_time = DateTime.Now;
             IBlueRow nr = new BlueRow(_schema, _id++, _last_fire_time,
             _last_fire_time.ToString("yyyy-MM-dd HH:mm:ss.fff"));
-            _table.add_row(nr);
+            _table = _table.Add(nr);
             var dp = BlueDataUtility.build_dp_inc(_table, myLastValue as DataPacket);
             myFireEvent(dp);
          }

src/DataStreamSuite/DataStream/BlueOrb/MultiRowOpHelper.cs

@@ -1,6 +1,8 @@
 namespace bts.BlueOrb {
    using BCG;
 
+   using bts.collections;
+
    using System;
    using System.Collections.Generic;
    using System.Linq;
@@ -73,30 +75,20 @@ namespace bts.BlueOrb {
       }
 
       // aggregates multiple rows into one, used by MultiRowRollingMathStream
-      public static IBlueRow group_by_compute(List<IBlueRow> rows, int start_idx, int n_rows, MultiRowOpHelper parameters) {
-         if (rows.Count() == 0) {
-            return null;
-         }
-         IBlueRow ir = rows.ElementAt(start_idx);
-         IBlueTableSchema in_schema = ir.schema;
-         IBlueTableSchema out_schema = in_schema;
-         // out_col vs inp_cal + type
-         IDictionary<string, Tuple<string, MultiRowOp>> out_col_map =
-            new Dictionary<string, Tuple<string, MultiRowOp>>();
-         foreach (var t in parameters.operations) {
-            string out_col_name = $"{t.Item1}_{t.Item2}".ToLower();
-            out_schema = out_schema.add_column(TypeCode.Double, out_col_name);
-            out_col_map[out_col_name] = t;
-         }
-         MutableBlueRow or = new MutableBlueRow(out_schema, ir);
+      public static IBlueRow group_by_compute(
+         IBlueRow row,
+         int start_idx,
+         int n_rows,
+         IBlueTableSchema out_schema,
+         IDictionary<string, (MultiRowOp, IList<double>)> out_col_map) {
+         MutableBlueRow or = new MutableBlueRow(out_schema, row);
          foreach (var kv in out_col_map) {
             string out_col_name = kv.Key;
-            string in_col_name = kv.Value.Item1;
-            MultiRowOp op = kv.Value.Item2;
-            IEnumerable<double> col_vals = BlueDataUtility.get_colvalues<double>(rows, start_idx, n_rows, in_col_name);
+            MultiRowOp op = kv.Value.Item1;
+            var col_vals = kv.Value.Item2.WrapSubRange(start_idx, Math.Min(n_rows, kv.Value.Item2.Count - start_idx));
             double comp_res = compute_op(col_vals, op);
             or.setValue(out_col_name, comp_res);
-            or.setValue("id", ir["id"]);
+            or.setValue("id", row["id"]);
          }
          return or;
       }
@@ -150,6 +142,14 @@ namespace bts.BlueOrb {
                return col_vals.Last();
             case MultiRowOp.StdDev:
                return MathNet.Numerics.Statistics.Statistics.StandardDeviation(col_vals);
+            case MultiRowOp.EdgeDiff:
+               return col_vals.First() - col_vals.Last();
+            case MultiRowOp.EdgeDiffPercentage: {
+                  double first = col_vals.First();
+                  double last = col_vals.Last();
+                  return ((first - last) / last) * 100.0;
+                  return col_vals.First() - col_vals.Last();
+               }
             default:
                // TODO
                Log.assert(false);

src/DataStreamSuite/DataStream/BlueOrb/MultiRowRollingMathStream.cs

@@ -1,8 +1,12 @@
 namespace bts.BlueOrb {
    using BCG;
 
+   using bts.collections;
+
+   using mbts.utils;
+
+   using System;
    using System.Collections.Generic;
-   using System.Net;
 
    [DataStreamUserRespresentation("MultiRowRollingMathStream")]
    public class MultiRowRollingMathStream : ViewStream {
@@ -24,11 +28,19 @@
       }
 
       private void _parent_str_onchange(object dp_obj) {
-         DataPacket dp = dp_obj as DataPacket;
-         var rows = dp.newRows.to_list();
-         BlueTable out_table = null;
-         for (int i = 0; i < rows.Count; i++) {
-            var out_r = MultiRowOpHelper.group_by_compute(rows, i, _rolling_window_sz, _op);
+         var dp = ExceptionAssert.Argument.Is<IDataPacket>(dp_obj);
+         BlueTable? out_table = null;
+         IBlueTableSchema out_schema = dp.newRows.schema;
+         // out_col vs inp_cal + type
+         var out_col_map = new Dictionary<string, (MultiRowOp, IList<double>)>();
+         foreach (var t in _op.operations) {
+            string out_col_name = $"{t.Item1}_{t.Item2}".ToLower();
+            out_schema = out_schema.add_column(TypeCode.Double, out_col_name);
+            out_col_map[out_col_name] = (t.Item2, dp.newRows.GetColumn(t.Item1).WrapSelector(o => (double)o, null));
+         }
+         int i = 0;
+         foreach (IBlueRow row in dp.newRows) {
+            var out_r = MultiRowOpHelper.group_by_compute(row, i++, _rolling_window_sz, out_schema, out_col_map);
             if (out_table == null)
                out_table = new BlueTable(out_r.schema);
             out_table.addRow(out_r);

src/DataStreamSuite/DataStream/BlueOrb/OrderedTable.cs

@@ -0,0 +1,49 @@
+namespace bts.BlueOrb;
+
+using BCG;
+
+using Bts.DataStreamCore;
+
+using System.Collections.Generic;
+
+public class OrderedTable : IBlueRowCollection {
+   readonly IBlueTableSchema schema;
+   readonly List<IBlueRow> rows;
+   readonly Dictionary<int, int> idToIndex;
+
+   public OrderedTable(IBlueTableSchema schema, List<IBlueRow> rows, Dictionary<int, int> idToIndex) {
+      this.schema = schema;
+      this.rows = rows;
+      this.idToIndex = idToIndex;
+   }
+
+   public static OrderedTable FromRowCollection(IBlueRowCollection rowCollection) {
+      var rowList = new List<IBlueRow>(rowCollection.Count);
+      var idToIndex = new Dictionary<int, int>();
+      foreach (var r in rowCollection) {
+         idToIndex[r.Id()] = rowList.Count;
+         rowList.Add(r);
+      }
+      return new(rowCollection.schema, rowList, idToIndex);
+   }
+
+   public IBlueTableSchema Schema => schema;
+
+   public int Count => rows.Count;
+
+   public IBlueRow this[int id] => idToIndex.TryGetValue(id, out var idx) ? rows[idx] : throw new KeyNotFoundException();
+
+   IBlueTableSchema IBlueRowCollection.schema => schema;
+
+   public bool containsID(int id) =>
+      idToIndex.ContainsKey(id);
+
+   public IEnumerator<IBlueRow> GetEnumerator() =>
+      rows.GetEnumerator();
+
+   public IBlueRow GetRow(int index) =>
+      rows[index];
+
+   public int IndexOf(IBlueRow item) =>
+      idToIndex.TryGetValue(item.Id(), out var idx) ? idx : -1;
+}

src/DataStreamSuite/DataStream/BlueOrb/Parquet/IParquetColumnReader.cs

@@ -0,0 +1,32 @@
+namespace bts.BlueOrb.Parquet;
+
+using bts.collections;
+
+using Spark.Core;
+
+using System;
+using System.Collections.Generic;
+
+public interface IParquetColumnReader : IDisposable {
+   IList<object?> Read();
+   IList<object?> Read(int? start, int? stop);
+   void ReadTo(Variant[] dest, int start, int count);
+}
+
+public interface IParquetColumnReader<TLogical, TPhysical> : IParquetColumnReader {
+   new IList<TLogical?> Read();
+   new IList<TLogical?> Read(int? start, int? stop);
+
+   IList<object?> IParquetColumnReader.Read() =>
+      Read().WrapSelector(v => (object?)v, null);
+
+   IList<object?> IParquetColumnReader.Read(int? start, int? stop) =>
+      Read(start, stop).WrapSelector(v => (object?)v, null);
+}
+
+public interface IParquetColumnReader<T> : IParquetColumnReader<T, T> {
+   new IList<T?> Read();
+
+   IList<object?> IParquetColumnReader.Read() =>
+      Read().WrapSelector(v => (object?)v, null);
+}

src/DataStreamSuite/DataStream/BlueOrb/Parquet/ParquetAdapter.cs

@@ -0,0 +1,506 @@
+namespace bts.BlueOrb.Parquet;
+
+using bts.collections;
+
+using mbts.utils;
+
+using ParquetSharp;
+
+using Spark.Core;
+
+using System;
+using System.Collections.Generic;
+
+public static class ParquetAdapter {
+   public static IParquetColumnReader GetColumnReader(this RowGroupReader rowGroupReader, int index, Type? colType = null) {
+      int rowCount = checked((int)rowGroupReader.MetaData.NumRows);
+      var colReader = rowGroupReader.Column(index);
+      return getColumnReader(colReader, rowCount, colType: colType);
+   }
+
+   public static IList<object?>[] ReadColumns(this ParquetFileReader fileReader) {
+      var schema = fileReader.FileMetaData.Schema.ToBlueSchema("parquetReader");
+      var colInfos = schema.columnInfo;
+      var colTypes = Array.ConvertAll(colInfos, c => c.colType);
+      var colValues = new IList<object?>[fileReader.FileMetaData.NumColumns];
+      int rowCount = checked((int)fileReader.FileMetaData.NumRows);
+      for (int i = 0; i < colValues.Length; i++) {
+         colValues[i] = new List<object?>(rowCount);
+      }
+      int rowGroupCount = fileReader.FileMetaData.NumRowGroups;
+      for (int rowGroup = 0; rowGroup < rowGroupCount; ++rowGroup) {
+         using var rowGroupReader = fileReader.RowGroup(rowGroup);
+         rowGroupReader.ReadColumnsTo(ref colValues, colTypes);
+      }
+      return colValues;
+   }
+
+   public static IList<object?>[] ReadColumns(this ParquetFileReader fileReader, int? start, int? stop) {
+      if (!start.HasValue && !stop.HasValue) {
+         return ReadColumns(fileReader);
+      }
+      var schema = fileReader.FileMetaData.ToBlueSchema("parquetReader");
+      var colInfos = schema.columnInfo;
+      var colTypes = Array.ConvertAll(colInfos, c => c.colType);
+      var colValues = new IList<object?>[fileReader.FileMetaData.NumColumns];
+      int rowCount = checked((int)fileReader.FileMetaData.NumRows);
+      start ??= 0;
+      stop ??= rowCount;
+      var sliceLength = stop.Value - start.Value;
+      for (int i = 0; i < colValues.Length; i++) {
+         colValues[i] = new List<object?>(sliceLength);
+      }
+      int numRowGroups = fileReader.FileMetaData.NumRowGroups;
+      int? numRowsPerGroup = null;
+      int rowIdx = 0;
+      for (int rowGroup = 0; rowGroup < numRowGroups; ++rowGroup) {
+         if (stop.Value <= rowIdx) {
+            break;
+         }
+         using var rowGroupReader = fileReader.RowGroup(rowGroup);
+         int numRowsInGroup = checked((int)rowGroupReader.MetaData.NumRows);
+         numRowsPerGroup ??= numRowsInGroup;
+         if (rowGroup < numRowGroups - 1) {
+            Log.assert(numRowsInGroup == numRowsPerGroup.Value);
+         } else {
+            Log.assert(numRowsInGroup <= numRowsPerGroup.Value);
+         }
+         int startIdx = start.Value - rowIdx;
+         int stopIdx = stop.Value - rowIdx;
+         rowIdx += numRowsInGroup;
+         if (startIdx >= numRowsInGroup) {
+            continue;
+         }
+         startIdx = Math.Max(0, startIdx);
+         stopIdx = Math.Min(stopIdx, numRowsInGroup);
+         int n = rowGroupReader.ReadColumnsTo(ref colValues, colTypes, startIdx, stopIdx);
+         Log.assert(n == (stopIdx - startIdx));
+      }
+      return colValues;
+   }
+
+   public static int ReadColumnsTo(this RowGroupReader rowGroupReader, ref IList<object?>[] columnValues, Type[] colTypes) {
+      int rowCount = checked((int)rowGroupReader.MetaData.NumRows);
+      for (int colIdx = rowGroupReader.MetaData.NumColumns - 1; colIdx > -1; colIdx--) {
+         using var colReader = rowGroupReader.GetColumnReader(colIdx, colTypes[colIdx]);
+         columnValues[colIdx].AddRange(colReader.Read());
+      }
+      return rowCount;
+   }
+
+   public static int ReadColumnsTo(this RowGroupReader rowGroupReader, ref IList<object?>[] columnValues, Type[] colTypes, int? start, int? stop) {
+      if (!start.HasValue && !stop.HasValue) {
+         return ReadColumnsTo(rowGroupReader, ref columnValues, colTypes);
+      }
+      int numRows = checked((int)rowGroupReader.MetaData.NumRows);
+      if (start.HasValue && start.Value >= numRows) {
+         return 0;
+      }
+      int? readRowCount = null;
+      for (int colIdx = rowGroupReader.MetaData.NumColumns - 1; colIdx > -1; colIdx--) {
+         using var colReader = rowGroupReader.GetColumnReader(colIdx, colTypes[colIdx]);
+         var colVals = colReader.Read(start, stop);
+         readRowCount ??= colVals.Count;
+         Log.assert(readRowCount == colVals.Count);
+         columnValues[colIdx].AddRange(colVals);
+      }
+      return readRowCount.assert_value();
+   }
+
+   static IParquetColumnReader getColumnReader(ColumnReader reader, int bufferSize = 4096, Type? colType = null) {
+      var desc = reader.ColumnDescriptor;
+      var logicalType = desc.LogicalType;
+      return logicalType.Type switch {
+         LogicalTypeEnum.Int => IntReader(),
+         LogicalTypeEnum.String => new ParquetColumnReader<string?>(reader, bufferSize),
+         LogicalTypeEnum.Date => new ParquetColumnReader<DateTime?>(reader, bufferSize),
+         LogicalTypeEnum.Decimal => DecimalReader(),
+         LogicalTypeEnum.Timestamp => TimestampReader(),
+         LogicalTypeEnum.None => DefaultReader(),
+         _ => throw new NotImplementedException(),
+      };
+
+      IParquetColumnReader DefaultReader() {
+         return desc.PhysicalType switch {
+            PhysicalType.Boolean => new ParquetColumnReader<bool?>(reader, bufferSize),
+            PhysicalType.Int32 => new ParquetColumnReader<int?>(reader, bufferSize),
+            PhysicalType.Int64 => new ParquetColumnReader<long?>(reader, bufferSize),
+            PhysicalType.Float => new ParquetColumnReader<float?>(reader, bufferSize),
+            PhysicalType.Double => new ParquetColumnReader<double?>(reader, bufferSize),
+            _ => throw new NotImplementedException(),
+         };
+      }
+
+      IParquetColumnReader TimestampReader() {
+         var tlt = ExceptionAssert.of_type<TimestampLogicalType>(logicalType);
+         return new ParquetColumnReader<DateTimeNanos?, DateTime?>(reader, dtn => dtn?.DateTime, bufferSize);
+      }
+
+      IParquetColumnReader IntReader() {
+         var ilt = ExceptionAssert.of_type<IntLogicalType>(logicalType);
+         if (!ilt.IsSigned) {
+            if (colType is null) {
+               return ilt.BitWidth switch {
+                  8 => new ParquetColumnReader<byte?>(reader, bufferSize),
+                  16 => new ParquetColumnReader<ushort?>(reader, bufferSize),
+                  32 => new ParquetColumnReader<uint?>(reader, bufferSize),
+                  64 => new ParquetColumnReader<ulong?>(reader, bufferSize),
+                  _ => throw new NotImplementedException(),
+               };
+            } else if (colType == typeof(byte)) {
+               return ilt.BitWidth switch {
+                  8 => new ParquetColumnReader<byte?>(reader, bufferSize),
+                  16 => new ParquetColumnReader<ushort?, byte?>(reader, v => v.HasValue ? checked((byte)v.Value) : null, bufferSize),
+                  32 => new ParquetColumnReader<uint?, ushort?>(reader, v => v.HasValue ? checked((byte)v.Value) : null, bufferSize),
+                  64 => new ParquetColumnReader<ulong?, ushort?>(reader, v => v.HasValue ? checked((byte)v.Value) : null, bufferSize),
+                  _ => throw new NotImplementedException(),
+               };
+            } else if (colType == typeof(ushort)) {
+               return ilt.BitWidth switch {
+                  8 => new ParquetColumnReader<byte?, ushort?>(reader, v => v.HasValue ? v.Value : null, bufferSize),
+                  16 => new ParquetColumnReader<ushort?>(reader, bufferSize),
+                  32 => new ParquetColumnReader<uint?, ushort?>(reader, v => v.HasValue ? checked((ushort)v.Value) : null, bufferSize),
+                  64 => new ParquetColumnReader<ulong?, ushort?>(reader, v => v.HasValue ? checked((ushort)v.Value) : null, bufferSize),
+                  _ => throw new NotImplementedException(),
+               };
+            } else if (colType == typeof(uint)) {
+               return ilt.BitWidth switch {
+                  8 => new ParquetColumnReader<byte?, uint?>(reader, v => v.HasValue ? v.Value : null, bufferSize),
+                  16 => new ParquetColumnReader<ushort?, uint?>(reader, v => v.HasValue ? v.Value : null, bufferSize),
+                  32 => new ParquetColumnReader<uint?>(reader, bufferSize),
+                  64 => new ParquetColumnReader<ulong?, uint?>(reader, v => v.HasValue ? checked((uint)v.Value) : null, bufferSize),
+                  _ => throw new NotImplementedException(),
+               };
+            } else if (colType == typeof(ulong)) {
+               return ilt.BitWidth switch {
+                  8 => new ParquetColumnReader<byte?, ulong?>(reader, v => v.HasValue ? v.Value : null, bufferSize),
+                  16 => new ParquetColumnReader<ushort?, ulong?>(reader, v => v.HasValue ? v.Value : null, bufferSize),
+                  32 => new ParquetColumnReader<uint?, ulong?>(reader, v => v.HasValue ? v.Value : null, bufferSize),
+                  64 => new ParquetColumnReader<ulong?>(reader, bufferSize),
+                  _ => throw new NotImplementedException(),
+               };
+            }
+            throw new NotImplementedException($"Expected signed int column type, but got {colType}");
+         } else {
+            if (colType is null) {
+               return ilt.BitWidth switch {
+                  8 => new ParquetColumnReader<sbyte?>(reader, bufferSize),
+                  16 => new ParquetColumnReader<short?>(reader, bufferSize),
+                  32 => new ParquetColumnReader<int?>(reader, bufferSize),
+                  64 => new ParquetColumnReader<long?>(reader, bufferSize),
+                  _ => throw new NotImplementedException(),
+               };
+            } else if (colType == typeof(sbyte)) {
+               return ilt.BitWidth switch {
+                  8 => new ParquetColumnReader<sbyte?>(reader, bufferSize),
+                  16 => new ParquetColumnReader<short?, sbyte?>(reader, v => v.HasValue ? checked((sbyte)v.Value) : null, bufferSize),
+                  32 => new ParquetColumnReader<int?, sbyte?>(reader, v => v.HasValue ? checked((sbyte)v.Value) : null, bufferSize),
+                  64 => new ParquetColumnReader<long?, sbyte?>(reader, v => v.HasValue ? checked((sbyte)v.Value) : null, bufferSize),
+                  _ => throw new NotImplementedException(),
+               };
+            } else if (colType == typeof(short)) {
+               return ilt.BitWidth switch {
+                  8 => new ParquetColumnReader<char?, short?>(reader, v => v.HasValue ? checked((short)v.Value) : null, bufferSize),
+                  16 => new ParquetColumnReader<short?>(reader, bufferSize),
+                  32 => new ParquetColumnReader<int?, short?>(reader, v => v.HasValue ? checked((short)v.Value) : null, bufferSize),
+                  64 => new ParquetColumnReader<long?, short?>(reader, v => v.HasValue ? checked((short)v.Value) : null, bufferSize),
+                  _ => throw new NotImplementedException(),
+               };
+            } else if (colType == typeof(int)) {
+               return ilt.BitWidth switch {
+                  8 => new ParquetColumnReader<char?, int?>(reader, v => v.HasValue ? v.Value : null, bufferSize),
+                  16 => new ParquetColumnReader<short?, int?>(reader, v => v.HasValue ? v.Value : null, bufferSize),
+                  32 => new ParquetColumnReader<int?>(reader, bufferSize),
+                  64 => new ParquetColumnReader<long?, int?>(reader, v => v.HasValue ? checked((int)v.Value) : null, bufferSize),
+                  _ => throw new NotImplementedException(),
+               };
+            } else if (colType == typeof(long)) {
+               return ilt.BitWidth switch {
+                  8 => new ParquetColumnReader<char?, long?>(reader, v => v.HasValue ? v.Value : null, bufferSize),
+                  16 => new ParquetColumnReader<short?, long?>(reader, v => v.HasValue ? v.Value : null, bufferSize),
+                  32 => new ParquetColumnReader<int?, long?>(reader, v => v.HasValue ? v.Value : null, bufferSize),
+                  64 => new ParquetColumnReader<long?>(reader, bufferSize),
+                  _ => throw new NotImplementedException(),
+               };
+            }
+            throw new NotImplementedException($"Expected unsigned int column type, but got {colType}");
+         }
+      }
+
+      IParquetColumnReader DecimalReader() {
+         var dlt = ExceptionAssert.of_type<DecimalLogicalType>(logicalType);
+         return new ParquetColumnReader<double?>(reader, bufferSize);
+      }
+   }
+
+   public static IColumnWriter GetColumnWriter(TypeCode typeCode, ColumnWriter writer, int bufferSize = 4096) {
+      return typeCode switch {
+         TypeCode.Boolean => new ColumnWriter<bool?>(writer, bufferSize),
+         TypeCode.SByte => new ColumnWriter<sbyte?>(writer, bufferSize, GetInt8Converter((IntLogicalType)writer.ColumnDescriptor.LogicalType)),
+         TypeCode.Int16 => new ColumnWriter<short?>(writer, bufferSize, GetInt16Converter((IntLogicalType)writer.ColumnDescriptor.LogicalType)),
+         TypeCode.Int32 => new ColumnWriter<int?>(writer, bufferSize, GetInt32Converter((IntLogicalType)writer.ColumnDescriptor.LogicalType)),
+         TypeCode.Int64 => new ColumnWriter<long?>(writer, bufferSize, GetInt64Converter((IntLogicalType)writer.ColumnDescriptor.LogicalType)),
+         TypeCode.Byte => new ColumnWriter<byte?>(writer, bufferSize, GetUInt8Converter((IntLogicalType)writer.ColumnDescriptor.LogicalType)),
+         TypeCode.UInt16 => new ColumnWriter<ushort?>(writer, bufferSize, GetUInt16Converter((IntLogicalType)writer.ColumnDescriptor.LogicalType)),
+         TypeCode.UInt32 => new ColumnWriter<uint?>(writer, bufferSize, GetUInt32Converter((IntLogicalType)writer.ColumnDescriptor.LogicalType)),
+         TypeCode.UInt64 => new ColumnWriter<ulong?>(writer, bufferSize, GetUInt64Converter((IntLogicalType)writer.ColumnDescriptor.LogicalType)),
+         TypeCode.Single => new ColumnWriter<float?>(writer, bufferSize),
+         TypeCode.Double => new ColumnWriter<double?>(writer, bufferSize),
+         TypeCode.DateTime => new ColumnWriter<DateTimeNanos?>(writer, bufferSize, ConvertDateTimeObj),
+         TypeCode.String => new ColumnWriter<string?>(writer, bufferSize),
+         _ => throw new NotSupportedException($"Unsupported column TypeCode {typeCode}"),
+      };
+
+      static DateTimeNanos? ConvertDateTimeObj(object? obj) {
+         if (obj is DateTime dt) {
+            return new(dt);
+         }
+         if (obj is DateTimeNanos dtn) {
+            return dtn;
+         }
+         throw new InvalidCastException($"Unable to convert object of type {obj?.GetType()} to DateTimeNanos");
+      }
+   }
+
+   public static Func<object?, sbyte?> GetInt8Converter(IntLogicalType logicalType) =>
+      GetInt8Converter(logicalType.IsSigned, logicalType.BitWidth);
+
+   public static Func<object?, sbyte?> GetInt8Converter(bool isSigned, int bitWidth) =>
+      isSigned
+         ? bitWidth switch {
+            8 => obj => obj as sbyte?,
+            16 => obj => obj is short v ? checked((sbyte)v) : null,
+            32 => obj => obj is int v ? checked((sbyte)v) : null,
+            64 => obj => obj is long v ? checked((sbyte)v) : null,
+            _ => throw new NotImplementedException(),
+         }
+         : bitWidth switch {
+            8 => obj => obj is byte v ? checked((sbyte)v) : null,
+            16 => obj => obj is ushort v ? checked((sbyte)v) : null,
+            32 => obj => obj is uint v ? checked((sbyte)v) : null,
+            64 => obj => obj is ulong v ? checked((sbyte)v) : null,
+            _ => throw new NotImplementedException(),
+         };
+
+   public static Func<object?, short?> GetInt16Converter(IntLogicalType logicalType) =>
+      GetInt16Converter(logicalType.IsSigned, logicalType.BitWidth);
+
+   public static Func<object?, short?> GetInt16Converter(bool isSigned, int bitWidth) =>
+      isSigned
+         ? bitWidth switch {
+            8 => obj => obj is sbyte v ? v : null,
+            16 => obj => obj as short?,
+            32 => obj => obj is int v ? checked((short)v) : null,
+            64 => obj => obj is long v ? checked((short)v) : null,
+            _ => throw new NotImplementedException(),
+         }
+         : bitWidth switch {
+            8 => obj => obj is byte v ? v : null,
+            16 => obj => obj is ushort v ? checked((short)v) : null,
+            32 => obj => obj is uint v ? checked((short)v) : null,
+            64 => obj => obj is ulong v ? checked((short)v) : null,
+            _ => throw new NotImplementedException(),
+         };
+
+   public static Func<object?, int?> GetInt32Converter(IntLogicalType logicalType) =>
+      GetInt32Converter(logicalType.IsSigned, logicalType.BitWidth);
+
+   public static Func<object?, int?> GetInt32Converter(bool isSigned, int bitWidth) =>
+      isSigned
+         ? bitWidth switch {
+            8 => obj => obj is sbyte v ? v : null,
+            16 => obj => obj is short v ? v : null,
+            32 => obj => obj as int?,
+            64 => obj => obj is long v ? checked((int)v) : null,
+            _ => throw new NotImplementedException(),
+         }
+         : bitWidth switch {
+            8 => obj => obj is byte v ? v : null,
+            16 => obj => obj is ushort v ? v : null,
+            32 => obj => obj is uint v ? checked((int)v) : null,
+            64 => obj => obj is ulong v ? checked((int)v) : null,
+            _ => throw new NotImplementedException(),
+         };
+
+   public static Func<object?, long?> GetInt64Converter(IntLogicalType logicalType) =>
+      GetInt64Converter(logicalType.IsSigned, logicalType.BitWidth);
+
+   public static Func<object?, long?> GetInt64Converter(bool isSigned, int bitWidth) =>
+      isSigned
+         ? bitWidth switch {
+            8 => obj => obj is sbyte v ? v : null,
+            16 => obj => obj is short v ? v : null,
+            32 => obj => obj is int v ? v : null,
+            64 => obj => obj as long?,
+            _ => throw new NotImplementedException(),
+         }
+         : bitWidth switch {
+            8 => obj => obj is byte v ? v : null,
+            16 => obj => obj is ushort v ? v : null,
+            32 => obj => obj is uint v ? v : null,
+            64 => obj => obj is ulong v ? checked((long)v) : null,
+            _ => throw new NotImplementedException(),
+         };
+
+   public static Func<object?, byte?> GetUInt8Converter(IntLogicalType logicalType) =>
+      GetUInt8Converter(logicalType.IsSigned, logicalType.BitWidth);
+
+   public static Func<object?, byte?> GetUInt8Converter(bool isSigned, int bitWidth) =>
+      isSigned
+         ? bitWidth switch {
+            8 => obj => obj is sbyte v ? checked((byte)v) : null,
+            16 => obj => obj is short v ? checked((byte)v) : null,
+            32 => obj => obj is int v ? checked((byte)v) : null,
+            64 => obj => obj is long v ? checked((byte)v) : null,
+            _ => throw new NotImplementedException(),
+         }
+         : bitWidth switch {
+            8 => obj => obj as byte?,
+            16 => obj => obj is ushort v ? checked((byte)v) : null,
+            32 => obj => obj is uint v ? checked((byte)v) : null,
+            64 => obj => obj is ulong v ? checked((byte)v) : null,
+            _ => throw new NotImplementedException(),
+         };
+
+   public static Func<object?, ushort?> GetUInt16Converter(IntLogicalType logicalType) =>
+      GetUInt16Converter(logicalType.IsSigned, logicalType.BitWidth);
+
+   public static Func<object?, ushort?> GetUInt16Converter(bool isSigned, int bitWidth) =>
+      isSigned
+         ? bitWidth switch {
+            8 => obj => obj is sbyte v ? checked((ushort)v) : null,
+            16 => obj => obj is short v ? checked((ushort)v) : null,
+            32 => obj => obj is int v ? checked((ushort)v) : null,
+            64 => obj => obj is long v ? checked((ushort)v) : null,
+            _ => throw new NotImplementedException(),
+         }
+         : bitWidth switch {
+            8 => obj => obj is byte v ? v : null,
+            16 => obj => obj is ushort v ? v : null,
+            32 => obj => obj is uint v ? checked((ushort)v) : null,
+            64 => obj => obj is ulong v ? checked((ushort)v) : null,
+            _ => throw new NotImplementedException(),
+         };
+
+   public static Func<object?, uint?> GetUInt32Converter(IntLogicalType logicalType) =>
+      GetUInt32Converter(logicalType.IsSigned, logicalType.BitWidth);
+
+   public static Func<object?, uint?> GetUInt32Converter(bool isSigned, int bitWidth) =>
+      isSigned
+         ? bitWidth switch {
+            8 => obj => obj is sbyte v ? checked((uint)v) : null,
+            16 => obj => obj is short v ? checked((uint)v) : null,
+            32 => obj => obj is int v ? checked((uint)v) : null,
+            64 => obj => obj is long v ? checked((uint)v) : null,
+            _ => throw new NotImplementedException(),
+         }
+         : bitWidth switch {
+            8 => obj => obj is byte v ? v : null,
+            16 => obj => obj is ushort v ? v : null,
+            32 => obj => obj is uint v ? v : null,
+            64 => obj => obj is ulong v ? checked((uint)v) : null,
+            _ => throw new NotImplementedException(),
+         };
+
+   public static Func<object?, ulong?> GetUInt64Converter(IntLogicalType logicalType) =>
+      GetUInt64Converter(logicalType.IsSigned, logicalType.BitWidth);
+
+   public static Func<object?, ulong?> GetUInt64Converter(bool isSigned, int bitWidth) =>
+      isSigned
+         ? bitWidth switch {
+            8 => obj => obj is sbyte v ? checked((ulong)v) : null,
+            16 => obj => obj is short v ? checked((ulong)v) : null,
+            32 => obj => obj is int v ? checked((ulong)v) : null,
+            64 => obj => obj is long v ? checked((ulong)v) : null,
+            _ => throw new NotImplementedException(),
+         }
+         : bitWidth switch {
+            8 => obj => obj is byte v ? v : null,
+            16 => obj => obj is ushort v ? v : null,
+            32 => obj => obj is uint v ? v : null,
+            64 => obj => obj is ulong v ? v : null,
+            _ => throw new NotImplementedException(),
+         };
+
+   public static string ColumnEnumTypeKey(string colName) =>
+      $"{colName}.EnumType";
+
+   public static KeyValuePair<string, string> ColumnEnumTypeEntry(string colName, Type enumType) {
+      if (enumType.IsNullable()) {
+         enumType = enumType.GetGenericArguments()[0];
+      }
+      if (!enumType.IsEnum) {
+         throw new ArgumentException($"Must be enum type (actually {enumType})", nameof(enumType));
+      }
+      return new(ColumnEnumTypeKey(colName), enumType.ToString());
+   }
+
+   public static Type? GetColumnEnumType(IReadOnlyDictionary<string, string>? keyValueMetadata, string columnName) {
+      if (keyValueMetadata is not null) {
+         var key = ColumnEnumTypeKey(columnName);
+         if (keyValueMetadata.TryGetValue(key, out var typeName)) {
+            return Type.GetType(typeName);
+         }
+      }
+      return null;
+   }
+
+   public interface IColumnWriter : IDisposable {
+      void Write(IList<object?> values);
+   }
+
+   public interface IColumnWriter<T> : IColumnWriter {
+      void Write(IList<T?> values);
+   }
+
+   class ColumnWriter<T> : IColumnWriter<T> {
+      readonly T?[] buffer;
+      readonly LogicalColumnWriter<T> writer;
+      readonly Func<object?, T?> convert;
+
+      public ColumnWriter(ColumnWriter writer, int bufferSize)
+         : this(writer, new T?[bufferSize]) { }
+
+      public ColumnWriter(ColumnWriter writer, int bufferSize, Func<object?, T?> convert)
+         : this(writer, new T?[bufferSize], convert) { }
+
+      public ColumnWriter(ColumnWriter writer, T?[] buffer)
+         : this(writer, buffer, obj => (T?)obj) { }
+
+      public ColumnWriter(ColumnWriter writer, T?[] buffer, Func<object?, T?> convert) {
+         this.buffer = buffer;
+         this.writer = writer.LogicalWriter<T>();
+         this.convert = convert;
+      }
+
+      public void Write(IList<T?> values) {
+         if (buffer.Length == 0) {
+            throw new InvalidOperationException("Cannot write because buffer length is 0");
+         }
+         var count = values.Count;
+         var nPages = count / buffer.Length;
+         using var en = values.GetEnumerator();
+         for (int pg = 0; pg < nPages; pg++) {
+            for (int i = 0; i < buffer.Length; i++) {
+               Log.assert(en.MoveNext(), "Enumeration failed before reaching expected count");
+               buffer[i] = en.Current;
+            }
+            writer.WriteBatch(buffer!);
+         }
+         var remainder = count - nPages * buffer.Length;
+         if (remainder != 0) {
+            for (int i = 0; i < remainder; i++) {
+               Log.assert(en.MoveNext(), "Enumeration failed before reaching expected count");
+               buffer[i] = en.Current;
+            }
+            writer.WriteBatch(buffer!, 0, remainder);
+         }
+      }
+
+      public void Dispose() =>
+         writer.Dispose();
+
+      void IColumnWriter.Write(IList<object?> values) =>
+         Write(values.WrapSelector(convert, null));
+   }
+}

src/DataStreamSuite/DataStream/BlueOrb/Parquet/ParquetColumnReader.cs

@@ -0,0 +1,120 @@
+namespace bts.BlueOrb.Parquet;
+
+using bts.collections;
+
+using ParquetSharp;
+
+using Spark.Core;
+
+using System;
+using System.Collections.Generic;
+
+class ParquetColumnReader<T> : IParquetColumnReader<T> {
+   protected readonly LogicalColumnReader<T> reader;
+   protected readonly T?[] buffer;
+
+   public ParquetColumnReader(ColumnReader reader, int bufferSize)
+      : this(reader, new T?[bufferSize]) { }
+
+   protected ParquetColumnReader(ColumnReader reader, T?[] buffer) {
+      this.buffer = buffer;
+      this.reader = reader.LogicalReader<T>();
+   }
+
+   ~ParquetColumnReader() =>
+      Dispose();
+
+   public IList<T?> Read() {
+      if (buffer.Length == 0) {
+         throw new InvalidOperationException("Cannot write because buffer length is 0");
+      }
+      int n = reader.ReadBatch(buffer!);
+      return buffer.WrapSubRange(0, n);
+   }
+
+   public IList<T?> Read(int? start, int? stop) {
+      if (buffer.Length == 0) {
+         throw new InvalidOperationException("Cannot write because buffer length is 0");
+      }
+      start ??= 0;
+      stop ??= buffer.Length;
+      int length = stop.Value - start.Value;
+      if (length <= 0) {
+         return new List<T?>(0);
+      }
+      if (start.Value > 0) {
+         var nSkipped = checked((int)reader.Skip(start.Value));
+         Log.assert(nSkipped >= 0 && nSkipped <= start.Value);
+         if (nSkipped < start.Value) {
+            return new List<T?>(0);
+         }
+      }
+      int n = reader.ReadBatch(buffer!, 0, length);
+      return buffer.WrapSubRange(0, n);
+   }
+
+   public virtual void ReadTo(Variant[] dest, int start, int count) {
+      throw new NotImplementedException();
+   }
+
+   public void Dispose() {
+      GC.SuppressFinalize(this);
+      reader.Dispose();
+   }
+}
+
+class ParquetColumnReader<TIn, TOut> : IParquetColumnReader<TOut> {
+   readonly Func<TIn, TOut?> convert;
+   readonly LogicalColumnReader<TIn> reader;
+   readonly TIn[] buffer;
+
+   public ParquetColumnReader(ColumnReader reader, Func<TIn, TOut?> convert, int bufferSize)
+      : this(reader, convert, new TIn[bufferSize]) { }
+
+   public ParquetColumnReader(ColumnReader reader, Func<TIn, TOut?> convert, TIn[] buffer) {
+      this.convert = convert;
+      this.reader = reader.LogicalReader<TIn>();
+      this.buffer = buffer;
+   }
+
+   ~ParquetColumnReader() =>
+      Dispose();
+
+   public IList<TOut?> Read() {
+      if (buffer.Length == 0) {
+         throw new InvalidOperationException("Cannot write because buffer length is 0");
+      }
+      int n = reader.ReadBatch(buffer!);
+      return buffer.WrapSelector(convert, null).WrapSubRange(0, n);
+   }
+
+   public IList<TOut?> Read(int? start, int? stop) {
+      if (buffer.Length == 0) {
+         throw new InvalidOperationException("Cannot write because buffer length is 0");
+      }
+      start ??= 0;
+      stop ??= buffer.Length;
+      int length = stop.Value - start.Value;
+      if (length <= 0) {
+         return new List<TOut?>(0);
+      }
+      if (start.Value > 0) {
+         var nSkipped = checked((int)reader.Skip(start.Value));
+         Log.assert(nSkipped >= 0 && nSkipped <= start.Value);
+         if (nSkipped < start.Value) {
+            return new List<TOut?>(0);
+         }
+      }
+      int n = reader.ReadBatch(buffer!, 0, length);
+      return buffer.WrapSelector(convert, null).WrapSubRange(0, n);
+   }
+
+   public void ReadTo(Variant[] dest, int start, int count) {
+      throw new NotImplementedException();
+   }
+
+   public void Dispose() {
+      GC.SuppressFinalize(this);
+      reader.Dispose();
+   }
+}

src/DataStreamSuite/DataStream/BlueOrb/Parquet/ParquetExtensions.cs

@@ -0,0 +1,158 @@
+namespace bts.BlueOrb.Parquet;
+
+using BCG;
+
+using bts.collections;
+
+using Microsoft.Scripting.Utils;
+
+using ParquetSharp;
+
+using System;
+using System.Collections.Generic;
+using System.Linq;
+
+public static class ParquetExtensions {
+   public static int RowCount(this FileMetaData fileMetaData) =>
+      checked((int)fileMetaData.NumRows);
+
+   public static IBlueTableSchema ToBlueSchema(this SchemaDescriptor schemaDescriptor, string tableName = "parquet", IReadOnlyDictionary<string, string>? keyValueMetadata = null) {
+      var colInfos = new ColumnInfo[schemaDescriptor.NumColumns];
+      for (int i = 0; i < colInfos.Length; i++) {
+         colInfos[i] = schemaDescriptor.BlueColumn(i, keyValueMetadata);
+      }
+      return new BlueTableSchema(tableName, colInfos);
+   }
+
+   public static IBlueTableSchema ToBlueSchema(this FileMetaData fileMetaData, string tableName = "parquet") =>
+      ToBlueSchema(fileMetaData.Schema, tableName, fileMetaData.KeyValueMetadata);
+
+   public static ColumnInfo BlueColumn(this SchemaDescriptor schemaDescriptor, int index, IReadOnlyDictionary<string, string>? keyValueMetadata = null) {
+      var c = schemaDescriptor.Column(index);
+      var lt = c.LogicalType;
+      if (lt.Type == LogicalTypeEnum.Enum) {
+         var enumType = ParquetAdapter.GetColumnEnumType(keyValueMetadata, c.Name);
+         if (enumType is not null) {
+            return new(c.Name, index, enumType);
+         }
+      }
+      return new(c.Name, index, getTypeCode(c.LogicalType, c.PhysicalType));
+   }
+
+   public static IBlueRowCollection ReadBlueTable(this ParquetFileReader fileReader, int? start = null, int? stop = null) {
+      var schema = fileReader.FileMetaData.ToBlueSchema("parquetReader");
+      var colInfos = schema.columnInfo;
+      IBlueRowCollection? rv;
+      if (!schema.columnInfo.Any(c => c.colName == "id")) {
+         rv = BlueAddIndexColumnTable.Create(schema, fileReader.ReadColumns(start, stop));
+      } else {
+         rv = new BlueIndexedColumnTable(schema, fileReader.ReadColumns(start, stop));
+      }
+      Log.assert(!schema.columnInfo.Any(c => c is null));
+      return rv;
+   }
+
+   public static (Column[] Column, IReadOnlyDictionary<string, string> Meta) ToParquetSchema(this IBlueTableSchema schema) {
+      var blueColumns = schema.columnInfo;
+      var parquetColumns = new Column[blueColumns.Length];
+      var parquetMeta = new Dictionary<string, string>();
+      KeyValuePair<string, string>[] parquetColMeta;
+      for (int i = 0; i < blueColumns.Length; i++) {
+         (parquetColumns[i], parquetColMeta) = ToParquetColumn(blueColumns[i]);
+         parquetMeta.AddRange(parquetColMeta);
+      }
+      return (parquetColumns, parquetMeta);
+   }
+
+   public static (Column Column, KeyValuePair<string, string>[] Meta) ToParquetColumn(this ColumnInfo columnInfo) {
+      var colType = columnInfo.colType;
+      bool isNullable = colType.IsNullable();
+      if (isNullable) {
+         colType = colType.GetGenericArguments()[0];
+      }
+      KeyValuePair<string, string>[] meta;
+      TypeCode typeCode;
+      var colName = columnInfo.colName;
+      if (colType.IsEnum) {
+         typeCode = colType.GetEnumUnderlyingType().GetTypeCode();
+         meta = new[] { ParquetAdapter.ColumnEnumTypeEntry(columnInfo.colName, colType) };
+      } else {
+         typeCode = colType.GetTypeCode();
+         meta = Array.Empty<KeyValuePair<string, string>>();
+      }
+      return (ToParquetColumn(colName, typeCode), meta);
+
+      static Column ToParquetColumn(string colName, TypeCode typeCode) =>
+         typeCode switch {
+            TypeCode.Boolean => new Column<bool?>(colName),
+            TypeCode.Byte => new Column<byte?>(colName),
+            TypeCode.Int16 => new Column<short?>(colName),
+            TypeCode.Int32 => new Column<int?>(colName),
+            TypeCode.Int64 => new Column<long?>(colName),
+            TypeCode.UInt16 => new Column<ushort?>(colName),
+            TypeCode.UInt32 => new Column<uint?>(colName),
+            TypeCode.UInt64 => new Column<ulong?>(colName),
+            TypeCode.Single => new Column<float?>(colName),
+            TypeCode.Double => new Column<double?>(colName),
+            TypeCode.DateTime => new Column<DateTimeNanos?>(colName),
+            TypeCode.String => new Column<string?>(colName),
+            _ => throw new NotSupportedException($"Unsupported column TypeCode {typeCode}"),
+         };
+   }
+
+   static TypeCode getTypeCode(LogicalType logicalType, PhysicalType physicalType) {
+      return logicalType.Type switch {
+         // LogicalTypeEnum.Int => TypeCode.Int32,
+         // LogicalTypeEnum.Decimal => TypeCode.Double,
+         LogicalTypeEnum.Date or LogicalTypeEnum.Time or LogicalTypeEnum.Timestamp => TypeCode.DateTime,
+         LogicalTypeEnum.String => TypeCode.String,
+         LogicalTypeEnum.Int => FromInt(),
+         LogicalTypeEnum.Decimal => FromDecimal(),
+         _ => FromPhysicalType(),
+      };
+
+      TypeCode FromPhysicalType() =>
+         physicalType switch {
+            PhysicalType.Boolean => TypeCode.Boolean,
+            PhysicalType.Int32 => TypeCode.Int32,
+            PhysicalType.Int64 => TypeCode.Int64,
+            PhysicalType.Float => TypeCode.Single,
+            PhysicalType.Double => TypeCode.Double,
+            _ => throw new NotImplementedException($"TypeCode not implemented for {logicalType} (logical) {physicalType} (physical)"),
+         };
+
+      TypeCode FromInt() {
+         var ilt = (IntLogicalType)logicalType;
+         var bitWidth = ilt.BitWidth;
+         return ilt.IsSigned ? FromSignedInt() : FromUnsignedInt();
+
+         TypeCode FromSignedInt() =>
+            bitWidth switch {
+               8 => TypeCode.SByte,
+               16 => TypeCode.Int16,
+               32 => TypeCode.Int32,
+               64 => TypeCode.Int64,
+               _ => throw new NotSupportedException($"Bit width not supported for signed int ({bitWidth})"),
+            };
+
+         TypeCode FromUnsignedInt() =>
+            bitWidth switch {
+               8 => TypeCode.Byte,
+               16 => TypeCode.UInt16,
+               32 => TypeCode.UInt32,
+               64 => TypeCode.UInt64,
+               _ => throw new NotSupportedException($"Bit width not supported for unsigned int ({bitWidth})"),
+            };
+      }
+
+      TypeCode FromDecimal() {
+         var dlt = (DecimalLogicalType)logicalType;
+         var scale = dlt.Scale;
+         var precision = dlt.Precision;
+         return scale switch {
+            0 => throw new NotSupportedException($"Scale not supported for decimal ({scale})"),
+            _ => TypeCode.Double,
+         };
+      }
+   }
+}

src/DataStreamSuite/DataStream/BlueOrb/Parquet/ParquetGroupTable.cs

@@ -0,0 +1,234 @@
+namespace bts.BlueOrb.Parquet;
+
+using BCG;
+
+using bts.collections;
+
+using Bts.DataStreamCore;
+
+using ParquetSharp;
+
+using Spark.Core.Collections;
+
+using System;
+using System.Collections.Generic;
+using System.Diagnostics.CodeAnalysis;
+
+public class ParquetGroupTable : IBlueRowCollection, IDisposable {
+   readonly RowGroupReader reader;
+   readonly IBlueTableSchema schema;
+   readonly bool leaveOpen;
+   readonly int fullCount;
+   readonly object syncRoot = new();
+   readonly int? start;
+   readonly int? stop;
+   readonly int startIndex;
+   readonly int stopIndex;
+   readonly int idColumnIndex;
+   readonly int idColumnIndexInFile;
+   readonly Func<int, int>? rowIdFactory;
+   DataCache? dataCacheBacking;
+
+   public ParquetGroupTable(
+      RowGroupReader reader,
+      IBlueTableSchema schema,
+      int? start = null,
+      int? stop = null,
+      int idColumnIndex = -1,
+      int idColumnIndexInFile = -1,
+      Func<int, int>? rowIdFactory = null,
+      bool leaveOpen = false
+   ) {
+      this.reader = reader;
+      this.schema = schema;
+      this.start = start;
+      this.stop = stop;
+      this.idColumnIndex = idColumnIndex;
+      this.idColumnIndexInFile = idColumnIndexInFile;
+      this.rowIdFactory = rowIdFactory;
+      this.leaveOpen = leaveOpen;
+      fullCount = checked((int)reader.MetaData.NumRows);
+      Log.assert(fullCount >= 0);
+      startIndex = SliceList.GetStartIndex(start, fullCount, true);
+      stopIndex = SliceList.GetStopIndex(stop, fullCount, true);
+      stopIndex = Math.Max(stopIndex, startIndex);
+      Log.assert(
+         startIndex >= 0
+         && startIndex <= fullCount
+         && stopIndex >= 0
+         && stopIndex <= fullCount
+         && stopIndex >= startIndex);
+   }
+
+   ~ParquetGroupTable() =>
+      Dispose();
+
+   public int Count => stopIndex - startIndex;
+
+   IBlueTableSchema IBlueRowCollection.schema => schema;
+
+   DataCache dataCache =>
+      EnsureInit();
+
+   public IBlueRow this[int index] => dataCache[index];
+
+   IBlueRow IList<IBlueRow>.this[int index] {
+      get => dataCache.Rows[index];
+      set => throw new NotSupportedException();
+   }
+
+   public bool containsID(int id) =>
+      dataCache.RowIdToIndex.ContainsKey(id);
+
+   public IBlueRow GetRow(int index) =>
+      dataCache.Rows[index];
+
+   public int IndexOf(IBlueRow item) {
+      int id = item.Id();
+      return dataCache.RowIdToIndex.TryGetValue(id, out var idx) && BlueRowEqualityComparer.Equals(dataCache.Rows[idx], item)
+         ? idx
+         : -1;
+   }
+
+   public bool TryGetValue(int id, [NotNullWhen(true)] out IBlueRow? row) {
+      if (dataCache.RowIdToIndex.TryGetValue(id, out var idx)) {
+         row = dataCache.Rows[idx];
+         return true;
+      }
+      row = null;
+      return false;
+   }
+
+   public IEnumerator<IBlueRow> GetEnumerator() {
+      for (int i = 0; i < dataCache.Rows.Length; i++) {
+         yield return dataCache.Rows[i];
+      }
+   }
+
+   public void Dispose() {
+      GC.SuppressFinalize(this);
+      if (!leaveOpen) {
+         reader.Dispose();
+      }
+   }
+
+   [MemberNotNull(nameof(dataCacheBacking))]
+   internal DataCache EnsureInit() {
+      if (dataCacheBacking.HasValue) {
+         return dataCacheBacking.Value;
+      }
+      lock (syncRoot) {
+         if (dataCacheBacking.HasValue) {
+            return dataCacheBacking.Value;
+         }
+         var rows = new IBlueRow[Count];
+         int numColumns = reader.MetaData.NumColumns;
+         var blueColumns = schema.columnInfo;
+         Log.assert((blueColumns.Length - numColumns) switch {
+            0 or 1 => true,
+            _ => false,
+         });
+         var columnValues = allocateColumnValues(
+            reader, blueColumns.Length, Count, idColumnIndex, idColumnIndexInFile, startIndex, stopIndex, rowIdFactory);
+         var rowIdToIndex = new Dictionary<int, int>();
+         for (int i = 0; i < rows.Length; i++) {
+            rows[i] = new BlueRowColumnWrapper(schema, i, columnValues);
+         }
+         dataCacheBacking = new(rows, rowIdToIndex, columnValues);
+      }
+      return dataCacheBacking.Value;
+   }
+
+   static IList<object?>[] allocateColumnValues(
+      RowGroupReader reader,
+      int numColumns,
+      int numRows,
+      int idColIdx,
+      int idColIdxInFile,
+      int startIndex,
+      int stopIndex,
+      Func<int, int>? idFactory
+   ) {
+      if (idColIdx < 0 || idColIdx >= numColumns) {
+         throw new ArgumentException("Id column index out of bounds", nameof(idColIdx));
+      }
+      var columnValues = new IList<object?>[numColumns];
+      if (idColIdxInFile == -1) {
+         // Generate ids (there is no id column in the file)
+         ArgumentNullException.ThrowIfNull(idFactory);
+         for (int i = 0; i < numColumns; i++) {
+            if (i < idColIdx) {
+               ReadColumn(i, i);
+            } else if (i == idColIdx) {
+               columnValues[idColIdx] = new StepList<int>(0, 1, numRows).WrapSelector(idx => (object?)idFactory(idx), null);
+            } else {
+               ReadColumn(i - 1, i);
+            }
+         }
+      } else if (idFactory is not null) {
+         // Generate ids (skip id column in the file)
+         for (int i = 0; i < numColumns; i++) {
+            if (i == idColIdx) {
+               columnValues[idColIdx] = new StepList<int>(0, 1, numRows).WrapSelector(idx => (object?)idFactory(idx), null);
+            } else {
+               if (i > idColIdx && i < idColIdxInFile) {
+                  ReadColumn(i - 1, i);
+               } else if (i > idColIdxInFile && i < idColIdx) {
+                  ReadColumn(i + 1, i);
+               } else {
+                  ReadColumn(i, i);
+               }
+            }
+         }
+      } else {
+         // Get ids from the file (output table id column index may differ from file)
+         // Case 1) File id col index > output
+         //    src cols: a, b, c, d, (id), e, f, ...
+         //    dst cols: a, b, (id), c, d, e, f, ...
+         //
+         // Case 2) File id col index == output
+         //    src cols: a, b, (id), c, d, e, f, ...
+         //    dst cols: a, b, (id), c, d, e, f, ...
+         //
+         // Case 3) File id col index < output
+         //    src cols: a, b, (id), c, d, e, f, ...
+         //    dst cols: a, b, c, d, (id), e, f, ...
+
+         for (int i = 0; i < numColumns; i++) {
+            if (i == idColIdx) {
+               ReadColumn(idColIdxInFile, idColIdx);
+            } else if (i > idColIdx && i < idColIdxInFile) {
+               ReadColumn(i - 1, i);
+            } else if (i > idColIdxInFile && i < idColIdx) {
+               ReadColumn(i, i - 1);
+            } else {
+               ReadColumn(i, i);
+            }
+         }
+      }
+      return columnValues;
+
+      void ReadColumn(int srcIdx, int dstIdx) {
+         using var colReader = reader.GetColumnReader(srcIdx);
+         columnValues[dstIdx] = colReader.Read(startIndex, stopIndex);
+      }
+   }
+
+   internal readonly struct DataCache {
+      public readonly IBlueRow[] Rows;
+      public readonly Dictionary<int, int> RowIdToIndex;
+      public readonly IList<object?>[] ColumnValues;
+
+      public DataCache(IBlueRow[] rows, Dictionary<int, int> rowIdToIndex, IList<object?>[] columnValues) {
+         Rows = rows;
+         RowIdToIndex = rowIdToIndex;
+         ColumnValues = columnValues;
+      }
+
+      public IBlueRow this[int index] =>
+         Rows[RowIdToIndex[index]];
+
+      public IBlueRow GetRow(int index) =>
+         Rows[index];
+   }
+}

src/DataStreamSuite/DataStream/BlueOrb/Parquet/ParquetReaderStream.cs

@@ -0,0 +1,66 @@
+namespace bts.BlueOrb.Parquet;
+
+using BCG;
+
+using bts.collections;
+
+using Bts.DataStreamCore;
+using Bts.PubSub;
+
+using Microsoft.Extensions.Logging;
+
+using Spark.Core.Disposables;
+
+using System;
+using System.Threading;
+
+[DataStreamUserRespresentation(false)]
+public class ParquetReaderStream : ViewStream {
+   readonly string file;
+   readonly int? start;
+   readonly int? stop;
+   readonly IWorkScheduler workScheduler;
+   IBlueRowCollection? table;
+   IDisposable? workerSub;
+
+   public ParquetReaderStream(string file, IWorkScheduler workScheduler) {
+      this.file = file;
+      this.workScheduler = workScheduler;
+   }
+
+   public ParquetReaderStream(string file, int? start, int? stop, IWorkScheduler workScheduler) {
+      this.file = file;
+      this.start = start;
+      this.stop = stop;
+      this.workScheduler = workScheduler;
+   }
+
+   protected override void myOnFirstSubscriber() {
+      Start();
+      var worker = new AnonymousWorker(DoWork);
+      workScheduler.Add(worker);
+      workerSub = Disposable.Create(() => workScheduler.Remove(worker));
+
+      async void Start() {
+         try {
+            Volatile.Write(ref table, await ParquetTableReader.ReadAsync(file, start, stop));
+         } catch (Exception ex) {
+            Log.DataFlow.error($"[parquet_reader_stream_err.json]{{ \"err\": \"{ex}\" }}");
+         }
+      }
+
+      void DoWork() {
+         var table = Volatile.Read(ref this.table);
+         if (table is null) {
+            return;
+         }
+         Disposable.DisposeThreadSafe(ref workerSub);
+         if (table.schema.columnInfo.Length > 1) {
+            myFireEvent(new DataPacket(table, table, new BlueTableImmutable(table.schema)));
+         }
+      }
+   }
+
+   protected override void myOnLastSubscriber() =>
+      Disposable.DisposeThreadSafe(ref workerSub);
+}

src/DataStreamSuite/DataStream/BlueOrb/Parquet/ParquetRow.cs

@@ -0,0 +1,26 @@
+// namespace bts.BlueOrb.Parquet;
+
+// using BCG;
+
+// using global::Parquet.Data;
+
+// public class ParquetRow : IBlueRow {
+//    readonly int id;
+//    readonly int rowIndex;
+//    readonly DataColumn[] data;
+
+//    public ParquetRow(int id, int rowIndex, DataColumn[] data, IBlueTableSchema schema) {
+//       this.id = id;
+//       this.rowIndex = rowIndex;
+//       this.data = data;
+//       Schema = schema;
+//    }
+
+//    public object? this[int index] =>
+//       index == 0
+//          ? id
+//          : data[index - 1].Data.GetValue(rowIndex);
+
+//    public IBlueTableSchema Schema { get; }
+//    IBlueTableSchema IBlueRow.schema => Schema;
+// }

src/DataStreamSuite/DataStream/BlueOrb/Parquet/ParquetTable.cs

@@ -0,0 +1,145 @@
+namespace bts.BlueOrb.Parquet;
+
+using BCG;
+
+using bts.collections;
+
+using Bts.DataStreamCore;
+
+using ParquetSharp;
+
+using Spark.Core.Collections;
+
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Threading.Tasks;
+
+public class ParquetTable : IBlueRowCollection, IDisposable {
+   readonly SortedList<int, ParquetGroupTable> groups = new();
+   readonly int fullCount;
+   readonly IBlueTableSchema schema;
+   readonly ParquetFileReader? reader;
+   readonly int startIndex;
+   readonly int stopIndex;
+
+   public ParquetTable(ParquetFileReader reader, string tableName = "parquet", int? start = null, int? stop = null, bool leaveOpen = false) {
+      schema = reader.FileMetaData.ToBlueSchema(tableName);
+      int idColIdx;
+      if (schema.TryResolveIndices("id", out int idColIdxInFile)) {
+         idColIdx = idColIdxInFile;
+      } else {
+         var origCols = schema.columnInfo;
+         var cols = new ColumnInfo[origCols.Length + 1];
+         idColIdx = 0;
+         cols[0] = new("id", 0, typeof(int));
+         for (int i = 0; i < origCols.Length; i++) {
+            var c = origCols[i];
+            cols[i + 1] = new(c.colName, i + 1, c.colType);
+         }
+         schema = new BlueTableSchema(cols, schema.tableName);
+      }
+      fullCount = checked((int)reader.FileMetaData.NumRows);
+      Log.assert(fullCount >= 0);
+      startIndex = SliceList.GetStartIndex(start, fullCount, true);
+      stopIndex = SliceList.GetStopIndex(stop, fullCount, true);
+      stopIndex = Math.Max(stopIndex, startIndex);
+      Log.assert(startIndex >= 0 && startIndex <= fullCount && stopIndex >= 0 && stopIndex <= fullCount && stopIndex >= startIndex);
+      int numRowGroups = reader.FileMetaData.NumRowGroups;
+      int rowGroupLastIdx = 0;
+      object syncRoot = new();
+      for (int i = 0; i < numRowGroups && rowGroupLastIdx < Count; i++) {
+         int? grpStart = start.HasValue ? startIndex - rowGroupLastIdx : null;
+         int? grpStop = stop.HasValue ? stopIndex - rowGroupLastIdx : null;
+         int offset = rowGroupLastIdx;
+         var grp = new ParquetGroupTable(reader.RowGroup(i), schema, grpStart, grpStop, idColIdx, idColIdxInFile, GetAutoId);
+         if (grp.Count == 0) {
+            continue;
+         }
+         rowGroupLastIdx += grp.Count;
+         groups.Add(rowGroupLastIdx, grp);
+
+         int GetAutoId(int index) =>
+            index + offset;
+      }
+      Parallel.For(0, groups.Count, i => groups.GetValueAtIndex(i).EnsureInit());
+      if (!leaveOpen) {
+         this.reader = reader;
+      }
+   }
+
+   ~ParquetTable() =>
+      Dispose();
+
+   public IBlueTableSchema Schema => schema;
+
+   public int Count => stopIndex - startIndex;
+
+   IBlueTableSchema IBlueRowCollection.schema => schema;
+
+   public IBlueRow this[int index] {
+      get {
+         foreach (var grp in groups.Values) {
+            if (grp.TryGetValue(index, out var r)) {
+               return r;
+            }
+         }
+         throw new KeyNotFoundException();
+      }
+   }
+
+   IBlueRow IList<IBlueRow>.this[int index] {
+      get => GetRow(index);
+      set => throw new NotSupportedException();
+   }
+
+   public static ParquetTable FromFile(string path, int? start = null, int? stop = null, string tableName = "parquet") =>
+      new(new(path), tableName, start, stop, false);
+
+   public IBlueRow GetRow(int index) {
+      if (index < 0 || index >= Count) {
+         throw new IndexOutOfRangeException();
+      }
+      int rowGroupIdx = groups.Keys.BinarySearch(index);
+      if (rowGroupIdx < 0) {
+         rowGroupIdx = ~rowGroupIdx;
+      }
+      Log.assert(rowGroupIdx <= groups.Count);
+      var grp = groups[rowGroupIdx];
+      int idxWithinGrp = index;
+      if (rowGroupIdx > 0) {
+         idxWithinGrp -= groups.Keys[rowGroupIdx - 1];
+      }
+      return grp[idxWithinGrp];
+   }
+
+   public bool containsID(int id) =>
+      groups.Values.Any(rgl => rgl.containsID(id));
+
+   public int IndexOf(IBlueRow item) {
+      int idx;
+      int grpFirstIdx = 0;
+      foreach (var (grpLastIdx, grp) in groups) {
+         idx = grp.IndexOf(item);
+         if (idx >= 0) {
+            return idx + grpFirstIdx;
+         }
+         grpFirstIdx = grpLastIdx;
+      }
+      return -1;
+   }
+
+   public IEnumerator<IBlueRow> GetEnumerator() {
+      foreach (var rgl in groups.Values) {
+         foreach (var r in rgl) {
+            yield return r;
+         }
+      }
+   }
+
+   public void Dispose() {
+      GC.SuppressFinalize(this);
+      groups.Values.DisposeAll();
+      reader?.Dispose();
+   }
+}

src/DataStreamSuite/DataStream/BlueOrb/Parquet/ParquetTableReader.cs

@@ -0,0 +1,33 @@
+namespace bts.BlueOrb;
+
+using BCG;
+
+using bts.BlueOrb.Parquet;
+
+using Bts.DataStreamCore;
+
+using ParquetSharp;
+
+using System.IO;
+using System.Threading.Tasks;
+
+public class ParquetTableReader {
+   public static Task<IBlueRowCollection> ReadAsync(Stream stream, bool leaveOpen = false, int? start = null, int? stop = null) =>
+      Task.Run(() => Read(stream, leaveOpen, start, stop));
+
+   public static Task<IBlueRowCollection> ReadAsync(string path, int? start = null, int? stop = null) =>
+      Task.Run(() => Read(path, start, stop));
+
+   public static IBlueRowCollection Read(Stream stream, bool leaveOpen = false, int? start = null, int? stop = null) {
+      using var fileReader = new ParquetFileReader(stream, leaveOpen);
+      var rv = fileReader.ReadBlueTable(start, stop);
+      rv.schema.Validate();
+      return rv;
+   }
+
+   public static IBlueRowCollection Read(string path, int? start = null, int? stop = null) {
+      IBlueRowCollection rv = ParquetTable.FromFile(path, start, stop);
+      rv.schema.Validate();
+      return rv;
+   }
+}

src/DataStreamSuite/DataStream/BlueOrb/Parquet/ParquetTableWriter.cs

@@ -0,0 +1,34 @@
+namespace bts.BlueOrb.Parquet;
+
+using BCG;
+
+using ParquetSharp;
+
+using System.IO;
+using System.Threading.Tasks;
+
+public static class ParquetTableWriter {
+   public static Task WriteAsync(IBlueRowCollection blueRows, Stream outStream, bool leaveOpen = false) =>
+      Task.Run(() => Write(blueRows, outStream, leaveOpen));
+
+   public static void Write(IBlueRowCollection blueRows, Stream outStream, bool leaveOpen = false) {
+      var (parquetColumns, parquetMeta) = blueRows.schema.ToParquetSchema();
+      using var schema = Column.CreateSchemaNode(parquetColumns);
+      using var writerProps = createWriterProperties();
+      using var writer = new ParquetFileWriter(outStream, schema, writerProps, keyValueMetadata: parquetMeta, leaveOpen: leaveOpen);
+      using var rowGroup = writer.AppendRowGroup();
+      for (int i = 0; i < blueRows.schema.columnInfo.Length; i++) {
+         var col = blueRows.schema.columnInfo[i];
+         Log.assert(col.colNo == i);
+         using var colWriter = ParquetAdapter.GetColumnWriter(col.type_code(), rowGroup.NextColumn());
+         colWriter.Write(blueRows.GetColumn(i));
+      }
+      writer.Close();
+   }
+
+   static WriterProperties createWriterProperties(Compression compression = Compression.Snappy) {
+      using var builder = new WriterPropertiesBuilder();
+      builder.Compression(compression);
+      return builder.Build();
+   }
+}

src/DataStreamSuite/DataStream/BlueOrb/Parquet/ParquetWriterStream.cs

@@ -0,0 +1,87 @@
+namespace bts.BlueOrb.Parquet;
+
+using BCG;
+
+using bts.collections;
+
+using Bts.DataStreamCore;
+using Bts.PubSub;
+
+using mbts.utils;
+
+using Spark.Core.Disposables;
+using Spark.Core.Threading;
+
+using System;
+using System.IO;
+
+[DataStreamUserRespresentation(false)]
+public class ParquetWriterStream : ViewStream {
+   readonly string file;
+   readonly ViewStream inputStream;
+   readonly IWorkScheduler workScheduler;
+   readonly AtomicGate gate = new();
+   IDataPacket? pendingOutput;
+   IDisposable? dispose;
+
+   public ParquetWriterStream(ViewStream inputStream, string file, IWorkScheduler workScheduler) {
+      this.inputStream = inputStream;
+      this.file = file;
+      this.workScheduler = workScheduler;
+   }
+
+   protected override void myOnFirstSubscriber() {
+      BlueTableImmutable.DataPacketBuilder? outputBuilder = null;
+      bool isFirstUpdate = false;
+      var worker = new AnonymousWorker(DoWork);
+      inputStream.onChange += OnInputStreamChange;
+      workScheduler.Add(worker);
+      dispose = Disposable.Create(Dispose);
+
+      void Dispose() {
+         workScheduler.Remove(worker);
+         inputStream.onChange -= OnInputStreamChange;
+      }
+
+      void OnInputStreamChange(object? dpObj) {
+         var dp = ExceptionAssert.Argument.Is<IDataPacket>(dpObj);
+         using (gate.BeginTransaction()) {
+            outputBuilder ??= new BlueTableImmutable(dp.Schema).ToDataPacketBuilder();
+            foreach (var r in dp.addedRows) {
+               outputBuilder.AddOrUpdate(r);
+            }
+            foreach (var r in dp.deletedRows) {
+               outputBuilder.Remove(r.Id());
+            }
+         }
+      }
+
+      async void DoWork() {
+         using (gate.BeginTransaction()) {
+            if (outputBuilder is null) {
+               return;
+            }
+            if (pendingOutput is not null) {
+               myFireEvent(pendingOutput);
+               pendingOutput = null;
+            }
+            var dp = outputBuilder.ToDataPacket();
+            if (isFirstUpdate) {
+               isFirstUpdate = false;
+            } else if (!dp.HasChanges()) {
+               return;
+            }
+            {
+               using var outStream = File.OpenWrite(file);
+               await ParquetTableWriter.WriteAsync(dp.newRows, outStream);
+            }
+            // Fire the update at the beginning of the next doWork call so it happens on the
+            // 'work' thread.
+            pendingOutput = dp;
+         }
+      }
+   }
+
+   protected override void myOnLastSubscriber() =>
+      Disposable.DisposeThreadSafe(ref dispose);
+}

src/DataStreamSuite/DataStream/BlueOrb/PythonDataframeStream.cs

@@ -1,33 +1,120 @@
 namespace bts.BlueOrb {
    using BCG;
-   using System.IO;
-   using System.Diagnostics;
+
+   using bts.BlueOrb.Parquet;
+
+   using Bts.DataStream;
+   using Bts.DataStreamCore;
+   using Bts.PubSub;
+
+   using SharpCompress.Writers;
+
+   using Spark.Core;
+   using Spark.Core.Disposables;
+
    using System;
+   using System.Diagnostics;
+   using System.IO;
+   using System.Reactive.Linq;
+   using System.Threading.Tasks;
 
    [DataStreamUserRespresentation("PythonDataframeStream")]
-   public class PythonDataframeStream : ViewStream {
+   public sealed class PythonDataframeStream : ViewStream, IDisposable {
       private ViewStream _parent_str;
-      private readonly string _in_csv_file_name;
-      private readonly string _out_csv_filename;
       private readonly string _transform_function;
-      public PythonDataframeStream(ViewStream parent_str, string transform_function) {
-         var guid = System.Guid.NewGuid();
-         _in_csv_file_name = $"pdfs_in_{guid}.csv";
-         _out_csv_filename = $"pdfs_out_{guid}.csv";
-         _parent_str = new CSVSaverStream(parent_str, _in_csv_file_name);
+      readonly string pyScriptPath = $"py_df_stream_{Guid.NewGuid()}.py";
+      readonly IWorkScheduler workScheduler;
+      readonly object syncRoot = new();
+      readonly MemoryStream writeBuffer = new();
+      IBlueTableSchema? outputSchema;
+      enum State {
+         Idle,
+         Busy,
+         Stopping,
+         Disposed
+      }
+      State state = State.Idle;
+      IDisposable? dispose;
+
+      public PythonDataframeStream(ViewStream parent_str, string transform_function)
+         : this(parent_str, transform_function, new WorkScheduler(true, 1, TimeSpan.FromMilliseconds(100))) { }
+
+      public PythonDataframeStream(ViewStream parent_str, string transform_function, IWorkScheduler workScheduler) {
+         _parent_str = parent_str;
          _transform_function = transform_function;
+         this.workScheduler = workScheduler;
+      }
+
+      ~PythonDataframeStream() =>
+         Dispose();
+
+      // Idempotent dispose
+      public void Dispose() {
+         GC.SuppressFinalize(this);
+         lock (syncRoot) {
+            switch (state) {
+               case State.Disposed:
+                  return;
+               case State.Busy:
+                  state = State.Stopping;
+                  return;
+               case State.Idle:
+                  break;
+               default:
+                  Log.assert(false, $"Unexpected state ({state}) when Dispose called");
+                  return;
+            }
+            state = State.Disposed;
+         }
+         Disposable.dispose(ref dispose);
       }
 
       protected override void myOnFirstSubscriber() {
-         _parent_str.onChange += _parent_str_onchange;
+         lock (syncRoot) {
+            ObjectDisposedException.ThrowIf(state == State.Disposed, this);
+            Log.isnull(dispose);
+            var scriptText = $"""
+            import sys
+            import pandas as pd
+            import io
+
+            {_transform_function}
+
+            if __name__ == '__main__':
+            # 1. read input from stdin to df
+            # 2. do some test calc
+            # 3. write the output to stdout
+               try:
+                  bytesio_in = io.BytesIO(sys.stdin.buffer.read())
+                  df = pd.read_parquet(bytesio_in)
+                  df_out = transform(df)
+                  bytesio_out = io.BytesIO()
+                  df_out.to_parquet(bytesio_out, index=False)
+                  sys.stdout.buffer.write(bytesio_out.getvalue())
+               except Exception as e:
+                  import traceback
+                  sys.stderr.write(traceback.format_exc())
+                  sys.stderr.write(str(sys.exc_info()[2]))
+                  exit(1)
+            """;
+            var pyScriptTempFile = TemporaryFile.WriteAllText(pyScriptPath, scriptText);
+            dispose = Disposable.Create(
+               Disposable.Create(DeletePyScriptTempFile),
+               _parent_str.AsObservable().NonIncrementalAsyncTransform(transformAsync, workScheduler).Subscribe(myFireEvent));
+
+            void DeletePyScriptTempFile() {
+                              
+            }
+         }
       }
 
       protected override void myOnLastSubscriber() {
-         _parent_str.onChange -= _parent_str_onchange;
+         lock (syncRoot) {
+            Disposable.dispose(ref dispose);
+         }
       }
 
-      private void _parent_str_onchange(object dp_obj) {
-         Log.assert(dp_obj != null);
+      async Task<IBlueRowCollection?> transformAsync(IBlueRowCollection rowsIn) {
          // the file _in_csv_file_name is now populated.
          // call py_func_wrapper_main and pass this file name as well
          // as the output file name
@@ -36,46 +123,87 @@
          // writes it into the _out_csv_filename file.
          // now this function should read _out_csv_filename into a dp
          // and fire it
-         _compute_and_write();
-         ViewStream rs = new SchemaGuesserStream(new CSVReaderStream(_out_csv_filename));
-         IBlueRowCollection nrows = null;
-         rs.onChange += (dp_obj) => {
-            nrows = (dp_obj as DataPacket).newRows;
-         };
-         var dp = BlueDataUtility.build_dp(nrows, myLastValue as DataPacket);
-         myFireEvent(dp);
-      }
-
-      private void _compute_and_write() {
-         File.WriteAllText("user_code.py", _transform_function);
-         // wrapper function
-         string py_func_wrapper_main = @"
-import sys
-from user_code import transform
-import pandas as pd
-
-if __name__ == '__main__':
-   input_file_name = sys.argv[1]
-   output_file_name = sys.argv[2]
-# 1. load input_file_name into df
-# 2. do some test calc
-# 3. write the output of 2 into output_file_name
-   df = pd.read_csv(input_file_name)   
-   df_out = transform(df)
-   df_out.to_csv(output_file_name, index=False)";
-
-
-         File.WriteAllText("runner.py", py_func_wrapper_main);
-
+         lock (syncRoot) {
+            switch (state) {
+               case State.Idle:
+                  break;
+               default:
+                  Log.assert(false, $"PythonDataframeStream.transformAsync called with unexpected state ({state})");
+                  return null;
+            }
+            state = State.Busy;
+         }
          ProcessStartInfo psi = new ProcessStartInfo {
             FileName = "python",
-            Arguments = $"runner.py {_in_csv_file_name} {_out_csv_filename}",
-            RedirectStandardOutput = false,
+            Arguments = $"{pyScriptPath}",
+            RedirectStandardInput = true,
+            RedirectStandardOutput = true,
+            RedirectStandardError = true,
             UseShellExecute = false
          };
          var proc = Process.Start(psi);
-         proc.WaitForExit();
-
+         Log.notnull(proc);
+         try {
+            writeBuffer.SetLength(0);
+            writeBuffer.Position = 0;
+            var stdIn = proc.StandardInput.BaseStream;
+            await ParquetTableWriter.WriteAsync(rowsIn, writeBuffer, true);
+            writeBuffer.Position = 0;
+            await writeBuffer.CopyToAsync(stdIn);
+            proc.StandardInput.Close();
+         } catch (Exception ex) {
+            Log.assert(false, $"Exception writing parquet to py stdin: {ex}");
+         }
+         bool ok = true;
+         Exception? err = null;
+         IBlueRowCollection? result = null;
+         try {
+            using var outputStream = new MemoryStream();
+            proc.StandardOutput.BaseStream.CopyTo(outputStream);
+            proc.WaitForExit();
+            result = await ParquetTableReader.ReadAsync(outputStream);
+         } catch (Exception ex) {
+            ok = false;
+            err = ex;
+         }
+         bool shouldDispose = false;
+         lock (syncRoot) {
+            switch (state) {
+               case State.Busy:
+                  state = State.Idle;
+                  break;
+               case State.Stopping:
+                  shouldDispose = true;
+                  break;
+               default:
+                  Log.assert(false, $"PythonDataframeStream.transformAsync unexpected state after py proc exited ({state})");
+                  return null;
+            }
+         }
+         var exitCode = proc.ExitCode;
+         if (exitCode != 0) {
+            var stdErr = proc.StandardError.ReadToEnd();
+            Log.Gen.error(
+               $"PythonDataframeStream proc exited with code {exitCode}. stderr (below):\n{stdErr}");
+         }
+         Log.assert(ok, $"Exception reading parquet from py stdout: {err}");
+         Log.notnull(result);
+         if (shouldDispose) {
+            lock (syncRoot) {
+               state = State.Disposed;
+            }
+            Disposable.dispose(ref dispose);
+         }
+         // Empty ParquetTables don't have valid schema (we give it a default schema which is simply
+         // an 'id' column and nothing else). If we receive an empty table BEFORE we know the
+         // output schema, return null so downstream handlers don't bootstrap themselves with an
+         // invalid/empty schema. Get the output schema from the first non-empty table so we can
+         // fill in the schema for empty tables after that.
+         if (result.Count == 0) {
+            return outputSchema is null ? null : new BlueTableImmutable(outputSchema);
+         }
+         outputSchema ??= result.schema;
+         return result;
       }
 
       #region Unused test code
@@ -104,7 +232,7 @@ if __name__ == '__main__':
    rv = rv + 12
    with open(out_path, 'w') as f:to \
       f.write(str(rv))";
-   
+
          File.WriteAllText("runner.py", py_func_wrapper_main);
 
          string outputPath = "py_out.txt";

src/DataStreamSuite/DataStream/BlueOrb/ReplayStream.cs

@@ -0,0 +1,193 @@
+namespace bts.BlueOrb;
+
+using BCG;
+
+using Bts.DataStream;
+using Bts.DataStreamCore;
+using Bts.PubSub;
+
+using Spark.Core;
+using Spark.Core.Collections;
+using Spark.Core.Disposables;
+using Spark.Core.Reactive;
+
+using System;
+using System.Collections.Generic;
+using System.Collections.Immutable;
+using System.Diagnostics.CodeAnalysis;
+using System.Linq;
+using System.Reactive.Linq;
+using System.Threading;
+using System.Threading.Tasks;
+
+public class ReplayStream : ViewStream {
+   public ReplayStream(IEnumerable<IBlueRow> source, string timestampColumn, double replaySpeed, IWorkScheduler workScheduler) {
+      mySetRealSource(ViewStreamFactory.Create(Subscribe));
+
+      IDisposable Subscribe(Action<IDataPacket> onData) {
+         int timestampIndex = -1;
+         IEnumerator<IBlueRow>? en = null;
+         TimeSpan? timeOffset = null;
+         BlueTableImmutable? lastTable = null;
+         var pendingUpdates = ImmutableList<(List<(DateTime? Timestamp, IBlueRow Row)> Records, int Pos)>.Empty;
+         var pendingRows = ImmutableSortedSet.Create(TupleComparer.Comparer<DateTime?, IBlueRow>(0, 1).AsComparer());
+         IDisposable? workSub = null;
+         workSub = workScheduler.Subscribe(DoWork);
+         return Disposable.Create(Dispose);
+
+         void DoWork() {
+            var now = DateTime.Now;
+            en ??= source.GetEnumerator();
+            if (lastTable is null) {
+               if (!en.MoveNext()) {
+                  Disposable.dispose(ref workSub);
+                  return;
+               }
+               lastTable = new BlueTableImmutable(en.Current.schema);
+            }
+            var bd = lastTable.ToDataPacketBuilder();
+            do {
+               var ts = GetTimeStamp(en.Current);
+               if (!ts.HasValue) {
+                  continue;
+               }
+               timeOffset ??= now - ts.Value;
+               if (ts.Value + timeOffset.Value > now) {
+                  break;
+               }
+               bd.AddOrUpdate(en.Current);
+            } while (Advance());
+            var dpOut = bd.ToDataPacket(out lastTable);
+            if (dpOut.HasChanges()) {
+               onData(dpOut);
+            }
+
+            bool Advance() {
+               if (en.MoveNext()) {
+                  return true;
+               }
+               Disposable.dispose(ref workSub);
+               return false;
+            }
+         }
+
+         DateTime? GetTimeStamp(IBlueRow r) {
+            if (timestampIndex == -1) {
+               timestampIndex = r.schema.ResolveIndices(timestampColumn);
+            }
+            var ts = r.GetDateTime(timestampIndex);
+            if (replaySpeed > 0 && replaySpeed != 1) {
+               // Divide timestamps by replaySpeed to control the speed at which rows are emitted.
+               // This works because it also divides the time deltas between rows. For example, if
+               // the replaySpeed is 2, rows will be emitted twice as fast.
+               ts = ts?.Divide(replaySpeed);
+            }
+            return ts;
+         }
+
+         void Dispose() {
+            Disposable.dispose(ref workSub);
+         }
+      }
+   }
+
+   public static ViewStream Create(
+      IEnumerable<Func<Task<IBlueRowCollection>>> tableFactories,
+      string timestampColumn,
+      double replaySpeed,
+      IWorkScheduler workScheduler
+   ) {
+      return ViewStreamFactory.Create(Subscribe);
+
+      IDisposable Subscribe(Action<IDataPacket> onData) {
+         int? timestampIndex = null;
+         IEnumerator<IBlueRow>? en = null;
+         TimeSpan? timeOffset = null;
+         BlueTableImmutable? lastTable = null;
+         int tableIdx = 0;
+         var tableFactoryList = tableFactories.ToList();
+         int isBusy = 0;
+         IDisposable? workSub = null;
+         workSub = workScheduler.Subscribe(DoWork);
+         return Disposable.Create(Dispose);
+
+         /// This method acts as a gate to prevent concurrent DoWork calls (since DoWork is async).
+         bool TryBegin([NotNullWhen(true)] out IDisposable? end) {
+            if (Interlocked.CompareExchange(ref isBusy, 1, 0) == 0) {
+               end = Disposable.Create(End);
+               return true;
+            }
+            end = null;
+            return false;
+
+            void End() =>
+               Log.assert(Interlocked.CompareExchange(ref isBusy, 0, 1) == 1);
+         }
+
+         async void DoWork() {
+            if (!TryBegin(out var end)) {
+               return;
+            }
+            using (end) {
+               // Load the next table if necessary
+               while (en is null) {
+                  if (tableIdx == tableFactoryList.Count) {
+                     Disposable.dispose(ref workSub);
+                     return;
+                  }
+                  var nextTable = await tableFactoryList[tableIdx++]();
+                  var nextEn = nextTable.GetEnumerator();
+                  if (nextEn.MoveNext()) {
+                     en = nextEn;
+                  }
+               }
+               lastTable ??= new BlueTableImmutable(en.Current.schema);
+               var bd = lastTable.ToDataPacketBuilder();
+               var now = DateTime.Now;
+               // Fire rows whose timestamps are within the elapsed window (now - start).
+               do {
+                  var ts = GetTimeStamp(en.Current);
+                  if (!ts.HasValue) {
+                     continue;
+                  }
+                  timeOffset ??= now - ts.Value;
+                  if (ts.Value + timeOffset.Value > now) {
+                     break;
+                  }
+                  bd.AddOrUpdate(en.Current);
+               } while (Advance());
+               var dpOut = bd.ToDataPacket(out lastTable);
+               if (dpOut.HasChanges()) {
+                  onData(dpOut);
+               }
+
+               bool Advance() {
+                  if (en.MoveNext()) {
+                     return true;
+                  }
+                  en = null;
+                  timestampIndex = null;
+                  tableIdx++;
+                  return false;
+               }
+            }
+         }
+
+         DateTime? GetTimeStamp(IBlueRow r) {
+            timestampIndex ??= r.schema.ResolveIndices(timestampColumn);
+            var ts = r.GetDateTime(timestampIndex.Value);
+            if (replaySpeed > 0 && replaySpeed != 1) {
+               // Divide timestamps by replaySpeed to control the speed at which rows are emitted.
+               // This works because it also divides the time deltas between rows. For example, if
+               // the replaySpeed is 2, rows will be emitted twice as fast.
+               ts = ts?.Divide(replaySpeed);
+            }
+            return ts;
+         }
+
+         void Dispose() {
+            Disposable.dispose(ref workSub);
+         }
+      }
+   }
+}

src/DataStreamSuite/DataStream/BlueOrb/SelectorStream.cs

@@ -0,0 +1,42 @@
+namespace bts.BlueOrb;
+
+using BCG;
+
+using Bts.DataStream;
+
+using Spark.Core.Disposables;
+
+using System;
+
+public class SelectorStream : ViewStream, IDisposable {
+   readonly ViewStream source;
+   readonly IBlueTableSchema outputSchema;
+   readonly Func<IBlueRow, IBlueRow> selector;
+
+   IDisposable? sub;
+
+   public SelectorStream(ViewStream source, IBlueTableSchema outputSchema, Func<IBlueRow, IBlueRow> selector) {
+      this.source = source;
+      this.outputSchema = outputSchema;
+      this.selector = selector;
+   }
+
+   public void Dispose() {
+      GC.SuppressFinalize(this);
+      Disposable.dispose(ref sub);
+   }
+
+   protected override void myOnFirstSubscriber() {
+      Disposable.dispose(ref sub);
+      sub = source.Subscribe(OnInputChanged);
+
+      void OnInputChanged(IDataPacket dp) =>
+         myFireEvent(new DataPacket(
+            new SelectorTable(dp.newRows, outputSchema, selector),
+            new SelectorTable(dp.addedRows, outputSchema, selector),
+            new SelectorTable(dp.deletedRows, outputSchema, selector)));
+   }
+
+   protected override void myOnLastSubscriber() =>
+      Disposable.dispose(ref sub);
+}

src/DataStreamSuite/DataStream/BlueOrb/SelectorTable.cs

@@ -0,0 +1,58 @@
+namespace bts.BlueOrb;
+
+using BCG;
+
+using Bts.DataStreamCore;
+
+using System;
+using System.Collections.Generic;
+
+/// <summary>
+/// Represents a row transformation table (all columns except 'id' can be changed).
+/// </summary>
+public class SelectorTable : IBlueRowCollection {
+   readonly IBlueRowCollection source;
+   readonly IBlueTableSchema outputSchema;
+   readonly Func<IBlueRow, IBlueRow> selector;
+
+   public SelectorTable(IBlueRowCollection source, IBlueTableSchema outputSchema, Func<IBlueRow, IBlueRow> selector) {
+      this.source = source;
+      this.outputSchema = outputSchema;
+      this.selector = selector;
+   }
+
+   public IBlueRow this[int id] =>
+      selector(source[id]);
+
+   IBlueRow IList<IBlueRow>.this[int index] {
+      get => GetRow(index);
+      set => throw new NotSupportedException();
+   }
+
+   public IBlueTableSchema schema => outputSchema;
+
+   public int Count => source.Count;
+
+   public bool containsID(int id) =>
+      source.containsID(id);
+
+   public IEnumerator<IBlueRow> GetEnumerator() {
+      foreach (var r in source) {
+         yield return selector(r);
+      }
+   }
+
+   public IBlueRow GetRow(int index) =>
+      selector(GetRow(index));
+
+   public int IndexOf(IBlueRow item) {
+      int id = item.Id();
+      if (!source.containsID(id)) {
+         return -1;
+      }
+      var inputRow = source[id];
+      return BlueRowEqualityComparer.Equals(selector(inputRow), item)
+         ? source.IndexOf(inputRow)
+         : -1;
+   }
+}

src/DataStreamSuite/DataStream/BlueOrb/ShiftStream.cs

@@ -0,0 +1,19 @@
+namespace bts.BlueOrb;
+
+using BCG;
+
+using Bts.DataStream;
+
+using System.Reactive.Linq;
+
+[DataStreamUserRespresentation("shift")]
+public class ShiftStream : ViewStream {
+   public ShiftStream(ViewStream source, int offset) {
+      mySetRealSource(source.AsObservable()
+         .Select(dp => new DataPacket(
+            new ShiftTable(dp.newRows, offset),
+            new ShiftTable(dp.addedRows, offset),
+            new ShiftTable(dp.deletedRows, offset)))
+         .AsViewStream());
+   }
+}

src/DataStreamSuite/DataStream/BlueOrb/ShiftTable.cs

@@ -0,0 +1,150 @@
+namespace bts.BlueOrb;
+
+using BCG;
+
+using Bts.DataStreamCore;
+
+using System.Collections.Generic;
+
+public class ShiftTable : IBlueRowCollection {
+   readonly IBlueRowCollection source;
+   readonly int shift;
+   readonly int idColIndex;
+
+   public ShiftTable(IBlueRowCollection source, int shift) {
+      this.source = source;
+      this.shift = shift;
+      idColIndex = source.schema.ResolveIndices("id");
+   }
+
+   public IBlueRow this[int id] {
+      get {
+         var row = source[id];
+         int idx = source.IndexOf(row);
+         return shift switch {
+            > 0 => new Row(schema, id, idColIndex, idx >= shift ? source.GetRow(idx - shift) : null),
+            < 0 => new Row(schema, id, idColIndex, idx < Count + shift ? source.GetRow(idx - shift) : null),
+            0 => row,
+         };
+      }
+   }
+
+   public IBlueTableSchema schema => source.schema;
+
+   public int Count => source.Count;
+
+   public bool containsID(int id) =>
+      source.containsID(id);
+
+   public IEnumerator<IBlueRow> GetEnumerator() {
+      return shift switch {
+         > 0 => ShiftUpImpl(),
+         < 0 => ShiftDownImpl(),
+         0 => source.GetEnumerator(),
+      };
+
+      IEnumerator<IBlueRow> ShiftUpImpl() {
+         var buffer = new List<IBlueRow>();
+         int bufferPos = 0;
+         // Columns have been "shifted up", so output row vals come from earlier rows.
+         foreach (var r in source) {
+            if (buffer.Count < shift) {
+               yield return new Row(schema, r.Id(), idColIndex, null);
+               buffer.Add(r);
+            } else {
+               int rowValsPos = (bufferPos == 0 ? shift : bufferPos) - 1;
+               yield return new Row(schema, r.Id(), idColIndex, buffer[rowValsPos]);
+               buffer[bufferPos] = r;
+            }
+            if (++bufferPos == shift) {
+               bufferPos = 0;
+            }
+         }
+      }
+
+      IEnumerator<IBlueRow> ShiftDownImpl() {
+         var buffer = new List<IBlueRow>();
+         int bufferPos = 0;
+         int shift = -this.shift;
+         // Columns have been shifted down, so output row vals come from later rows.
+         foreach (var r in source) {
+            if (buffer.Count < shift) {
+               buffer.Add(r);
+            } else {
+               int rowIdPos = bufferPos == shift - 1 ? 0 : bufferPos + 1;
+               yield return new Row(schema, buffer[rowIdPos].Id(), idColIndex, r);
+               buffer[bufferPos] = r;
+            }
+            if (++bufferPos == shift) {
+               bufferPos = 0;
+            }
+         }
+         if (buffer.Count < shift) {
+            // Column values have been shifted down by an amount that exceeds the total row count.
+            // All values (other than 'id') are nan/null.
+            for (int i = 0; i < buffer.Count; i++) {
+               var r = buffer[i];
+               yield return new Row(schema, r.Id(), idColIndex, null);
+            }
+         } else {
+            int i = bufferPos;
+            while (i < shift) {
+               var r = buffer[i];
+               yield return new Row(schema, r.Id(), idColIndex, null);
+               i++;
+            }
+            for (i = 0; i < bufferPos; i++) {
+               var r = buffer[i];
+               yield return new Row(schema, r.Id(), idColIndex, null);
+            }
+         }
+      }
+   }
+
+   public IBlueRow GetRow(int index) =>
+      source.GetRow(index - shift);
+
+   public int IndexOf(IBlueRow item) {
+      int idx = source.IndexOf(item);
+      if (idx != -1) {
+         idx -= shift;
+      }
+      return idx;
+   }
+
+   class Row : IBlueRow {
+      readonly IBlueTableSchema schema;
+      readonly int id;
+      readonly int idColIndex;
+      readonly IBlueRow? values;
+
+      public Row(IBlueTableSchema schema, int id, int idColIndex, IBlueRow? values) {
+         this.schema = schema;
+         this.id = id;
+         this.idColIndex = idColIndex;
+         this.values = values;
+      }
+
+      IBlueTableSchema IBlueRow.schema => schema;
+
+      public object? this[int index] {
+         get {
+            if (index == idColIndex) {
+               return id;
+            }
+            if (values is not null) {
+               return values[index];
+            }
+            var cols = schema.columnInfo;
+            var colType = cols[index].colType;
+            if (colType == typeof(double)) {
+               return double.NaN;
+            }
+            if (colType == typeof(float)) {
+               return float.NaN;
+            }
+            return null;
+         }
+      }
+   }
+}

src/DataStreamSuite/DataStream/BlueOrb/SliceStream.cs

@@ -0,0 +1,33 @@
+namespace bts.BlueOrb;
+
+using BCG;
+
+using Bts.DataStream;
+using Bts.DataStreamCore;
+
+using Spark.Core.Collections;
+
+using System;
+using System.Collections.Generic;
+using System.Reactive.Linq;
+
+[DataStreamUserRespresentation("slice")]
+public class SliceStream : ViewStream {
+   public SliceStream(ViewStream inputStream, int? start, int? stop) {
+      mySetRealSource(Observable.Create<IDataPacket>(Subscribe).AsViewStream());
+
+      IDisposable Subscribe(IObserver<IDataPacket> o) {
+         var rows = new SliceList<IBlueRow>(new List<IBlueRow>(0), start, stop);
+         return inputStream.Subscribe(OnData);
+
+         void OnData(IDataPacket dp) {
+            rows = rows.OnChange(dp.newRows, dp.deletedRows, dp.addedRows, out var sliceDeletes, out var sliceAdds);
+            var dpOut = new DataPacket(
+               new BlueTableListWrapper(dp.Schema, rows),
+               new BlueTableListWrapper(dp.Schema, sliceAdds),
+               new BlueTableListWrapper(dp.Schema, sliceDeletes));
+            o.OnNext(dpOut);
+         }
+      }
+   }
+}

src/DataStreamSuite/DataStream/BlueOrb/SliceTable.cs

@@ -0,0 +1,74 @@
+namespace bts.BlueOrb;
+
+using BCG;
+
+using bts.collections;
+
+using Spark.Core.Collections;
+
+using System;
+using System.Collections.Generic;
+using System.Linq;
+
+public class SliceTable : IBlueRowCollection {
+   readonly IBlueRowCollection inputRows;
+   readonly int? start;
+   readonly int? stop;
+
+   public SliceTable(IBlueRowCollection inputRows, int? start, int? stop) {
+      this.inputRows = inputRows;
+      this.start = start;
+      this.stop = stop;
+   }
+
+   public IBlueRow this[int id] {
+      get {
+         var r = inputRows[id];
+         int idx = inputRows.IndexOf(r);
+         return idx >= startIndex && idx < stopIndex
+            ? r
+            : throw new KeyNotFoundException();
+      }
+   }
+
+   public IBlueTableSchema schema => inputRows.schema;
+
+   public int Count => Math.Max(0, stopIndex - startIndex);
+
+   IBlueRow IList<IBlueRow>.this[int index] {
+      get => GetRow(index);
+      set => throw new NotSupportedException();
+   }
+
+   int startIndex => SliceList.GetStartIndex(start, inputRows.Count);
+   int stopIndex => SliceList.GetStopIndex(stop, inputRows.Count);
+
+   public bool containsID(int id) {
+      var r = inputRows[id];
+      int idx = inputRows.IndexOf(r);
+      return idx >= startIndex && idx < stopIndex;
+   }
+
+   public IEnumerator<IBlueRow> GetEnumerator() {
+      foreach (var r in inputRows.Skip(startIndex).Take(stopIndex - startIndex)) {
+         yield return r;
+      }
+   }
+
+   public IBlueRow GetRow(int index) =>
+      index >= 0 && index < Count
+         ? inputRows.GetRow(startIndex + index)
+         : throw new IndexOutOfRangeException();
+
+   public IList<object?> GetColumn(int index) {
+      var colWrapper = this.WrapSelector(r => r[index], null);
+      return colWrapper;
+   }
+
+   public int IndexOf(IBlueRow item) {
+      int idx = inputRows.IndexOf(item);
+      return idx >= startIndex && idx < stopIndex
+         ? idx - startIndex
+         : -1;
+   }
+}

src/DataStreamSuite/DataStream/BlueOrb/SortStream.cs

@@ -1,17 +1,51 @@
-namespace bts.BlueOrb {
-   using BCG;
+namespace bts.BlueOrb;
 
-   using System;
-   using System.Collections.Generic;
-   using System.Linq;
-   using System.Text;
-   using System.Threading.Tasks;
+using BCG;
 
-   [DataStreamUserRespresentation("SortStream")]
-   public class SortStream: ViewStream {
-      public SortStream (ViewStream parent_stream, string sort_expr) {
-         ViewStream rs = new BCG.SortStream(parent_stream, sort_expr);
-         mySetRealSource(rs);
+using bts.utils;
+
+using Bts.DataStream;
+
+using mbts.utils;
+
+using System.Reactive.Linq;
+
+[DataStreamUserRespresentation("sort")]
+public class SortStream : ViewStream {
+   readonly ViewStream source;
+   readonly bool isAscending;
+
+   public SortStream(ViewStream source, params string[] sortColumns)
+      : this(source, true, sortColumns) { }
+
+   public SortStream(ViewStream source, bool isAscending, params string[] sortColumns) {
+      this.source = source;
+      this.isAscending = isAscending;
+      mySetRealSource(source.AsObservable().Scan<IDataPacket, State?>(null, Sort).Select(t => t!.Value.Update).AsViewStream());
+
+      State? Sort(State? last, IDataPacket dpObj) {
+         var dp = ExceptionAssert.Argument.Is<IDataPacket>(dpObj);
+         if (!last.HasValue) {
+            var comp = BlueRowComparer.HierarchicalColumnComparer(dp.Schema.columnInfo, sortColumns);
+            if (!isAscending) {
+               comp = comp.reverse();
+            }
+            last = new(new DataPacket(dpObj.Schema), new(dpObj.Schema, comp));
+         }
+         var bd = last.Value.Table.ToBuilder();
+         bd.Merge(dp);
+         var dpOut = bd.ToDataPacket(out var table);
+         return new(dpOut, table);
       }
    }
+
+   public SortStream WithSortColumns(params string[] sortColumns) =>
+      new(source, isAscending, sortColumns);
+
+   public SortStream WithSortColumns(bool isAscending, params string[] sortColumns) =>
+      new(source, isAscending, sortColumns);
+
+   record struct State(
+      IDataPacket Update,
+      SortedTable Table);
 }

src/DataStreamSuite/DataStream/BlueOrb/SortedTable.cs

@@ -0,0 +1,118 @@
+namespace bts.BlueOrb;
+
+using BCG;
+
+using Bts.DataStreamCore;
+
+using Spark.Core.Collections;
+
+using System.Collections.Generic;
+using System.Collections.Immutable;
+
+public class SortedTable : IBlueRowCollection {
+   readonly ImmutableSortedSet<IBlueRow> sortedRows;
+   readonly ImmutableDictionary<int, IBlueRow> cachedRows;
+   // readonly ImmutableSortedSet<KeyValuePair<long, int>> ordinalToId;
+   // readonly ImmutableDictionary<int, long> idToOrdinal;
+
+   public SortedTable(IBlueTableSchema schema, IComparer<IBlueRow> comparer)
+      : this(schema, ImmutableSortedSet.Create(comparer), ImmutableDictionary<int, IBlueRow>.Empty) { }
+
+   public SortedTable(IBlueTableSchema schema, ImmutableSortedSet<IBlueRow> sortedRows, ImmutableDictionary<int, IBlueRow> cachedRows) {
+      this.schema = schema;
+      this.sortedRows = sortedRows;
+      this.cachedRows = cachedRows;
+   }
+
+   public IBlueRow this[int id] => cachedRows[id];
+
+   public IBlueTableSchema schema { get; }
+
+   public IComparer<IBlueRow> Comparer => sortedRows.KeyComparer;
+
+   public int Count => cachedRows.Count;
+
+   public bool containsID(int id) =>
+      cachedRows.ContainsKey(id);
+
+   public IEnumerator<IBlueRow> GetEnumerator() =>
+      sortedRows.GetEnumerator();
+
+   public IBlueRow GetRow(int index) =>
+      sortedRows[index];
+
+   public int IndexOf(IBlueRow item) =>
+      sortedRows.IndexOf(item);
+
+   public Builder ToBuilder() =>
+      new(schema, sortedRows, cachedRows);
+
+   public class Builder {
+      readonly IBlueTableSchema schema;
+      readonly ImmutableSortedSet<IBlueRow> originalSortedRows;
+      readonly ImmutableDictionary<int, IBlueRow> originalCachedRows;
+      readonly ImmutableDictionary<int, IBlueRow>.Builder cachedDeletedRows = ImmutableDictionary.CreateBuilder<int, IBlueRow>();
+      readonly ImmutableDictionary<int, IBlueRow>.Builder cachedAddedRows = ImmutableDictionary.CreateBuilder<int, IBlueRow>();
+
+      public Builder(
+         IBlueTableSchema schema,
+         ImmutableSortedSet<IBlueRow> sortedRows,
+         ImmutableDictionary<int, IBlueRow> cachedRows) {
+         this.schema = schema;
+         originalSortedRows = sortedRows;
+         originalCachedRows = cachedRows;
+      }
+
+      public void Merge(IDataPacket dp) {
+         foreach (var r in dp.deletedRows) {
+            Remove(r.Id());
+         }
+         foreach (var r in dp.addedRows) {
+            AddOrUpdate(r);
+         }
+      }
+
+      public bool TryAdd(IBlueRow row) {
+         int id = row.Id();
+         if (originalCachedRows.ContainsKey(id)) {
+            return false;
+         }
+         cachedAddedRows[id] = row;
+         return true;
+      }
+
+      public void AddOrUpdate(IBlueRow row) {
+         int id = row.Id();
+         if (originalCachedRows.TryGetValue(id, out var cachedRow)) {
+            cachedDeletedRows[id] = cachedRow;
+         }
+         cachedAddedRows[row.Id()] = row;
+      }
+
+      public void Remove(int id) {
+         if (originalCachedRows.TryGetValue(id, out var cachedRow)) {
+            cachedDeletedRows[id] = cachedRow;
+         }
+         cachedAddedRows.Remove(id);
+      }
+
+      public IDataPacket ToDataPacket(out SortedTable allRows, bool reset = true) {
+         var sortedDeletedRows = ImmutableSortedSet.CreateRange(originalSortedRows.KeyComparer, cachedDeletedRows.Values);
+         var sortedAddedRows = ImmutableSortedSet.CreateRange(originalSortedRows.KeyComparer, cachedAddedRows.Values);
+         var sortedRowsBd = originalSortedRows.ToBuilder();
+         var cachedRowsBd = originalCachedRows.ToBuilder();
+         sortedRowsBd.RemoveRange(sortedDeletedRows);
+         cachedRowsBd.RemoveRange(cachedDeletedRows.Keys);
+         sortedRowsBd.AddRange(sortedAddedRows);
+         cachedRowsBd.AddRange(cachedAddedRows);
+         var deletedTable = new SortedTable(schema, sortedDeletedRows, cachedDeletedRows.ToImmutable());
+         var addedTable = new SortedTable(schema, sortedAddedRows, cachedAddedRows.ToImmutable());
+         allRows = new(schema, sortedRowsBd.ToImmutable(), cachedRowsBd.ToImmutable());
+         if (reset) {
+            cachedDeletedRows.Clear();
+            cachedAddedRows.Clear();
+         }
+         return new DataPacket(allRows, addedTable, deletedTable);
+      }
+   }
+}

src/DataStreamSuite/DataStream/BlueOrb/SumTable.cs

@@ -0,0 +1,107 @@
+namespace bts.BlueOrb;
+
+using BCG;
+
+using bts.collections;
+
+using Bts.DataStreamCore;
+
+using Spark.Core.Collections;
+
+using System;
+using System.Collections.Generic;
+using System.Linq;
+
+class SumTable : IBlueRowCollection {
+   readonly IBlueRowCollection input;
+   readonly int period;
+   readonly IList<object?>[] columns;
+
+   public SumTable(IBlueRowCollection input, int period) {
+      this.input = input;
+      this.period = period;
+      var colInfos = input.schema.columnInfo.OrderBy(c => c.colNo).ToArray();
+      if (colInfos.Length > 0) {
+         Log.assert(colInfos[0].colNo == 0 && colInfos[^1].colNo == colInfos.Length - 1);
+      }
+      columns = colInfos.WrapSelector(c => GetColumn(c.colNo), null).ToArray();
+   }
+
+   public IBlueRow this[int id] {
+      get {
+         var inputRow = input[id];
+         var idx = input.IndexOf(inputRow);
+         var row = GetRow(idx);
+         return row;
+      }
+   }
+
+   public IBlueTableSchema schema => input.schema;
+
+   public int Count => input.Count;
+
+   public bool containsID(int id) =>
+      input.containsID(id);
+
+   public IList<object?> GetColumn(int colIndex) {
+      var count = input.Count;
+      var colInfo = input.schema.columnInfo[colIndex];
+      if (colInfo.colName == "id") {
+         return new StepList<int>(0, 1, count).as_mutable().WrapSelector(GetId, null);
+      }
+      var typeCode = colInfo.type_code();
+      return new StepList<int>(0, 1, count).as_mutable().WrapSelector(GetValue, null);
+
+      object? GetId(int rowIndex) {
+         var row = input.GetRow(rowIndex);
+         return row.Id();
+      }
+
+      object? GetValue(int rowIndex) {
+         var rightIndex = rowIndex - period;
+         if (rowIndex < 0 || rowIndex >= count || rightIndex < 0 || rightIndex >= count) {
+            return null;
+         }
+         var leftRow = input.GetRow(rowIndex);
+         var rightRow = input.GetRow(rightIndex);
+         return GetSum(leftRow[colIndex], rightRow[colIndex], typeCode);
+      }
+   }
+
+   public IEnumerator<IBlueRow> GetEnumerator() {
+      var count = input.Count;
+      for (int i = 0; i < count; i++) {
+         yield return GetRow(i);
+      }
+   }
+
+   public IBlueRow GetRow(int index) =>
+      new BlueRowColumnWrapper(input.schema, index, columns);
+
+   public int IndexOf(IBlueRow item) {
+      int id = item.Id();
+      var inputRow = input[id];
+      int idx = input.IndexOf(inputRow);
+      if (idx != -1) {
+         var row = GetRow(idx);
+         if (BlueRowEqualityComparer.Equals(item, row)) {
+            return idx;
+         }
+      }
+      return -1;
+   }
+
+   static object? GetSum(object? leftVal, object? rightVal, TypeCode colType) {
+      if (leftVal is null || rightVal is null) {
+         return null;
+      }
+      return colType switch {
+         TypeCode.Int32 => (int)leftVal + (int)rightVal,
+         TypeCode.Int64 => (long)leftVal + (long)rightVal,
+         TypeCode.Double => (double)leftVal + (double)rightVal,
+         TypeCode.String => ((string)leftVal) + ((string)rightVal, string.Empty),
+         TypeCode.DateTime => new DateTime(((DateTime)leftVal).Ticks + ((DateTime)rightVal).Ticks),
+         _ => null,
+      };
+   }
+}

src/DataStreamSuite/DataStream/BlueOrb/ThrottleStream.cs

@@ -0,0 +1,35 @@
+namespace bts.BlueOrb;
+
+using BCG;
+
+using Bts.DataStream;
+using Bts.DataStreamCore;
+
+using System;
+using System.Collections.Generic;
+using System.Reactive.Linq;
+
+public class ThrottleStream : ViewStream {
+   public ThrottleStream(ViewStream source, TimeSpan interval) {
+      var rowComp = new BlueRowComparer("id");
+      ImmutableOrderedTable? table = null;
+      mySetRealSource(source.AsObservable().Buffer(interval).Where(upds => upds.Count > 0).Select(Merge).AsViewStream());
+
+      IDataPacket Merge(IList<IDataPacket> upds) {
+         Log.assert(upds.Count > 0);
+         var lastDp = upds[^1];
+         table ??= new(lastDp.Schema);
+         var bd = table.ToBuilder();
+         foreach (var dp in upds) {
+            foreach (var r in dp.deletedRows) {
+               bd.Remove(r.Id());
+            }
+            foreach (var r in dp.addedRows) {
+               bd.AddOrUpdate(r);
+            }
+         }
+         table = bd.ToImmutable(out var dpOut);
+         return dpOut;
+      }
+   }
+}

src/DataStreamSuite/DataStream/BlueOrb/TimeSeriesStream.cs

@@ -1,18 +1,39 @@
-using System;
-
 using BCG;
 
+using bts.utils;
+
+using Bts.DataStreamCore;
+
+using mbts.utils;
+
+using Spark.Core.Collections;
+using Spark.Core.Threading;
+
+using System;
+using System.Collections.Generic;
+using System.Collections.Immutable;
+using System.Linq;
+
 namespace bts.BlueOrb {
    // Keeps adding changing rows to the stream, hence preserving the history
    // and creating a time series, for example this can be used to make bbo updates
-   // into a time series of prices/qty 
+   // into a time series of prices/qty
+   [DataStreamUserRespresentation("TimeSeriesStream")]
    public class TimeSeriesStream : ViewStream {
-      private BlueTable _table;
-      private ViewStream _stream;
+      readonly ViewStream _stream;
+      readonly string timeColumn;
+      readonly AtomicGate gate = new();
+      CircularBufferTable? _table;
       private int _id = 0;
+      private int _size = 0;
 
-      public TimeSeriesStream(ViewStream stream) {
+      public TimeSeriesStream(ViewStream stream)
+         : this(stream, Int32.MaxValue) { }
+
+      public TimeSeriesStream(ViewStream stream, int size, string timeColumn = "ts_event") {
          _stream = stream;
+         _size = size;
+         this.timeColumn = timeColumn;
       }
 
       protected override void myOnFirstSubscriber() {
@@ -24,28 +45,36 @@ namespace bts.BlueOrb {
 
 
       private void _onstream_change(object o) {
-         DataPacket dp = o as DataPacket;
+         var dp = ExceptionAssert.Argument.Is<IDataPacket>(o);
          DateTime now = DateTime.Now;
+         var addedRowsBd = ImmutableSortedSet.CreateBuilder(KeyValuePairComparer.KeyComparer<int, IBlueRow>());
+         var deletedRowsBd = ImmutableSortedSet.CreateBuilder(KeyValuePairComparer.KeyComparer<int, IBlueRow>());
+         using var _ = gate.BeginTransaction();
          if (_table == null) {
-            IBlueTableSchema schema = new BlueTableSchema(dp.newRows.schema.columnInfo, dp.newRows.schema.tableName);
-            schema = schema.add_columns(Tuple.Create(typeof(DateTime), "timestamp"));
-            _table = new BlueTable(schema);
+            var colInfos = dp.newRows.schema.columnInfo;
+            if (colInfos.TryGetFirst(c => c.colName == timeColumn, out var currTimeCol, out var currTimeColIdx)) {
+               colInfos = colInfos.ToArray();
+               colInfos[currTimeColIdx] = new(timeColumn, currTimeCol.colNo, typeof(DateTime));
+            } else {
+               colInfos = ArrayUtils.Add(colInfos, new(timeColumn, colInfos.Length, typeof(DateTime)));
+            }
+            _table = new CircularBufferTable(new BlueTableSchema(colInfos, dp.newRows.schema.tableName), _size);
          }
-         BlueTable added_rows = new BlueTable(_table.schema);
          foreach (IBlueRow r in dp.addedRows) {
             MutableBlueRow ar = new MutableBlueRow(_table.schema, r);
-            ar.setValue("timestamp", now);
-            ar.setValue("id", _id++);
-            added_rows.addRow(ar);
+            ar.setValue(timeColumn, now);
+            int id = _id++;
+            ar.setValue("id", id);
+            _table = _table.Add(ar, out var oldRow);
+            addedRowsBd.Add(new(id, ar));
+            if (oldRow is not null) {
+               deletedRowsBd.Add(new(oldRow.Id(), oldRow));
+            }
          }
-         _table.addRows(added_rows);
-         myFireEvent(new DataPacket(_table, added_rows, new BlueTable(_table.schema)));
+         var addedRows = new BlueTableImmutable(_table.schema, addedRowsBd.ToImmutable());
+         var deletedRows = new BlueTableImmutable(_table.schema, deletedRowsBd.ToImmutable());
+         var dp_out = new DataPacket(_table, addedRows, deletedRows);
+         myFireEvent(dp_out);
       }
    }
-
-
-
 }
-
-
-

src/DataStreamSuite/DataStream/BlueOrb/VarianceTable.cs

@@ -0,0 +1,113 @@
+namespace bts.BlueOrb;
+
+using BCG;
+
+using Bts.DataStreamCore;
+
+using MathNet.Numerics.Statistics;
+
+using System;
+using System.Collections.Generic;
+using System.Linq;
+
+public class VarianceTable : IBlueRowCollection {
+   readonly IBlueTableSchema schema;
+   readonly Lazy<IBlueRow> row;
+
+   public VarianceTable(IBlueRowCollection source) {
+      var inputSchema = source.schema;
+      var inputCols = inputSchema.columnInfo;
+      var outputCols = Array.ConvertAll(source.schema.columnInfo, ColumnFactory);
+      schema = new BlueTableSchema(outputCols, inputSchema.tableName + "_Variance");
+      row = new(RowFactory);
+
+      static ColumnInfo ColumnFactory(ColumnInfo inputCol) {
+         var t = inputCol.colType;
+         if (inputCol.colName != "id") {
+            if (t == typeof(short) || t == typeof(int) || t == typeof(long)
+               || t == typeof(ushort) || t == typeof(uint) || t == typeof(ulong)
+               || t == typeof(double) || t == typeof(float)) {
+               t = typeof(double);
+            } else if (t == typeof(DateTime)) {
+               t = typeof(TimeSpan);
+            }
+         }
+         return new(inputCol.colName, inputCol.colNo, t);
+      }
+
+      IBlueRow RowFactory() {
+         var rowVals = new object[inputCols.Length];
+         for (int i = 0; i < inputCols.Length; i++) {
+            var col = inputCols[i];
+            var colType = col.colType;
+            if (col.colName == "id") {
+               rowVals[i] = 1;
+            } else if (colType == typeof(short)) {
+               rowVals[i] = GetValue(v => v as short?);
+            } else if (colType == typeof(int)) {
+               rowVals[i] = GetValue(v => v as int?);
+            } else if (colType == typeof(long)) {
+               rowVals[i] = GetValue(v => v as long?);
+            } else if (colType == typeof(ushort)) {
+               rowVals[i] = GetValue(v => v as ushort?);
+            } else if (colType == typeof(uint)) {
+               rowVals[i] = GetValue(v => v as uint?);
+            } else if (colType == typeof(ulong)) {
+               rowVals[i] = GetValue(v => v as ulong?);
+            } else if (colType == typeof(double)) {
+               rowVals[i] = GetValue(v => v as double?);
+            } else if (colType == typeof(float)) {
+               rowVals[i] = GetValue(v => v as float?);
+            } else if (colType == typeof(DateTime)) {
+               rowVals[i] = TimeSpan.FromSeconds(GetValue(v => (v as DateTime?)?.Ticks / 10_000_000));
+            }
+
+            double GetValue(Func<object?, double?> convert) =>
+               source.GetColumn(i)
+                  .Select(convert)
+                  .SelectWhereValues()
+                  .Variance();
+         }
+         return new Row(schema, rowVals);
+      }
+   }
+
+   public IBlueRow this[int id] =>
+      id == 1 ? row.Value : throw new KeyNotFoundException();
+
+   IBlueTableSchema IBlueRowCollection.schema => schema;
+
+   public int Count => 1;
+
+   IBlueRow IList<IBlueRow>.this[int index] {
+      get => GetRow(index);
+      set => throw new NotSupportedException();
+   }
+
+   public bool containsID(int id) =>
+      id == 1;
+
+   public IEnumerator<IBlueRow> GetEnumerator() {
+      yield return row.Value;
+   }
+
+   public IBlueRow GetRow(int index) =>
+      index == 0 ? row.Value : throw new IndexOutOfRangeException();
+
+   public int IndexOf(IBlueRow item) =>
+      BlueRowEqualityComparer.Equals(item, row.Value) ? 0 : -1;
+
+   class Row : IBlueRow {
+      readonly IBlueTableSchema schema;
+      readonly object?[] values;
+
+      public Row(IBlueTableSchema schema, object?[] values) {
+         this.schema = schema;
+         this.values = values;
+      }
+
+      public object? this[int index] => values[index];
+
+      IBlueTableSchema IBlueRow.schema => schema;
+   }
+}

src/DataStreamSuite/DataStream/DataStream.csproj

@@ -22,7 +22,7 @@
     <PackageReference Include="Microsoft.Data.Analysis" Version="0.21.1" />
     <PackageReference Include="MongoDB.Bson" Version="2.23.1" />
     <PackageReference Include="MongoDB.Driver" Version="2.23.1" />
-    <PackageReference Include="Parquet.Net" Version="5.0.1" />
+    <PackageReference Include="ParquetSharp" Version="20.0.0" />
     <PackageReference Include="System.Data.SQLite" Version="1.0.118" />
     <PackageReference Include="MathNet.Numerics" Version="5.0.0" />
   </ItemGroup>

src/DataStreamSuite/DataStream/DataStreamExtensions.cs

@@ -2,31 +2,51 @@ namespace Bts.DataStream;
 
 using BCG;
 
+using bts;
 using bts.api.core;
+using bts.BlueOrb;
 
 using Bts.DataStreamCore;
+using Bts.PubSub;
+using Bts.SharedStreams.StreamFactories;
 
-using Microsoft.Extensions.Logging;
+using mbts.utils;
 
 using Spark.Core.Disposables;
+using Spark.Core.Threading;
 
 using System;
-using System.Collections;
-using System.Collections.Generic;
 using System.Linq;
 using System.Reactive.Linq;
+using System.Threading;
+using System.Threading.Tasks;
 
 public static class DataStreamExtensions {
-   public static ViewStream AsViewStream(this IObservable<DataPacket> source) =>
+   public static ViewStream AsViewStream(this IObservable<IDataPacket> source) =>
       source as ViewStream ?? new ObservableViewStream(source);
 
-   public static ViewStream AsViewStream<T>(this IObservable<IDataPacket<T>> source, string? name = null, ILogger? logger = null) where T : IBlueRow =>
-      source as ViewStream ?? new ObservableViewStream(source.Select(dp => new DataPacket(dp.newRows, dp.addedRows, dp.deletedRows)), name, logger);
+   public static ViewStream OrderBy(this ViewStream source, params string[] columns) {
+      if (source is bts.BlueOrb.SortStream sortStream) {
+         sortStream = sortStream.WithSortColumns(columns);
+      } else {
+         sortStream = new(source, columns);
+      }
+      return sortStream;
+   }
 
-   public static IObservable<DataPacket> AsObservable(this ViewStream source) {
-      return source as IObservable<DataPacket> ?? Observable.Create<DataPacket>(Subscribe);
+   public static IBlueRowCollection DropIndex(this IBlueRowCollection source) =>
+      IndexTable.Create(source);
 
-      IDisposable Subscribe(IObserver<DataPacket> observer) {
+   public static ViewStream DropIndex(this ViewStream source) =>
+      new IndexStream(source);
+
+   // public static ViewStream AsViewStream<T>(this IObservable<IDataPacket<T>> source, string? name = null, ILogger? logger = null) where T : IBlueRow =>
+   //    source as ViewStream ?? new ObservableViewStream(source, name, logger);
+
+   public static IObservable<IDataPacket> AsObservable(this ViewStream source) {
+      return source as IObservable<IDataPacket> ?? Observable.Create<IDataPacket>(Subscribe);
+
+      IDisposable Subscribe(IObserver<IDataPacket> observer) {
          source.onChange += OnChange;
          return Disposable.Create(() => source.onChange -= OnChange);
 
@@ -36,15 +56,41 @@ public static class DataStreamExtensions {
       }
    }
 
-   public static IObservable<IDataPacket<T>> SelectDataPacket<T>(this IObservable<IDataPacket> source, Func<IBlueRow, T> selector) where T : IBlueRow =>
-      source.Select(dp => {
-         dp.get_new_added_deleted(out var newRows, out var addedRows, out var deletedRows);
-         return new DataPacket<T>(CastRows(newRows), CastRows(addedRows), CastRows(deletedRows));
+   public static IObservable<IDataPacket> SelectColumn(this IObservable<IDataPacket> source, params (string ColumnName, Type OutputType, Func<object?, object?> GetValue)[] columnOverrides) {
+      return source.Scan(new ColumnSelectorStreamState(), Transform).Select(st => st.Data!);
 
-         IBlueRowCollection<T> CastRows(ICollection rows) =>
-            new BlueTableSelectorWrapper<T>(
-               rows as IBlueRowCollection ?? new BlueRowCollection(rows, dp.Schema), selector);
-      });
+      ColumnSelectorStreamState Transform(ColumnSelectorStreamState state, IDataPacket dp) {
+         var inputSchema = dp.Schema;
+         state.RowFactory ??= ColumnSelectorRowFactory.Create(
+            dp.Schema,
+            Array.ConvertAll(columnOverrides, c => (inputSchema.ResolveIndices(c.ColumnName), c.OutputType, c.GetValue)));
+         state.Data = new DataPacket(
+            new SelectorTable(dp.newRows, state.RowFactory.OutputSchema, state.RowFactory.Create),
+            new SelectorTable(dp.addedRows, state.RowFactory.OutputSchema, state.RowFactory.Create),
+            new SelectorTable(dp.deletedRows, state.RowFactory.OutputSchema, state.RowFactory.Create));
+         return state;
+      }
+   }
+
+   public static IObservable<IDataPacket> SelectRow(this IObservable<IDataPacket> source, IBlueTableSchema outputSchema, Func<IBlueRow, IBlueRow> rowSelector) {
+      return source.Scan((IDataPacket)null!, Transform);
+
+      IDataPacket Transform(IDataPacket lastOut, IDataPacket dp) =>
+         new DataPacket(
+            new SelectorTable(dp.newRows, outputSchema, rowSelector),
+            new SelectorTable(dp.addedRows, outputSchema, rowSelector),
+            new SelectorTable(dp.deletedRows, outputSchema, rowSelector));
+   }
+
+   // public static IObservable<IDataPacket<T>> SelectDataPacket<T>(this IObservable<IDataPacket> source, Func<IBlueRow, T> selector) where T : IBlueRow =>
+   //    source.Select(dp => {
+   //       dp.get_new_added_deleted(out var newRows, out var addedRows, out var deletedRows);
+   //       return new DataPacket<T>(CastRows(newRows), CastRows(addedRows), CastRows(deletedRows));
+
+   //       IBlueRowCollection<T> CastRows(ICollection rows) =>
+   //          new BlueTableSelectorWrapper<T>(
+   //             rows as IBlueRowCollection ?? new BlueRowCollection(rows, dp.Schema), selector);
+   //    });
 
    public static IObservable<TableUpdate<IBlueRow, int>> AsTables(this IObservable<IDataPacket> source) =>
       source.AsTables<IBlueRow, int>(r => r, r => r.Id());
@@ -72,9 +118,33 @@ public static class DataStreamExtensions {
       });
    }
 
-   public static IObservable<IDataPacket<T>> AsDataPackets<T>(this IObservable<TableUpdate<T, int>> source, IBlueTableSchema schema) where T : IBlueRow =>
+   public static IObservable<IDataPacket> AsDataPackets(this IObservable<TableUpdate<IBlueRow, int>> source) =>
       source.Scan(
-         (Table: new BlueTableImmutable<T>(schema), Dp: (IDataPacket<T>)DataPacket<T>.Empty(schema)),
+         (Table: (BlueTableImmutable?)null, Dp: (IDataPacket?)null),
+         (res, table) => {
+            var resTable = res.Table;
+            if (resTable is null) {
+               if (table.state.Count == 0) {
+                  return (null, null);
+               }
+               resTable = new(table.state.Values.First().schema);
+            }
+            var bd = resTable.ToDataPacketBuilder();
+            foreach (var r in table.removed) {
+               bd.Remove(r.Id());
+            }
+            foreach (var r in table.added) {
+               bd.AddOrUpdate(r);
+            }
+            var resDp = bd.ToDataPacket(out var newRows);
+            return (newRows, resDp);
+         })
+         .Where(t => t.Dp is not null && (t.Dp.addedRows.Any() || t.Dp.deletedRows.Any()))
+         .Select(t => t.Dp!);
+
+   public static IObservable<IDataPacket> AsDataPackets<T>(this IObservable<TableUpdate<T, int>> source, IBlueTableSchema schema) where T : IBlueRow =>
+      source.Scan(
+         (Table: new BlueTableImmutable(schema), Dp: (IDataPacket)new DataPacket(schema)),
          (res, table) => {
             var bd = res.Table.ToDataPacketBuilder();
             foreach (var r in table.removed) {
@@ -83,52 +153,290 @@ public static class DataStreamExtensions {
             foreach (var r in table.added) {
                bd.AddOrUpdate(r);
             }
-            var resDp = bd.ToDataPacketT(out var newRows, out var addedRows, out var deletedRows);
+            var resDp = bd.ToDataPacket(out var newRows);
             return (newRows, resDp);
          })
          .Where(t => t.Dp.addedRows.Any() || t.Dp.deletedRows.Any())
          .Select(t => t.Dp);
 
+   public static IObservable<IDataPacket> AsDataPackets<T, TKey>(this IObservable<TableUpdate<T, TKey>> source, BlueTableSchemaAdapter<T> schema) where TKey : IEquatable<TKey> {
+      return Observable.Create<IDataPacket>(Subscribe);
+
+      IDisposable Subscribe(IObserver<IDataPacket> o) {
+         var adapter = new BlueTableImmutableAdapter<T>(schema);
+         IDataPacket initDataPacket = new DataPacket(schema);
+         return source.Scan((Adapter: adapter, DataPacket: initDataPacket), OnUpdate).Select(res => res.DataPacket).Subscribe(o);
+
+         (BlueTableImmutableAdapter<T> Adapter, IDataPacket DataPacket) OnUpdate(
+            (BlueTableImmutableAdapter<T> Adapter, IDataPacket DataPacket) state, TableUpdate<T, TKey> tableUpdate) {
+            var bd = state.Adapter.ToDataPacketBuilder();
+            bd.Update(tableUpdate);
+            var dp = bd.ToDataPacket();
+            var tableAdapter = bd.ToImmutable();
+            return (bd.ToImmutable(), dp);
+         }
+      }
+   }
+
+   public static IObservable<IDataPacket> AsDataPackets(this IObservable<IBlueRowCollection> source) =>
+      Observable.Create<IDataPacket>(o => {
+         BlueTableImmutable? lastTable = null;
+         return source.Subscribe(OnNext);
+
+         void OnNext(IBlueRowCollection value) {
+            lastTable ??= new(value.schema);
+            var bd = lastTable.ToDataPacketBuilder();
+            bd.Clear();
+            foreach (var r in value) {
+               bd.AddOrUpdate(r, BlueRowEqualityComparer.Instance);
+            }
+            o.OnNext(bd.ToDataPacket(out lastTable));
+         }
+      });
+
    public static ViewStream AsViewStream<T>(this IObservable<TableUpdate<T, int>> source, IBlueTableSchema schema) where T : IBlueRow =>
       source.AsDataPackets(schema).AsViewStream();
 
-   public static int ResolveIndices(this IBlueTableSchema schema, string name) {
-      var indices = ResolveIndices(schema, new[] { name });
-      return indices[0];
+   public static ViewStream GroupByApply(this ViewStream viewStream, string[] columns, Func<IBlueRowCollection, IBlueRowCollection> apply) {
+      var groupByTables = GroupByApplyStream.CreateGroupByTable(viewStream, columns);
+      var applyTables = GroupByApplyStream.CreateApplyTable(groupByTables, FunctionEvaluator.Create(apply));
+      return applyTables.AsDataPackets().AsViewStream();
    }
 
-   public static (int, int) ResolveIndices(this IBlueTableSchema schema, string name1, string name2) {
-      var indices = ResolveIndices(schema, new[] { name1, name2 });
-      return (indices[0], indices[1]);
+   public static ViewStream GroupByApply(this ViewStream viewStream, string[] columns, Func<object?[], IBlueRowCollection, IBlueRowCollection?> apply) {
+      var groupByTables = GroupByApplyStream.CreateGroupByTable(viewStream, columns);
+      var applyTables = GroupByApplyStream.CreateApplyTable(groupByTables, new Func<EquatableKeyArrayWrapper, IBlueRowCollection, IBlueRowCollection?>(ApplyWrapper));
+      return applyTables.AsDataPackets().AsViewStream();
+
+      IBlueRowCollection? ApplyWrapper(EquatableKeyArrayWrapper groupKey, IBlueRowCollection groupRows) =>
+         apply(groupKey.Values, groupRows);
+   }
+   public static ViewStream Project(this ViewStream viewStream, params string[] columns) =>
+      new ColumnMaskStream(viewStream, string.Join(" ", columns));
+
+   public static ViewStream RenameColumns(this ViewStream viewStream, bool includeUnmapped, params (string From, string To)[] columnMap) =>
+      new ColumnMaskStream(viewStream, includeUnmapped, columnMap);
+
+   public static IBlueRowCollection AppendColumn(this IBlueRowCollection source, string name, Type type, IFunctionEvaluator eval) {
+      var newColumn = new InjectedColumn(name, source.schema.columnInfo.Length, type, GetValue);
+      return ColumnInjectionTable.Create(source, new IInjectedColumn[] { newColumn });
+
+      object? GetValue(IBlueRow row) =>
+         eval.Evaluate(string.Empty, new StreamArgument[] { (null, 0, row) });
    }
 
-   public static (int, int, int) ResolveIndices(this IBlueTableSchema schema, string name1, string name2, string name3) {
-      var indices = ResolveIndices(schema, new[] { name1, name2, name3 });
-      return (indices[0], indices[1], indices[2]);
-   }
+   public static ViewStream Filter(this ViewStream viewStream, string filterExpression) =>
+      new bts.BlueOrb.FilterStream(viewStream, filterExpression);
 
-   public static (int, int, int, int) ResolveIndices(this IBlueTableSchema schema, string name1, string name2, string name3, string name4) {
-      var indices = ResolveIndices(schema, new[] { name1, name2, name3, name4 });
-      return (indices[0], indices[1], indices[2], indices[3]);
-   }
+   public static ViewStream Filter(this ViewStream viewStream, string[] inputColumns, Func<object?[], bool> predicate) {
+      return ViewStreamFactory.Create(Subscribe);
 
-   public static int[] ResolveIndices(this IBlueTableSchema schema, string[] columnNames) {
-      var columns = schema.columnInfo;
-      var indices = new int[columnNames.Length];
-      int nResolved = 0;
-      for (int i = 0; i < columnNames.Length; i++) {
-         int origNResolved = nResolved;
-         for (int j = 0; j < columns.Length; j++) {
-            if (columns[j].colName == columnNames[i]) {
-               indices[i] = j;
-               nResolved++;
-               break;
+      IDisposable Subscribe(Action<IDataPacket> fire) {
+         BlueTableImmutable? lastOut = null;
+         int[]? inputColumnIndices = null;
+         object?[]? columnValues = new object?[inputColumns.Length];
+         return viewStream.Subscribe(OnData);
+
+         void OnData(IDataPacket dp) {
+            inputColumnIndices ??= dp.Schema.ResolveIndices(inputColumns);
+            lastOut ??= new(dp.Schema);
+            var bd = lastOut.ToDataPacketBuilder();
+            foreach (var r in dp.deletedRows) {
+               bd.Remove(r.Id());
+            }
+            foreach (var r in dp.addedRows) {
+               for (int i = 0; i < inputColumnIndices.Length; i++) {
+                  columnValues[i] = r[inputColumnIndices[i]];
+               }
+               if (predicate(columnValues)) {
+                  bd.AddOrUpdate(r);
+               }
+            }
+            var dpOut = bd.ToDataPacket(out lastOut);
+            if (dpOut.HasChanges()) {
+               fire(dpOut);
             }
          }
-         if (nResolved == origNResolved) {
-            throw new ArgumentException($"No column named '{columnNames[i]}'", nameof(schema));
-         }
       }
-      return indices;
+   }
+
+   public static IObservable<IDataPacket> TimeSeries(this IObservable<IDataPacket> source, int bufferSize, string timeColumn) =>
+      source.AsViewStream().TimeSeries(bufferSize, timeColumn).AsObservable();
+
+   public static ViewStream TimeSeries(this ViewStream source, int bufferSize, string timeColumn) =>
+      new TimeSeriesStream(source, bufferSize, timeColumn);
+
+   public static IDisposable Subscribe(this ViewStream viewStream, Action<IDataPacket> onChange) {
+      viewStream.onChange += OnChange;
+      return Disposable.Create(Dispose);
+
+      void OnChange(object? dpObj) =>
+         onChange(ExceptionAssert.Argument.Is<IDataPacket>(dpObj));
+
+      void Dispose() =>
+         viewStream.onChange -= OnChange;
+   }
+
+   public static IObservable<IDataPacket> NonIncrementalAsyncTransform(this IObservable<IDataPacket> source, Func<IBlueRowCollection, Task<IBlueRowCollection?>> transformAsync, IWorkScheduler workScheduler) {
+      const int STATE_IDLE = 0;
+      const int STATE_BUSY = 1;
+      const int STATE_DISPOSED = 2;
+      return Observable.Create<IDataPacket>(Subscribe);
+      IDisposable Subscribe(IObserver<IDataPacket> o) {
+         var syncRoot = new object();
+         IBlueRowCollection? lastTableIn = null;
+         BlueTableImmutable? lastTableOut = null;
+         int nConcurrentDoWorks = 0;
+         int state = STATE_IDLE;
+         IDisposable? dispose = null;
+         dispose = Disposable.Create(source.Subscribe(OnNext), workScheduler.Subscribe(DoWork));
+         return Disposable.Create(Dispose);
+
+         void Dispose() {
+            lock (syncRoot) {
+               switch (state) {
+                  case STATE_DISPOSED:
+                     return;
+                  case STATE_BUSY:
+                     state = STATE_DISPOSED;
+                     return;
+               }
+               state = STATE_DISPOSED;
+            }
+            Disposable.dispose(ref dispose);
+         }
+
+         void OnNext(IDataPacket value) =>
+            Volatile.Write(ref lastTableIn, value.newRows);
+
+         async void DoWork() {
+            lock (syncRoot) {
+               nConcurrentDoWorks++;
+               switch (state) {
+                  case STATE_DISPOSED:
+                  case STATE_BUSY:
+                     return;
+                  case STATE_IDLE:
+                     break;
+                  default:
+                     Log.assert(false, $"Unexpected state ({StateName(state)}) when DoWork called");
+                     return;
+               }
+               state = STATE_BUSY;
+            }
+            var dp = await PublishTable();
+            bool isDisposed = false;
+            lock (syncRoot) {
+               switch (state) {
+                  case STATE_DISPOSED:
+                     isDisposed = true;
+                     break;
+                  case STATE_BUSY:
+                     break;
+                  default:
+                     Log.assert(false, $"Unexpected state ({StateName(state)}) after building output table (async)");
+                     return;
+               }
+            }
+            if (isDisposed) {
+               Disposable.dispose(ref dispose);
+               return;
+            }
+            if (dp is not null) {
+               o.OnNext(dp);
+            }
+            lock (syncRoot) {
+               nConcurrentDoWorks = 0;
+               switch (state) {
+                  case STATE_DISPOSED:
+                     isDisposed = true;
+                     break;
+                  case STATE_BUSY:
+                     state = STATE_IDLE;
+                     break;
+                  default:
+                     Log.assert(false, $"Unexpected state ({StateName(state)}) after firing output table");
+                     return;
+               }
+            }
+            if (isDisposed) {
+               Disposable.dispose(ref dispose);
+            }
+
+            async Task<DataPacket?> PublishTable() {
+               var tableIn = Interlocked.Exchange(ref lastTableIn, null);
+               if (tableIn is null) {
+                  return null;
+               }
+               var tableOut = await transformAsync(tableIn);
+               if (tableOut is null) {
+                  return null;
+               }
+               lastTableOut ??= new(tableOut.schema);
+               var bd = lastTableOut.ToDataPacketBuilder();
+               bd.Clear();
+               foreach (var r in tableOut) {
+                  bd.AddOrUpdate(r, BlueRowEqualityComparer.Instance);
+               }
+               return bd.ToDataPacket(out lastTableOut);
+            }
+         }
+
+         static string StateName(int state) =>
+            state switch {
+               STATE_IDLE => "IDLE",
+               STATE_BUSY => "BUSY",
+               STATE_DISPOSED => "DISPOSED",
+               _ => $"STATE_{state}",
+            };
+      }
+   }
+
+   public static ViewStream Diff(this ViewStream source, int period = 1) =>
+      new bts.BlueOrb.DiffStream(source, period);
+
+   public static IObservable<IDataPacket> OnWorkScheduler(this IObservable<IDataPacket> source, IWorkScheduler scheduler) =>
+      Observable.Create<IDataPacket>(o => {
+         var inputGate = new AtomicGate();
+         var outputGate = new AtomicGate();
+         BlueTableImmutable? lastTable = null;
+         BlueTableImmutable.DataPacketBuilder? bd = null;
+         IDataPacket? pendingDpOut = null;
+         return Disposable.Create(source.Subscribe(OnNext), scheduler.Subscribe(DoWork));
+
+         void OnNext(IDataPacket value) {
+            using (inputGate.BeginTransaction()) {
+               lastTable ??= new(value.newRows.schema);
+               bd ??= lastTable.ToDataPacketBuilder();
+               if (pendingDpOut is null) {
+                  // The last outgoing datapacket was just fired. Don't merge this update with the
+                  // previous one (otherwise its adds/removes would be fired twice).
+                  bd.Reset();
+               }
+               foreach (var r in value.deletedRows) {
+                  bd.Remove(r.Id());
+               }
+               foreach (var r in value.addedRows) {
+                  bd.AddOrUpdate(r);
+               }
+               pendingDpOut = bd.ToDataPacket(out lastTable);
+            }
+         }
+
+         void DoWork() {
+            IDataPacket? dpOut = null;
+            if (outputGate.TryEnter()) {
+               using var exit = Disposable.Create(outputGate.Exit);
+               (dpOut, pendingDpOut) = (pendingDpOut, dpOut);
+               if (dpOut is not null) {
+                  o.OnNext(dpOut);
+               }
+            }
+         }
+      });
+
+   struct ColumnSelectorStreamState {
+      public IDataPacket? Data;
+      public ColumnSelectorRowFactory? RowFactory;
    }
 }

src/DataStreamSuite/DataStream/Misc/DSLStream.cs

@@ -1,27 +1,29 @@
-namespace bts.SilverPine {
+namespace bts.BlueOrb {
    using System;
    using BCG;
    using System.Collections.Generic;
    using Bts.DataStreamExpressions;
+   using Bts.DataStreamCore;
 
    // All this does is take in a dsl expression, parse it and fire the data that come out of running it.
    // So for example, if an expresison is passed 
    [DataStreamUserRespresentation("dsl")]
    public class DSLStream : ViewStream {
       private readonly string _dsl;
-      public DSLStream(string dsl) {
+      public DSLStream(string dsl, IDataStreamFactory streamFactory) {
          Log.Gen.info($"DSLStream.cons dsl: {dsl}");
          _dsl = dsl;
-         ViewStream rs = build_stream(dsl) ?? new LiteralStream("error", $"msg:invalid dsl");
+         if (dsl.StartsWith("```dsl\n")) {
+            dsl = string.Concat("```", dsl.AsSpan(7));
+         }
+         var rs = build_stream(streamFactory, dsl) ?? new LiteralStream("error", $"msg:invalid dsl");
          mySetRealSource(rs);
       }
 
-      static ViewStream? build_stream(string stream_expression) {
+      static ViewStream? build_stream(IDataStreamFactory streamFactory, string stream_expression) {
          try {
             var exp = DataStreamExpressionsInterpreter.Evaluate(stream_expression);
-            var rv = exp.getView(new Dictionary<string, StatementExpression>(0));
-            return rv;
-
+            return exp.Evaluate(streamFactory) as ViewStream;
          } catch (Exception ex) {
             Log.Gen.error($"Could not build stream: {stream_expression} ex: {ex}");
             return null;

src/DataStreamSuite/DataStream/Misc/PandasStream.cs

@@ -114,14 +114,13 @@ import sys
       executePythonCode(pythonCode.Value, pythonExe);
       var rows = BlueDataUtility.createRowsFromCSVFileWithoutTypes(outputFile, out var schema);
       var rowMap = rows as IDictionary<int, IBlueRow> ?? rows.Cast<IBlueRow>().ToDictionary(r => r.Id());
-      var comp = BlueRowEqualityComparer.Instance;
       var deletedRows = new BlueTable(schema);
       var addedRows = new BlueTable(schema);
       if (lastRowMap is not null) {
          deletedRows.SetItems(lastRowMap.Where(oldRow =>
-            !(rowMap.TryGetValue(oldRow.Key, out var newRow) && comp.Equals(newRow, oldRow.Value))));
+            !(rowMap.TryGetValue(oldRow.Key, out var newRow) && BlueRowEqualityComparer.Equals(newRow, oldRow.Value))));
          addedRows.SetItems(rowMap.Where(newRow =>
-            !(lastRowMap.TryGetValue(newRow.Key, out var oldRow) && comp.Equals(oldRow, newRow.Value))));
+            !(lastRowMap.TryGetValue(newRow.Key, out var oldRow) && BlueRowEqualityComparer.Equals(oldRow, newRow.Value))));
       }
       lastRowMap = rowMap;
       lastFireTime = DateTime.Now;

src/DataStreamSuite/DataStream/ObservableViewStream.cs

@@ -8,13 +8,13 @@ using Microsoft.Extensions.Logging;
 
 using System;
 
-public class ObservableViewStream : ViewStream, IObservable<DataPacket>, IDisposable {
-   readonly IObservable<DataPacket> source;
+public class ObservableViewStream : ViewStream, IObservable<IDataPacket>, IDisposable {
+   readonly IObservable<IDataPacket> source;
    readonly string? name;
    readonly ILogger? logger;
    IDisposable? subscription;
 
-   public ObservableViewStream(IObservable<DataPacket> source, string? name = null, ILogger? logger = null) {
+   public ObservableViewStream(IObservable<IDataPacket> source, string? name = null, ILogger? logger = null) {
       this.source = source;
       this.name = name;
       this.logger = logger;
@@ -45,6 +45,6 @@ public class ObservableViewStream : ViewStream, IObservable<DataPacket>, IDispos
       subscription = null;
    }
 
-   IDisposable IObservable<DataPacket>.Subscribe(IObserver<DataPacket> observer) =>
+   IDisposable IObservable<IDataPacket>.Subscribe(IObserver<IDataPacket> observer) =>
       source.Subscribe(observer);
 }

src/DataStreamSuite/DataStream/Offloading/RemoteQueryManager.cs

@@ -1,16 +1,18 @@
+using bts;
+
 using Bts.DataStream;
 using Bts.DataStreamCore;
 using Bts.DataStreamExpressions;
 using Bts.PubSub;
 
-using bts;
-
 using Microsoft.Extensions.Hosting;
 
 using Spark.Core.Disposables;
 
 using System;
 using System.Collections.Generic;
+using System.IO;
+using System.Linq;
 using System.Reactive.Linq;
 using System.Threading;
 using System.Threading.Tasks;
@@ -34,14 +36,12 @@ namespace BCG {
             view_publisher = view_publisher_;
          }
       }
-      static ViewStream? build_stream(string stream_expression)
+      static ViewStream? build_stream(IDataStreamFactory streamFactory, string stream_expression)
       {
          try
          {
             var exp = DataStreamExpressionsInterpreter.Evaluate(stream_expression);
-            var rv = exp.getView(new Dictionary<string, StatementExpression>(0));
-            return rv;
-
+            return exp.Evaluate(streamFactory) as ViewStream;
          } catch (Exception ex) {
             bts.Log.assert(false, $"Could not build stream: {stream_expression} ex: {ex}");
             return null;
@@ -51,15 +51,17 @@ namespace BCG {
          _stream_pub_map = new Dictionary<string, StreamPubUnit>();
       readonly IPubTableBroker _broker;
       readonly IManagedSubscriber _sub;
+      readonly IDataStreamFactory streamFactory;
       readonly object syncRoot = new();
       IDisposable? cloudStreamsSub;
       IDisposable? remoteRequestsSub;
       bool isDisposed;
 
 
-      public RemoteQueryManager(IManagedSubscriber managedSubscriber, IPubTableBroker pubTableBroker) {
+      public RemoteQueryManager(IManagedSubscriber managedSubscriber, IPubTableBroker pubTableBroker, IEnumerable<IDataStreamFactory> streamFactories) {
          _sub = managedSubscriber;
          _broker = pubTableBroker;
+         streamFactory = DataStreamFactory.Create(streamFactories);
       }
 
       public Task StartAsync(CancellationToken cancellationToken) {
@@ -67,7 +69,9 @@ namespace BCG {
             Log.assert(!isDisposed);
             Log.isnull(cloudStreamsSub);
             Log.isnull(remoteRequestsSub);
-            cloudStreamsSub = new TableStream("cloud_streams").AsObservable().Subscribe(_on_str);
+            var cloudStream = streamFactory.Create("table", new (string?, int?, object?)[] { ("name", 0, "cloud_streams") })
+               ?? throw new Exception("Failed to create TableStream 'cloud_stream'");
+            cloudStreamsSub = cloudStream.AsObservable().Subscribe(_on_str);
             remoteRequestsSub = _sub.Observe("RemoteRequest").Subscribe(data => {
                RemoteRequest req = RemoteRequest.construct_from_bytes(data);
                _spawn_remote_stream(req.published_table_name, req.stream_expression, false);
@@ -87,8 +91,7 @@ namespace BCG {
          return Task.CompletedTask;
       }
 
-      private void _on_str(DataPacket dp)
-      {
+      private void _on_str(IDataPacket dp) {
          foreach (IBlueRow r in dp.newRows)
          {
             string name = r.get<string>("name");
@@ -102,10 +105,17 @@ namespace BCG {
 
       private void _spawn_remote_stream(string name, string stream_expression, bool persist) {
          bts.Log.Gen.info($"CloudStreamManager.assign name: {name} stream_expression: {stream_expression}");
-         ViewStream stream = build_stream(stream_expression);
+         var stream = build_stream(streamFactory, stream_expression);
+         if (stream is null) {
+            Log.DataFlow.error($"Failed to spawn remote stream. name: {name}; expression: {stream_expression}");
+            return;
+         }
          _expose_to_ws(name, stream);
          if (persist) {
-            stream = new PersistorStream(stream, name, "C:\\Temp\\", 1000000);   // 10MBs
+            string tempDir = Path.GetTempPath();
+        Directory.CreateDirectory(tempDir); // make sure it exists
+
+        stream = new PersistorStream(stream, name, tempDir, 10_000_000); // 10 MB
          }
          ViewPublisher view_pub = new ViewPublisher(name, _broker, stream);
          StreamPubUnit u = new StreamPubUnit(name, stream_expression, stream, persist, view_pub);