WellKnownColumns.cs

Also change snake_case fn names to PascalCase

.config/dotnet-tools.json

@@ -9,7 +9,7 @@
       ]
     },
     "fable": {
-      "version": "4.0.6",
+      "version": "4.24.0",
       "commands": [
         "fable"
       ]

.devcontainer/Dockerfile

@@ -65,7 +65,7 @@ RUN \
    && apt-get install -y cmake
 
 # Dotnet
-RUN apt-get update && apt-get install -y dotnet7
+RUN apt-get update && apt-get install -y dotnet8
 
 # Node
 ARG node_major=18
@@ -129,5 +129,3 @@ RUN \
    -G sudo \
    --password "$(openssl passwd -1 "$docker_pass")" \
    "$docker_user"
-
-RUN usermod -aG docker "$docker_user"

.devcontainer/devcontainer.json

@@ -1,15 +1,10 @@
 {
   "dockerFile": "Dockerfile",
-  // Creates local dir to mount to HOME in docker 
-  "initializeCommand": "mkdir -p ${localEnv:HOME}/.bts/devcontainer-mounts/${devcontainerId}",
   "hostRequirements": {
-    "memory": "4gb",
-    "storage": "6gb"
+    "memory": "8gb",
+    "storage": "10gb"
   },
-  "postCreateCommand": "/home/bts/spark/docker/sparkd/dev/devcontainer_post_create.sh",
   "containerUser": "bts",
-  "workspaceMount": "source=${localWorkspaceFolder},target=/home/bts/spark,type=bind",
-  "workspaceFolder": "/home/bts/spark",
   "updateRemoteUserUID": true,
   "shutdownAction": "none",
   // Note: If running multiple containers on the same host, each must use a
@@ -17,26 +12,20 @@
   // example, if 8000-8003 is taken, go with 9000-9003, then 10000-10003, etc.
   "runArgs": [
     "--shm-size",
-    "2gb",
-    "--network",
-    "host"
+    "8gb"
   ],
   "mounts": [
-    "source=${localEnv:HOME}/.bts/devcontainer-mounts/${devcontainerId},target=/home/bts,type=bind",
     "source=/var/run/docker.sock,target=/var/run/docker.sock,type=bind"
   ],
   "customizations": {
     "vscode": {
-      "settings": {
-        "clangd.arguments": [
-          "--compile-commands-dir=${workspaceFolder}/build/clang/debug"
-        ]
-      },
       "extensions": [
         "eamodio.gitlens",
         "llvm-vs-code-extensions.vscode-clangd",
+        "ms-python.python",
         "ms-vscode.cmake-tools",
-        "ms-vscode.cpptools"
+        "ms-vscode.cpptools",
+        "xaver.clang-format"
       ]
     }
   }

.env

@@ -7,10 +7,10 @@
 #   We use Docker Compose to build images and launch containers for various services
 #   that make up our software suite:
 #     - sparkd
-#     - pubsub
-#     - md-publisher
-#     - webapp (DSQA on the web)
-#     - api-server (websocket based)
+#     - TODO: pubsub
+#     - TODO: md-publisher
+#     - TODO: webapp (DSQA on the web)
+#     - TODO: api-server (websocket based)
 #   From inside a devcontainer, you can launch all services in separate containers
 #   on your machine by running:
 #
@@ -31,51 +31,19 @@
 
 ### Sparkd ###
 # The internal image for sparkd to use. For more info, view:
-#   src/app_sparkd/Dockerfile
+#   src/server/sparkd/Dockerfile
 SPARKD__IMAGE=sparkd
-# The 'spark.conf' for this service to use
-SPARKD__CONF_FILE=src/app_sparkd/spark.conf
-# To run replay md, put recorded md files in SPARKD__MD_RECORDING_DIR, and use:
-#   SPARKD__IMAGE=sparkd_md_replay
-#   SPARKD__CONF_FILE=src/app_sparkd/spark.local.conf
-# (configure which md to replay in the new SPARKD__CONF_FILE)
-SPARKD__MD_RECORDING_DIR=.md_recording
+# The config dir for this service to use
+SPARKD__CONFIG_DIR=src/app_sparkd/configurations/dev
+# Mount a specific volume or dir to "~/sparkdata/data" inside the docker.
+# Defaults to a volume named "sparkd_<config_port>" (e.g., sparkd_8000).
+# SPARKD__DATA_DIR=/home/bts/sparkdata/data
 SPARKD__SOURCE_DIR=install/sparkd-release-gcc
+SPARKD__RUNTIME_CONF_DIR=/home/ubuntu/8000/sparkdata/conf/
 SPARKD__CONFIG_PORT=8000
 SPARKD__QUICKBUS_PORT=8001
 SPARKD__SIM_FIX_PORT=8002
+SPARKD__SHM_SIZE=4gb
+SPARKD__PYTHON_REQUIREMENTS=requirements.txt
 
-### Pubsub ###
-# The 'spark.conf' for this service to use
-PUBSUB__CONF_FILE=src/DataStreamSuite/PubSubServer/spark.conf
-PUBSUB__CS_SOURCE_DIR=build/Release/bin/net7.0/linux-x64/publish
-PUBSUB__CPP_SOURCE_DIR=install/data-stream-suite-linux
-# The appsettings to load at runtime. Expects a file named:
-#   src/DataStreamSuite/PubSub/appsettings.${PUBSUB__DOTNET_ENVIRONMENT}.json
-PUBSUB__DOTNET_ENVIRONMENT=Staging
-
-### Md Publisher ###
-# The 'spark.conf' for this service to use
-MD_PUBLISHER__CONF_FILE=src/DataStreamSuite/DataStreamPublisher/spark.conf
-MD_PUBLISHER__SOURCE_DIR=build/Release/bin/net7.0/linux-x64/publish
-# Determines which appsettings to load at runtime. Expects a file named:
-#   src/DataStreamSuite/DataStreamPublisher/appsettings.${MD_PUBLISHER__DOTNET_ENVIRONMENT}.json
-MD_PUBLISHER__DOTNET_ENVIRONMENT=MarketData
-
-### Webapp (DSQA on the web) ###
-# The 'spark.conf' for this service to use
-WEBAPP__CONF_FILE=src/client/spark_api_server/spark.conf
-WEBAPP__SOURCE_DIR=build/Release/bin/net7.0/linux-x64/publish
-WEBAPP__WEB_ROOT=build/Bts.DataStreamWebApp.Client/production/dist
-# Determines which appsettings to load at runtime. Expects a file named:
-#   src/DataStreamSuite/Bts.DataStreamWebApp.Server/appsettings.${WEBAPP__ASPNETCORE_ENVIRONMENT}.json
-WEBAPP__ASPNETCORE_ENVIRONMENT=Staging
-
-### Api Server ###
-# The 'spark.conf' for this service to use
-API_SERVER__CONF_FILE=src/client/spark_api_server/spark.conf
-API_SERVER__CS_SOURCE_DIR=build/Release/bin/net7.0/linux-x64/publish
-API_SERVER__CPP_SOURCE_DIR=install/spark-api-server-linux
-# Determines which appsettings to load at runtime. Expects a file named:
-#   src/client/spark_api_server/appsettings.${API_SERVER__ASPNETCORE_ENVIRONMENT}.json
-API_SERVER__ASPNETCORE_ENVIRONMENT=Staging
+# TODO: add config for other services (pubsub, md-publisher, webapp, api-server, etc.)

.gitattributes

@@ -1,4 +1,3 @@
 *.cmd text=auto eol=crlf
 *.cs text=auto eol=crlf
 *.sh text=auto eol=lf
-*.py text=auto eol=lf

.gitignore

@@ -11,6 +11,7 @@ cmake-build-debug/
 **.vscode/*
 !.vscode/launch.json
 !.vscode/tasks.json
+!.vscode/*.env
 .vs
 obj/
 *.csproj.user
@@ -54,11 +55,12 @@ trace_data/
 .cache/
 /src/Spark.Admin.WebApp/tsconfig.*.g.json
 /src/Spark.Admin.WebApp/src/bin/
+/src/client/**/*appsettings.Local.json
+/src/DataStreamSuite/**/*appsettings.Local.json
 **/CMakeDeps.cmake
 **/CMakeDeps_win.cmake
 .devcontainer/
 *.Generated.cs
 *.lst
-src/app_sparkd/spark.local.conf
 .env.local
-.md_recording
+.secrets

.vscode/launch.json

@@ -1,6 +1,14 @@
 {
   "version": "0.2.0",
   "configurations": [
+    {
+      "name": "Python Debugger: Current File with Arguments",
+      "type": "debugpy",
+      "request": "launch",
+      "program": "${file}",
+      "console": "integratedTerminal",
+      "args": "${command:pickArgs}"
+    },
     {
       "name": ".NET Core Attach",
       "type": "coreclr",
@@ -16,19 +24,10 @@
         "program": "spark_api_server.exe",
         "args": [
           "run",
+          "--sparkd",
+          "http://127.0.0.1:8000/config"
         ],
-        "cwd": "build/Debug/bin/net7.0",
-      },
-      "linux": {
-        "preLaunchTask": "Build Api Server (Debug)",
-        "program": "spark_api_server",
-        "args": [
-          "run",
-        ],
-        "cwd": "build/Debug/bin/net7.0",
-      },
-      "env": {
-        "ASPNETCORE_ENVIRONMENT": "Development",
+        "cwd": "build/Debug/bin/net8.0",
       },
       "stopAtEntry": false,
       "console": "integratedTerminal"
@@ -42,18 +41,10 @@
         "program": "spark_api_server.exe",
         "args": [
           "run",
+          "--sparkd",
+          "http://127.0.0.1:8000/config"
         ],
-        "cwd": "build/Debug/bin/net7.0",
-      },
-      "linux": {
-        "program": "spark_api_server",
-        "args": [
-          "run",
-        ],
-        "cwd": "build/Debug/bin/net7.0",
-      },
-      "env": {
-        "ASPNETCORE_ENVIRONMENT": "Development",
+        "cwd": "build/Debug/bin/net8.0",
       },
       "stopAtEntry": false,
       "console": "integratedTerminal"
@@ -62,9 +53,9 @@
       "name": "Bts.DataStreamWebApp.Server",
       "type": "coreclr",
       "request": "launch",
-      "preLaunchTask": "Build DataStream (Debug)",
-      "program": "${workspaceFolder}/build/Debug/bin/net7.0/Bts.DataStreamWebApp.Server.dll",
-      "cwd": "${workspaceFolder}/build/Debug/bin/net7.0",
+      "preLaunchTask": "Build DataStream WebApp (Debug)",
+      "program": "${workspaceFolder}/build/Debug/bin/net8.0/Bts.DataStreamWebApp.Server.dll",
+      "cwd": "${workspaceFolder}/build/Debug/bin/net8.0",
       "env": {
         "ASPNETCORE_HTTP_PORT": "8080",
         "ASPNETCORE_HTTPS_PORT": "8443",
@@ -76,8 +67,8 @@
       "name": "Bts.DataStreamWebApp.Server (Skip Build)",
       "type": "coreclr",
       "request": "launch",
-      "program": "${workspaceFolder}/build/Debug/bin/net7.0/Bts.DataStreamWebApp.Server.dll",
-      "cwd": "${workspaceFolder}/build/Debug/bin/net7.0",
+      "program": "${workspaceFolder}/build/Debug/bin/net8.0/Bts.DataStreamWebApp.Server.dll",
+      "cwd": "${workspaceFolder}/build/Debug/bin/net8.0",
       "env": {
         "ASPNETCORE_HTTP_PORT": "8080",
         "ASPNETCORE_HTTPS_PORT": "8443",
@@ -97,10 +88,10 @@
       "windows": {
         "program": "DataStreamPublisher.exe",
       },
+      "cwd": "build/Debug/bin/net8.0",
       "env": {
         "DOTNET_ENVIRONMENT": "${input:dataStreamPublisherEnvironment}",
       },
-      "cwd": "${workspaceFolder}/build/Debug/bin/net7.0",
       "stopAtEntry": false,
       "console": "integratedTerminal"
     },
@@ -115,10 +106,10 @@
       "windows": {
         "program": "DataStreamPublisher.exe",
       },
+      "cwd": "${workspaceFolder}/build/Debug/bin/net8.0",
       "env": {
         "DOTNET_ENVIRONMENT": "${input:dataStreamPublisherEnvironment}",
       },
-      "cwd": "${workspaceFolder}/build/Debug/bin/net7.0",
       "stopAtEntry": false,
       "console": "integratedTerminal"
     },
@@ -127,7 +118,39 @@
       "type": "cppdbg",
       "request": "launch",
       "program": "/home/bts/spark/build/sparkd-debug-clang/server/sparkd",
-      "args": ["run"],
+      "args": [
+        "run"
+      ],
+      "stopAtEntry": false,
+      "cwd": "/home/bts/spark/build/sparkd-debug-clang/server",
+      "environment": [],
+      "externalConsole": false,
+      "MIMode": "gdb",
+    },
+    {
+      "name": "spreader tests2",
+      "type": "cppdbg",
+      "request": "launch",
+      "program": "/home/bts/spark/build/sparkd-debug-clang/server/sparkd",
+      "args": [
+        "test",
+        "[spreader2]"
+      ],
+      "stopAtEntry": false,
+      "cwd": "/home/bts/spark/build/sparkd-debug-clang/server",
+      "environment": [],
+      "externalConsole": false,
+      "MIMode": "gdb",
+    },
+    {
+      "name": "spreader tests",
+      "type": "cppdbg",
+      "request": "launch",
+      "program": "/home/bts/spark/build/sparkd-debug-clang/server/sparkd",
+      "args": [
+        "test",
+        "[spreader]"
+      ],
       "stopAtEntry": false,
       "cwd": "/home/bts/spark/build/sparkd-debug-clang/server",
       "environment": [],
@@ -142,12 +165,25 @@
       "program": "",
       "windows": {
         "program": "DataStreamQueryAnalyzer.exe",
-        "args": [
-          "--config",
-          "${workspaceFolder}/src/DataStreamSuite/DataStreamQueryAnalyzer/Configurations/DataStreamQueryAnalyzer.json"
-        ],
-        "cwd": "build/Debug/bin/net7.0-windows",
+        "cwd": "build/Debug/bin/net8.0-windows",
         "env": {
+          "DOTNET_ENVIRONMENT": "${input:dataStreamQueryAnalyzerEnvironment}",
+          "PYTHON_EXE": "${workspaceFolder}/.venv/Scripts/python.exe"
+        },
+      },
+      "stopAtEntry": false,
+      "console": "integratedTerminal",
+    },
+    {
+      "name": "DataStreamQueryAnalyzer (Skip Build)",
+      "type": "coreclr",
+      "request": "launch",
+      "program": "",
+      "windows": {
+        "program": "DataStreamQueryAnalyzer.exe",
+        "cwd": "build/Debug/bin/net8.0-windows",
+        "env": {
+          "DOTNET_ENVIRONMENT": "${input:dataStreamQueryAnalyzerEnvironment}",
           "PYTHON_EXE": "${workspaceFolder}/.venv/Scripts/python.exe"
         },
       },
@@ -166,7 +202,7 @@
       "windows": {
         "program": "PubSubServer.exe",
       },
-      "cwd": "build/Debug/bin/net7.0",
+      "cwd": "build/Debug/bin/net8.0",
       "env": {
         "DOTNET_ENVIRONMENT": "${input:pubSubServerEnvironment}",
       },
@@ -184,7 +220,7 @@
       "windows": {
         "program": "PubSubServer.exe",
       },
-      "cwd": "build/Debug/bin/net7.0",
+      "cwd": "build/Debug/bin/net8.0",
       "env": {
         "DOTNET_ENVIRONMENT": "${input:pubSubServerEnvironment}",
       },
@@ -199,11 +235,25 @@
       "windows": {
         "preLaunchTask": "Build Spark (Debug)",
         "program": "Spark.exe",
-        "args": [
-          "--sparkd",
-          "http://127.0.0.1:8000/config"
-        ],
-        "cwd": "build/Debug/bin/net7.0-windows",
+        "cwd": "build/Debug/bin/net8.0-windows",
+        "env": {
+          "DOTNET_ENVIRONMENT": "${input:sparkEnvironment}",
+        }
+      },
+      "stopAtEntry": false,
+      "console": "integratedTerminal"
+    },
+    {
+      "name": "Spark (Skip Build)",
+      "type": "coreclr",
+      "request": "launch",
+      "program": "",
+      "windows": {
+        "program": "Spark.exe",
+        "cwd": "build/Debug/bin/net8.0-windows",
+        "env": {
+          "DOTNET_ENVIRONMENT": "${input:sparkEnvironment}",
+        }
       },
       "stopAtEntry": false,
       "console": "integratedTerminal"
@@ -220,7 +270,40 @@
           "--sparkd",
           "${input:sparkConfigServer}"
         ],
-        "cwd": "build/Debug/bin/net7.0-windows",
+        "cwd": "build/Debug/bin/net8.0-windows",
+      },
+      "stopAtEntry": false,
+      "console": "integratedTerminal"
+    },
+    {
+      "name": "Spark Test",
+      "type": "coreclr",
+      "request": "launch",
+      "program": "",
+      "windows": {
+        "preLaunchTask": "Build Spark (Debug)",
+        "program": "Spark.Console.exe",
+        "cwd": "build/Debug/bin/net8.0",
+        "args": [
+          "test",
+          "${input:sparkTest}"
+        ]
+      },
+      "stopAtEntry": false,
+      "console": "integratedTerminal"
+    },
+    {
+      "name": "Spark Test (Skip Build)",
+      "type": "coreclr",
+      "request": "launch",
+      "program": "",
+      "windows": {
+        "program": "Spark.Console.exe",
+        "cwd": "build/Debug/bin/net8.0",
+        "args": [
+          "test",
+          "${input:sparkTest}"
+        ]
       },
       "stopAtEntry": false,
       "console": "integratedTerminal"
@@ -229,7 +312,6 @@
       "name": "Sparkd",
       "type": "cppdbg",
       "request": "launch",
-      "preLaunchTask": "Build Sparkd (Debug | Clang)",
       "program": "",
       "linux": {
         "program": "${workspaceFolder}/build/sparkd-debug-clang/src/app_sparkd/sparkd",
@@ -282,34 +364,6 @@
         }
       ]
     },
-    {
-      "name": "Sparkd Web (Skip Build)",
-      "type": "cppdbg",
-      "request": "launch",
-      "program": "",
-      "linux": {
-        "program": "${workspaceFolder}/build/sparkd-debug-clang/src/app_sparkd/sparkd",
-      },
-      "args": [
-        "web"
-      ],
-      "cwd": "${workspaceFolder}/build/sparkd-debug-clang/src/app_sparkd",
-      "stopAtEntry": false,
-      "externalConsole": false,
-      "MIMode": "gdb",
-      "setupCommands": [
-        {
-          "description": "Enable pretty-printing for gdb",
-          "text": "-enable-pretty-printing",
-          "ignoreFailures": true
-        },
-        {
-          "description": "Set Disassembly Flavor to Intel",
-          "text": "-gdb-set disassembly-flavor intel",
-          "ignoreFailures": true
-        }
-      ]
-    },
   ],
   "inputs": [
     {
@@ -349,12 +403,42 @@
         "Staging",
       ]
     },
+    {
+      "id": "dataStreamQueryAnalyzerEnvironment",
+      "type": "pickString",
+      "description": "DataStreamQueryAnalyzer Configuration",
+      "options": [
+        "Development",
+        "Local",
+      ],
+      "default": "Development",
+    },
+    {
+      "id": "sparkEnvironment",
+      "type": "pickString",
+      "description": "Spark Configuration",
+      "options": [
+        "Development",
+        "Local",
+      ],
+      "default": "Development",
+    },
     {
       "id": "sparkConfigServer",
       "type": "promptString",
       "description": "Config Server URL",
       "default": "http://localhost:8000/config",
     },
+    {
+      "id": "sparkTest",
+      "type": "promptString",
+      "description": "Test to run",
+    },
+    {
+      "id": "sparkdTest",
+      "type": "promptString",
+      "description": "Test to run",
+    },
     {
       "id": "isSparkApiEnabled",
       "type": "pickString",

.vscode/python.env

@@ -0,0 +1 @@
+PYTHONPATH=/workspaces/spark/build/sparkd-debug-clang/src/lib_python_interop;/workspaces/spark/src/lib_python_interop/python_interop_plugins

.vscode/tasks.json

@@ -485,12 +485,12 @@
       "command": "",
       "windows": {
         "command": "DataStreamQueryAnalyzer.exe",
-        "args": [
-          "--config",
-          "${workspaceFolder}/src/DataStreamSuite/DataStreamQueryAnalyzer/Configurations/DataStreamQueryAnalyzer.json",
-        ],
         "options": {
-          "cwd": "${workspaceFolder}/build/${input:buildConfiguration}/bin/net7.0-windows"
+          "cwd": "${workspaceFolder}/build/${input:buildConfiguration}/bin/net8.0-windows",
+          "env": {
+            "DOTNET_ENVIRONMENT": "${input:dataStreamQueryAnalyzerEnvironment}",
+            "PYTHON_EXE": "${workspaceFolder}/.venv/Scripts/python.exe"
+          },
         },
       },
       "icon": {
@@ -510,7 +510,7 @@
           "${workspaceFolder}/src/DataStreamSuite/DataStreamQueryAnalyzer/Configurations/DataStreamQueryAnalyzer.json",
         ],
         "options": {
-          "cwd": "${workspaceFolder}/build/Debug/bin/net7.0-windows"
+          "cwd": "${workspaceFolder}/build/Debug/bin/net8.0-windows"
         },
       },
       "icon": {
@@ -530,7 +530,7 @@
           "${workspaceFolder}/src/DataStreamSuite/DataStreamQueryAnalyzer/Configurations/DataStreamQueryAnalyzer.json",
         ],
         "options": {
-          "cwd": "${workspaceFolder}/build/Release/bin/net7.0-windows"
+          "cwd": "${workspaceFolder}/build/Release/bin/net8.0-windows"
         },
       },
       "icon": {
@@ -549,12 +549,11 @@
       "windows": {
         "command": "DataStreamPublisher.exe",
       },
-      "args": [
-        "--config",
-        "${workspaceFolder}/src/DataStreamSuite/DataStreamPublisher/Configurations/${input:dataStreamPublisherConfig}.json",
-      ],
       "options": {
-        "cwd": "${workspaceFolder}/build/${input:buildConfiguration}/bin/net7.0"
+        "cwd": "${workspaceFolder}/build/${input:buildConfiguration}/bin/net8.0",
+        "env": {
+          "DOTNET_ENVIRONMENT": "${input:dataStreamPublisherEnvironment}",
+        },
       },
       "icon": {
         "id": "run",
@@ -572,12 +571,11 @@
       "windows": {
         "command": "DataStreamPublisher.exe",
       },
-      "args": [
-        "--config",
-        "${workspaceFolder}/src/DataStreamSuite/DataStreamPublisher/Configurations/${input:dataStreamPublisherConfig}.json",
-      ],
       "options": {
-        "cwd": "${workspaceFolder}/build/Debug/bin/net7.0"
+        "cwd": "${workspaceFolder}/build/Debug/bin/net8.0",
+        "env": {
+          "DOTNET_ENVIRONMENT": "${input:dataStreamPublisherEnvironment}",
+        },
       },
       "icon": {
         "id": "run",
@@ -595,12 +593,11 @@
       "windows": {
         "command": "DataStreamPublisher.exe",
       },
-      "args": [
-        "--config",
-        "${workspaceFolder}/src/DataStreamSuite/DataStreamPublisher/Configurations/${input:dataStreamPublisherConfig}.json",
-      ],
       "options": {
-        "cwd": "${workspaceFolder}/build/Release/bin/net7.0"
+        "cwd": "${workspaceFolder}/build/Release/bin/net8.0",
+        "env": {
+          "DOTNET_ENVIRONMENT": "${input:dataStreamPublisherEnvironment}",
+        },
       },
       "icon": {
         "id": "run",
@@ -701,12 +698,11 @@
       "windows": {
         "command": "PubSubServer.exe",
       },
-      "args": [
-        "--config",
-        "${workspaceFolder}/src/DataStreamSuite/PubSubServer/Configurations/${input:pubSubServerConfig}.json",
-      ],
       "options": {
-        "cwd": "${workspaceFolder}/build/${input:buildConfiguration}/bin/net7.0"
+        "cwd": "${workspaceFolder}/build/${input:buildConfiguration}/bin/net8.0",
+        "env": {
+          "DOTNET_ENVIRONMENT": "${input:pubSubServerEnvironment}",
+        },
       },
       "icon": {
         "id": "run",
@@ -724,12 +720,11 @@
       "windows": {
         "command": "PubSubServer.exe",
       },
-      "args": [
-        "--config",
-        "${workspaceFolder}/src/DataStreamSuite/PubSubServer/Configurations/${input:pubSubServerConfig}.json",
-      ],
       "options": {
-        "cwd": "${workspaceFolder}/build/Debug/bin/net7.0"
+        "cwd": "${workspaceFolder}/build/Debug/bin/net8.0",
+        "env": {
+          "DOTNET_ENVIRONMENT": "${input:pubSubServerEnvironment}",
+        },
       },
       "icon": {
         "id": "run",
@@ -746,12 +741,11 @@
       "windows": {
         "command": "PubSubServer.exe",
       },
-      "args": [
-        "--config",
-        "${workspaceFolder}/src/DataStreamSuite/PubSubServer/Configurations/${input:pubSubServerConfig}.json",
-      ],
       "options": {
-        "cwd": "${workspaceFolder}/build/Debug/bin/net7.0"
+        "cwd": "${workspaceFolder}/build/Debug/bin/net8.0",
+        "env": {
+          "DOTNET_ENVIRONMENT": "${input:pubSubServerEnvironment}",
+        },
       },
       "icon": {
         "id": "run",
@@ -769,12 +763,11 @@
       "windows": {
         "command": "PubSubServer.exe",
       },
-      "args": [
-        "--config",
-        "${workspaceFolder}/src/DataStreamSuite/PubSubServer/Configurations/${input:pubSubServerConfig}.json",
-      ],
       "options": {
-        "cwd": "${workspaceFolder}/build/Release/bin/net7.0"
+        "cwd": "${workspaceFolder}/build/Release/bin/net8.0",
+        "env": {
+          "DOTNET_ENVIRONMENT": "${input:pubSubServerEnvironment}",
+        },
       },
       "icon": {
         "id": "run",
@@ -791,12 +784,11 @@
       "windows": {
         "command": "PubSubServer.exe",
       },
-      "args": [
-        "--config",
-        "${workspaceFolder}/src/DataStreamSuite/PubSubServer/Configurations/${input:pubSubServerConfig}.json",
-      ],
       "options": {
-        "cwd": "${workspaceFolder}/build/Release/bin/net7.0"
+        "cwd": "${workspaceFolder}/build/Release/bin/net8.0",
+        "env": {
+          "DOTNET_ENVIRONMENT": "${input:pubSubServerEnvironment}",
+        },
       },
       "icon": {
         "id": "run",
@@ -811,7 +803,7 @@
       "windows": {
         "command": "Spark.exe",
         "options": {
-          "cwd": "${workspaceFolder}/build/${input:buildConfiguration}/bin/net7.0-windows"
+          "cwd": "${workspaceFolder}/build/${input:buildConfiguration}/bin/net8.0-windows"
         },
       },
       "icon": {
@@ -827,7 +819,7 @@
       "windows": {
         "command": "Spark.exe",
         "options": {
-          "cwd": "${workspaceFolder}/build/Debug/bin/net7.0-windows"
+          "cwd": "${workspaceFolder}/build/Debug/bin/net8.0-windows"
         },
       },
       "icon": {
@@ -843,7 +835,22 @@
       "windows": {
         "command": "Spark.exe",
         "options": {
-          "cwd": "${workspaceFolder}/build/Release/bin/net7.0-windows"
+          "cwd": "${workspaceFolder}/build/Release/bin/net8.0-windows"
+        },
+      },
+      "icon": {
+        "id": "run",
+        "color": "terminal.ansiGreen",
+      }
+    },
+    {
+      "label": "Run Spark (Skip Build)",
+      "type": "process",
+      "command": "",
+      "windows": {
+        "command": "Spark.exe",
+        "options": {
+          "cwd": "${workspaceFolder}/build/${input:buildConfiguration}/bin/net8.0-windows"
         },
       },
       "icon": {
@@ -868,7 +875,7 @@
         "${input:sparkConfigQuery}",
       ],
       "options": {
-        "cwd": "${workspaceFolder}/build/${input:buildConfiguration}/bin/net7.0"
+        "cwd": "${workspaceFolder}/build/${input:buildConfiguration}/bin/net8.0"
       },
       "icon": {
         "id": "run",
@@ -894,6 +901,24 @@
         "color": "terminal.ansiGreen",
       },
     },
+    {
+      "label": "Run Sparkd (Skip Build)",
+      "type": "process",
+      "command": "",
+      "linux": {
+        "command": "sparkd",
+      },
+      "args": [
+        "run"
+      ],
+      "options": {
+        "cwd": "${workspaceFolder}/build/${input:sparkdCmakePreset}/server"
+      },
+      "icon": {
+        "id": "run",
+        "color": "terminal.ansiGreen",
+      },
+    },
     {
       "label": "Run Sparkd Config Query",
       "dependsOn": "Build Sparkd",
@@ -915,6 +940,65 @@
         "color": "terminal.ansiGreen",
       },
     },
+    {
+      "label": "Test Spark (Debug)",
+      "dependsOn": "Build Spark (Debug)",
+      "type": "process",
+      "icon": {
+        "id": "coverage",
+        "color": "terminal.ansiYellow",
+      },
+      "command": "",
+      "windows": {
+        "command": "Spark.Console.exe",
+        "args": [
+          "test",
+          "${input:sparkTest}"
+        ],
+        "options": {
+          "cwd": "${workspaceFolder}/build/Debug/bin/net8.0",
+        }
+      },
+    },
+    {
+      "label": "Test Sparkd",
+      "dependsOn": "Build Sparkd",
+      "type": "process",
+      "command": "",
+      "linux": {
+        "command": "sparkd",
+      },
+      "args": [
+        "test",
+        "${input:sparkdTest}",
+      ],
+      "options": {
+        "cwd": "${workspaceFolder}/build/${input:sparkdCmakePreset}/server"
+      },
+      "icon": {
+        "id": "coverage",
+        "color": "terminal.ansiYellow",
+      },
+    },
+    {
+      "label": "Test Sparkd (Skip Build)",
+      "type": "process",
+      "command": "",
+      "linux": {
+        "command": "sparkd",
+      },
+      "args": [
+        "test",
+        "${input:sparkdTest}",
+      ],
+      "options": {
+        "cwd": "${workspaceFolder}/build/${input:sparkdCmakePreset}/server"
+      },
+      "icon": {
+        "id": "coverage",
+        "color": "terminal.ansiYellow",
+      },
+    },
     {
       "label": "View DataStreamQueryAnalyzer Logs (Last Run)",
       "type": "process",
@@ -1003,26 +1087,40 @@
       "default": "sparkd-debug-clang",
     },
     {
-      "id": "dataStreamPublisherConfig",
+      "id": "dataStreamPublisherEnvironment",
       "type": "pickString",
       "description": "Publisher Configuration",
       "options": [
-        "MarketDataPublisher",
-        "SimpleEchoClient",
-        "SimpleTestDataPublisher",
-        "TestDataPublisher",
+        "Development",
+        "MarketData",
+        "Production",
+        "ProductionAws",
+        "Staging",
       ],
       "default": "MarketDataPublisher",
     },
     {
-      "id": "pubSubServerConfig",
+      "id": "dataStreamQueryAnalyzerEnvironment",
+      "type": "pickString",
+      "description": "DataStreamQueryAnalyzer Configuration",
+      "options": [
+        "Development",
+        "Local",
+      ],
+      "default": "Development",
+    },
+    {
+      "id": "pubSubServerEnvironment",
       "type": "pickString",
       "description": "PubSub Configuration",
       "options": [
-        "PubSubServer",
-        "SimpleEchoClient",
+        "Development",
+        "EchoServer",
+        "Production",
+        "ProductionAws",
+        "Staging",
       ],
-      "default": "PubSubServer",
+      "default": "Development",
     },
     {
       "id": "sparkConfigQuery",
@@ -1033,6 +1131,16 @@
       "id": "sparkdConfigQuery",
       "type": "promptString",
       "description": "Config Type(s)"
-    }
+    },
+    {
+      "id": "sparkTest",
+      "type": "promptString",
+      "description": "Test to run",
+    },
+    {
+      "id": "sparkdTest",
+      "type": "promptString",
+      "description": "Test to run",
+    },
   ]
 }

CMakeLists.txt

@@ -1,7 +1,10 @@
 cmake_minimum_required(VERSION 3.12)
 project(spark)
+include(GNUInstallDirs)
+include(${CMAKE_SOURCE_DIR}/cmake/GitVersion.cmake)
 set(CMAKE_CXX_STANDARD 20)
 
+get_version_from_git()
 if(WIN32)
   set(platform "windows")
 else()
@@ -33,85 +36,98 @@ else()
   set(BUILD_ADMIN_WEB_APP ON CACHE BOOL "Build the Admin WebApp")
 endif()
 
+include_directories(BEFORE SYSTEM ${EXTERNALS_INCLUDE} ${EXTERNALS_ROOT}/include/zlib)
+link_directories(${EXTERNALS_LIB})
+
 add_compile_definitions(SHARED_MEMORY_PREFIX="${SHARED_MEMORY_PREFIX}")
 if(DEFINED APP_DATA_DIR)
   add_compile_definitions(APP_DATA_DIR="${APP_DATA_DIR}")
 endif()
 add_compile_definitions(EXPORT_MD_CONSUMER="${EXPORT_MD_CONSUMER}")
+if(WIN32)
+  # subdirs
+  add_subdirectory(src/lib_spark)
+else()
+  if(BUILD_SPARKD)
+    # Note: All macros that begin w/ 'CCAPI' are ccapi's macros (defined and/or
+    # read by their code). We override some below.
+    add_compile_definitions(SPARK_ENABLE_CCAPI)
 
-if(BUILD_SPARKD)
-  # Note: All macros that begin w/ 'CCAPI' are ccapi's macros (defined and/or
-  # read by their code). We override some below.
-  add_compile_definitions(SPARK_ENABLE_CCAPI)
+    # This allows us to override hardcoded exchange-urls with ones read from
+    # config (i.e., resolved at runtime rather than compile-time)
+    add_compile_definitions(CCAPI_EXPOSE_INTERNAL)
 
-  # This allows us to override hardcoded exchange-urls with ones read from
-  # config (i.e., resolved at runtime rather than compile-time)
-  add_compile_definitions(CCAPI_EXPOSE_INTERNAL)
+    # Enable ccapi services
+    add_compile_definitions(CCAPI_ENABLE_SERVICE_MARKET_DATA)
+    add_compile_definitions(CCAPI_ENABLE_SERVICE_EXECUTION_MANAGEMENT)
+    # Enable ccapi exchanges
+    add_compile_definitions(CCAPI_ENABLE_EXCHANGE_BINANCE)
+    add_compile_definitions(CCAPI_ENABLE_EXCHANGE_BINANCE_COIN_FUTURES)
+    add_compile_definitions(CCAPI_ENABLE_EXCHANGE_BINANCE_US)
+    add_compile_definitions(CCAPI_ENABLE_EXCHANGE_BINANCE_USDS_FUTURES)
+    add_compile_definitions(CCAPI_ENABLE_EXCHANGE_COINBASE)
+    add_compile_definitions(CCAPI_ENABLE_EXCHANGE_DERIBIT)
+    add_compile_definitions(CCAPI_ENABLE_EXCHANGE_OKX)
 
-  # Enable ccapi services
-  add_compile_definitions(CCAPI_ENABLE_SERVICE_MARKET_DATA)
-  add_compile_definitions(CCAPI_ENABLE_SERVICE_EXECUTION_MANAGEMENT)
-  # Enable ccapi exchanges
-  add_compile_definitions(CCAPI_ENABLE_EXCHANGE_BINANCE)
-  add_compile_definitions(CCAPI_ENABLE_EXCHANGE_BINANCE_COIN_FUTURES)
-  add_compile_definitions(CCAPI_ENABLE_EXCHANGE_BINANCE_US)
-  add_compile_definitions(CCAPI_ENABLE_EXCHANGE_BINANCE_USDS_FUTURES)
-  add_compile_definitions(CCAPI_ENABLE_EXCHANGE_COINBASE)
-  add_compile_definitions(CCAPI_ENABLE_EXCHANGE_DERIBIT)
-  add_compile_definitions(CCAPI_ENABLE_EXCHANGE_OKX)
+    set(CCAPI_LOG_LEVEL
+      "Warn"
+      CACHE STRING "<Fatal|Error|Warn|Info|Debug|Trace> Log level for ccapi")
+    string(TOUPPER "${CCAPI_LOG_LEVEL}" _ccapi_log_level_upper)
+    add_compile_definitions("CCAPI_ENABLE_LOG_${_ccapi_log_level_upper}")
 
-  set(CCAPI_LOG_LEVEL
-    "Warn"
-    CACHE STRING "<Fatal|Error|Warn|Info|Debug|Trace> Log level for ccapi")
-  string(TOUPPER "${CCAPI_LOG_LEVEL}" _ccapi_log_level_upper)
-  add_compile_definitions("CCAPI_ENABLE_LOG_${_ccapi_log_level_upper}")
+    # ** Binance **
+    add_compile_definitions(CCAPI_BINANCE_URL_REST_BASE="https://api.binance.com")
+    add_compile_definitions(BINANCE_INSTRUMENTS_PATH="/api/v3/exchangeInfo")
+    add_compile_definitions(BINANCE_PRICE_STATS_PATH="/api/v3/ticker/24hr")
 
-  # ** Binance **
-  add_compile_definitions(CCAPI_BINANCE_URL_REST_BASE="https://api.binance.com")
-  add_compile_definitions(BINANCE_INSTRUMENTS_PATH="/api/v3/exchangeInfo")
-  add_compile_definitions(BINANCE_PRICE_STATS_PATH="/api/v3/ticker/24hr")
+    # ** Binance US **
+    add_compile_definitions(BINANCE_US_INSTRUMENTS_PATH="/api/v3/exchangeInfo")
+    add_compile_definitions(BINANCE_US_PRICE_STATS_PATH="/api/v3/ticker/24hr")
 
-  # ** Binance US **
-  add_compile_definitions(BINANCE_US_INSTRUMENTS_PATH="/api/v3/exchangeInfo")
-  add_compile_definitions(BINANCE_US_PRICE_STATS_PATH="/api/v3/ticker/24hr")
+    # ** Binance USDS Futures **
+    add_compile_definitions(CCAPI_BINANCE_USDS_FUTURES_URL_REST_BASE="https://fapi.binance.com")
+    add_compile_definitions(BINANCE_USDS_FUTURES_INSTRUMENTS_PATH="/fapi/v1/exchangeInfo")
+    add_compile_definitions(BINANCE_USDS_FUTURES_PRICE_STATS_PATH="/fapi/v1/ticker/24hr")
+    add_compile_definitions(BINANCE_USDS_FUTURES_USER_DATA_PATH="/api/v3/userDataStream")
 
-  # ** Binance USDS Futures **
-  add_compile_definitions(CCAPI_BINANCE_USDS_FUTURES_URL_REST_BASE="https://dapi.binance.com")
-  add_compile_definitions(BINANCE_USDS_FUTURES_INSTRUMENTS_PATH="/dapi/v1/exchangeInfo")
-  add_compile_definitions(BINANCE_USDS_FUTURES_PRICE_STATS_PATH="/dapi/v1/ticker/24hr")
+    # ** Binance Coin Futures **
+    add_compile_definitions(CCAPI_BINANCE_COIN_FUTURES_URL_REST_BASE="https://dapi.binance.com")
+    add_compile_definitions(BINANCE_COIN_FUTURES_INSTRUMENTS_PATH="/dapi/v1/exchangeInfo")
+    add_compile_definitions(BINANCE_COIN_FUTURES_PRICE_STATS_PATH="/dapi/v1/ticker/24hr")
 
-  # ** Deribit **
-  add_compile_definitions(DERIBIT_INSTRUMENT_PATH="/api/v2/public/get_instrument")
-  add_compile_definitions(DERIBIT_INSTRUMENTS_PATH="/api/v2/public/get_instruments")
-  add_compile_definitions(DERIBIT_COMBOS_PATH="/api/v2/public/get_combos")
-  add_compile_definitions(DERIBIT_TICKER_PATH="/api/v2/public/ticker")
-  add_compile_definitions(DERIBIT_CURRENCIES_METHOD="public/get_currencies")
-  add_compile_definitions(DERIBIT_INSTRUMENT_METHOD="public/get_instrument")
-  add_compile_definitions(DERIBIT_INSTRUMENTS_METHOD="public/get_instruments")
-  add_compile_definitions(DERIBIT_COMBO_DETAILS_METHOD="public/get_combo_details")
-  add_compile_definitions(DERIBIT_COMBOS_METHOD="public/get_combos")
+    # ** Deribit **
+    add_compile_definitions(DERIBIT_INSTRUMENT_PATH="/api/v2/public/get_instrument")
+    add_compile_definitions(DERIBIT_INSTRUMENTS_PATH="/api/v2/public/get_instruments")
+    add_compile_definitions(DERIBIT_COMBOS_PATH="/api/v2/public/get_combos")
+    add_compile_definitions(DERIBIT_TICKER_PATH="/api/v2/public/ticker")
+    add_compile_definitions(DERIBIT_CURRENCIES_METHOD="public/get_currencies")
+    add_compile_definitions(DERIBIT_INSTRUMENT_METHOD="public/get_instrument")
+    add_compile_definitions(DERIBIT_INSTRUMENTS_METHOD="public/get_instruments")
+    add_compile_definitions(DERIBIT_COMBO_DETAILS_METHOD="public/get_combo_details")
+    add_compile_definitions(DERIBIT_COMBOS_METHOD="public/get_combos")
 
-  # ** Okx **
-  add_compile_definitions(OKX_INSTRUMENTS_PATH="/api/v5/public/instruments")
-  add_compile_definitions(OKX_UNDERLYING_PATH="/api/v5/public/underlying")
-endif()
+    # ** Okx **
+    add_compile_definitions(OKX_INSTRUMENTS_PATH="/api/v5/public/instruments")
+    add_compile_definitions(OKX_UNDERLYING_PATH="/api/v5/public/underlying")
+  endif()
 
-# subdirs
-add_subdirectory(src/lib_system)
-add_subdirectory(src/lib_domain)
-add_subdirectory(src/lib_http)
-if (BUILD_SPARKD)
-  add_subdirectory(src/lib_python_md)
-  add_subdirectory(src/app_sparkd)
-  add_subdirectory(src/data_files)
-endif()
-add_subdirectory(src/lib_spark)
-if (BUILD_ADMIN_WEB_APP)
-  add_subdirectory(src/Spark.Admin.WebApp)
-endif()
-if(BUILD_DOCS)
-  add_subdirectory(doc/user)
-endif()
-if(BUILD_SDK)
-  add_subdirectory(src/sdk)
+  # subdirs
+  add_subdirectory(src/lib_spark)
+  if (BUILD_SPARKD)
+    add_subdirectory(src/lib_python_interop)
+    add_subdirectory(src/lib_onixs_fix)
+    add_subdirectory(src/lib_binance)
+    add_subdirectory(src/lib_okx_execution)
+    add_subdirectory(src/app_sparkd)
+    add_subdirectory(src/data_files)
+  endif()
+  if (BUILD_ADMIN_WEB_APP)
+    add_subdirectory(src/Spark.Admin.WebApp)
+  endif()
+  if(BUILD_DOCS)
+    add_subdirectory(doc/user)
+  endif()
+  if(BUILD_SDK)
+    add_subdirectory(src/sdk)
+  endif()
 endif()

CMakePresets.json

@@ -24,12 +24,13 @@
         "CMAKE_CXX_FLAGS_DEBUG": "$env{SPARK_CMAKE_CXX_FLAGS} /MTd",
         "CMAKE_CXX_FLAGS_RELEASE": "$env{SPARK_CMAKE_CXX_FLAGS} /MT",
         "CMAKE_CXX_STANDARD": "20",
+        "CMAKE_INSTALL_PREFIX": "${sourceDir}/install/${presetName}",
         "CMAKE_SHARED_LINKER_FLAGS_DEBUG": "/DEBUG",
         "CMAKE_SHARED_LINKER_FLAGS_RELEASE": "/OPT:REF /OPT:ICF /LTCG",
         "CMAKE_TOOLCHAIN_FILE": "${sourceDir}/vcpkg/scripts/buildsystems/vcpkg.cmake",
         "EXTERNALS_ROOT": "${sourceDir}/externals/windows",
         "CONFIG_DEFAULTS_PATH": "spark/config/profiles/default/*.conf",
-        "DOTNET_BINARY_DIR": "${sourceDir}/build/$<CONFIG>/bin/net7.0;${sourceDir}/build/$<CONFIG>/bin/net7.0-windows",
+        "DOTNET_BINARY_DIR": "${sourceDir}/build/$<CONFIG>/bin/net8.0;${sourceDir}/build/$<CONFIG>/bin/net8.0-windows",
         "EXPORT_MD_CONSUMER": {
           "type": "BOOL",
           "value": "ON"
@@ -49,11 +50,20 @@
       "binaryDir": "${sourceDir}/build/${presetName}",
       "cacheVariables": {
         "platform": "",
+        "CMAKE_BUILD_WITH_INSTALL_RPATH": {
+          "type": "BOOL",
+          "value": "OFF"
+        },
         "CMAKE_CXX_STANDARD": "20",
+        "CMAKE_INSTALL_RPATH": ".",
         "CMAKE_INSTALL_PREFIX": "${sourceDir}/install/${presetName}",
         "CMAKE_PREFIX_PATH": "${sourceDir}/cmake/;${sourceDir}/.venv/lib/python3.10/site-packages/pybind11/share/cmake/pybind11",
         "CMAKE_EXE_LINKER_FLAGS": "-Wno-undef",
         "CMAKE_TOOLCHAIN_FILE": "${sourceDir}/vcpkg/scripts/buildsystems/vcpkg.cmake",
+        "DATABENTO_USE_EXTERNAL_JSON": {
+          "type": "BOOL",
+          "value": "ON"
+        },
         "EXTERNALS_ROOT": "${sourceDir}/externals/linux",
         "BUILD_SDK": {
           "type": "BOOL",
@@ -164,12 +174,13 @@
         "CMAKE_CXX_COMPILER": "clang++",
         "CMAKE_CXX_FLAGS_DEBUG": "-g -fPIC -Wall -pedantic -Wextra -Werror -Wno-missing-braces -Wno-ignored-qualifiers --std=c++20 -fdiagnostics-color -DDEBUG",
         "CMAKE_CXX_FLAGS_RELEASE": "-fPIC -Wall -pedantic -Wextra -Werror -Wno-missing-braces -Wno-ignored-qualifiers --std=c++20 -fdiagnostics-color -fno-omit-frame-pointer -DNDEBUG -O3",
+        "CMAKE_INSTALL_PREFIX": "${sourceDir}/install/${presetName}",
         "CMAKE_EXPORT_COMPILE_COMMANDS": {
           "type": "BOOL",
           "value": "ON"
         },
         "CONFIG_DEFAULTS_PATH": "spark/config/profiles/default/*.conf",
-        "DOTNET_BINARY_DIR": "${sourceDir}/build/$<CONFIG>/bin/net7.0",
+        "DOTNET_BINARY_DIR": "${sourceDir}/build/Spark.Api.Server/$<CONFIG>/bin/net8.0",
         "EXPORT_MD_CONSUMER": {
           "type": "BOOL",
           "value": "ON"
@@ -208,7 +219,7 @@
       "binaryDir": "${sourceDir}/build/${presetName}",
       "cacheVariables": {
         "CONFIG_DEFAULTS_PATH": "spark/config/profiles/default/*.conf;spark/config/profiles/data_stream/*.conf",
-        "DOTNET_BINARY_DIR": "${sourceDir}/build/$<CONFIG>/bin/net7.0;${sourceDir}/build/$<CONFIG>/bin/net7.0-windows"
+        "DOTNET_BINARY_DIR": "${sourceDir}/build/$<CONFIG>/bin/net8.0;${sourceDir}/build/$<CONFIG>/bin/net8.0-windows"
       }
     },
 
@@ -241,7 +252,7 @@
         "CMAKE_CXX_FLAGS_DEBUG": "-g -fPIC -Wall -pedantic -Wextra -Werror -Wno-missing-braces -Wno-ignored-qualifiers --std=c++20 -fdiagnostics-color -DDEBUG",
         "CMAKE_CXX_FLAGS_RELEASE": "-fPIC -Wall -pedantic -Wextra -Werror -Wno-missing-braces -Wno-ignored-qualifiers --std=c++20 -fdiagnostics-color -fno-omit-frame-pointer -DNDEBUG -O3",
         "CONFIG_DEFAULTS_PATH": "spark/config/profiles/default/*.conf;spark/config/profiles/data_stream/*.conf",
-        "DOTNET_BINARY_DIR": "${sourceDir}/build/$<CONFIG>/bin/net7.0;${sourceDir}/build/$<CONFIG>/bin/net8.0",
+        "DOTNET_BINARY_DIR": "${sourceDir}/build/$<CONFIG>/bin/net7.0",
         "EXPORT_MD_CONSUMER": {
           "type": "BOOL",
           "value": "ON"
@@ -294,14 +305,6 @@
       "inherits": "sparkd"
     },
 
-    {
-      "name": "sparkd-debug-clang-j6",
-      "configurePreset": "sparkd-debug-clang",
-      "displayName": "Sparkd (Debug | Clang | jobs=6)",
-      "inherits": "sparkd",
-      "jobs": 6
-    },
-
     {
       "name": "sparkd-debug-gcc",
       "configurePreset": "sparkd-debug-gcc",

Spark.Api.Server.sln

@@ -42,8 +42,6 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Bts.ColumnExpressions", "sr
 EndProject
 Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "DataStream", "src\DataStreamSuite\DataStream\DataStream.csproj", "{39AEA075-6457-46C0-8821-1158BE6EFA57}"
 EndProject
-Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Bts.Net", "src\DataStreamSuite\Bts.Net\Bts.Net.csproj", "{CCDCB00A-CABF-4D82-8A5F-33EEC8ACE51D}"
-EndProject
 Global
 	GlobalSection(SolutionConfigurationPlatforms) = preSolution
 		Debug|x64 = Debug|x64
@@ -196,14 +194,6 @@ Global
 		{6BD3F6D8-B6FC-44E8-B8FE-06352F638B9B}.Release|x64.Build.0 = Release|Any CPU
 		{6BD3F6D8-B6FC-44E8-B8FE-06352F638B9B}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
 		{6BD3F6D8-B6FC-44E8-B8FE-06352F638B9B}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
-		{CCDCB00A-CABF-4D82-8A5F-33EEC8ACE51D}.Debug|x64.ActiveCfg = Debug|Any CPU
-		{CCDCB00A-CABF-4D82-8A5F-33EEC8ACE51D}.Debug|x64.Build.0 = Debug|Any CPU
-		{CCDCB00A-CABF-4D82-8A5F-33EEC8ACE51D}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
-		{CCDCB00A-CABF-4D82-8A5F-33EEC8ACE51D}.MinSizeRel|x64.Build.0 = Debug|Any CPU
-		{CCDCB00A-CABF-4D82-8A5F-33EEC8ACE51D}.Release|x64.ActiveCfg = Release|Any CPU
-		{CCDCB00A-CABF-4D82-8A5F-33EEC8ACE51D}.Release|x64.Build.0 = Release|Any CPU
-		{CCDCB00A-CABF-4D82-8A5F-33EEC8ACE51D}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
-		{CCDCB00A-CABF-4D82-8A5F-33EEC8ACE51D}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
 	EndGlobalSection
 	GlobalSection(SolutionProperties) = preSolution
 		HideSolutionNode = FALSE

cmake/GitVersion.cmake

@@ -0,0 +1,48 @@
+include_guard(GLOBAL)
+
+function(get_version_from_git)
+  find_package(Git QUIET)
+  if(NOT Git_FOUND)
+    message(WARNING "Git not found")
+    return()
+  endif()
+
+  execute_process(
+    COMMAND ${GIT_EXECUTABLE} describe --tags --always
+    WORKING_DIRECTORY ${CMAKE_SOURCE_DIR}
+    OUTPUT_VARIABLE GIT_TAG
+    OUTPUT_STRIP_TRAILING_WHITESPACE
+    RESULT_VARIABLE GIT_RESULT
+  )
+
+  if(NOT GIT_RESULT EQUAL 0)
+    message(WARNING "Failed to get git tag")
+    return()
+  endif()
+
+  execute_process(
+    COMMAND ${GIT_EXECUTABLE} rev-parse --short=7 HEAD
+    WORKING_DIRECTORY ${CMAKE_SOURCE_DIR}
+    OUTPUT_VARIABLE GIT_COMMIT_SHORT_HASH
+    OUTPUT_STRIP_TRAILING_WHITESPACE
+  )
+  set(PROJECT_VERSION_BUILD_METADATA ${GIT_COMMIT_SHORT_HASH})
+  set(PROJECT_VERSION_BUILD_METADATA ${GIT_COMMIT_SHORT_HASH} PARENT_SCOPE)
+
+  string(REGEX REPLACE "^v" "" CLEAN_TAG "${GIT_TAG}")
+  if(CLEAN_TAG MATCHES "^([0-9]+)\\.([0-9]+)\\.([0-9]+)(-.*)?$")
+    set(PROJECT_VERSION_MAJOR ${CMAKE_MATCH_1})
+    set(PROJECT_VERSION_MAJOR ${CMAKE_MATCH_1} PARENT_SCOPE)
+    set(PROJECT_VERSION_MINOR ${CMAKE_MATCH_2})
+    set(PROJECT_VERSION_MINOR ${CMAKE_MATCH_2} PARENT_SCOPE)
+    set(PROJECT_VERSION_PATCH ${CMAKE_MATCH_3})
+    set(PROJECT_VERSION_PATCH ${CMAKE_MATCH_3} PARENT_SCOPE)
+
+    set(PROJECT_VERSION "${CMAKE_MATCH_1}.${CMAKE_MATCH_2}.${CMAKE_MATCH_3}") 
+    set(PROJECT_VERSION "${CMAKE_MATCH_1}.${CMAKE_MATCH_2}.${CMAKE_MATCH_3}" PARENT_SCOPE)
+    set(PROJECT_VERSION_FULL "${CMAKE_MATCH_1}.${CMAKE_MATCH_2}.${CMAKE_MATCH_3}+${GIT_COMMIT_SHORT_HASH}")
+    set(PROJECT_VERSION_FULL "${CMAKE_MATCH_1}.${CMAKE_MATCH_2}.${CMAKE_MATCH_3}+${GIT_COMMIT_SHORT_HASH}" PARENT_SCOPE)
+  else()
+    message(WARNING "Tag '${CLEAN_TAG}' does not match semver format")
+  endif()
+endfunction()

cmake/InitPythonModule.cmake

@@ -0,0 +1,71 @@
+include_guard(DIRECTORY)
+include(GNUInstallDirs)
+include(${CMAKE_SOURCE_DIR}/cmake/ExportToParent.cmake)
+include(${CMAKE_SOURCE_DIR}/cmake/SetFirstDefined.cmake)
+
+function(init_python_module)
+  set(_options)
+  set(_one_value_args
+    NAME
+  )
+  set(_multi_value_args
+    SOURCES
+  )
+  cmake_parse_arguments(_ARGS "${_options}" "${_one_value_args}" "${_multi_value_args}" ${ARGN})
+  set_first_defined(_name _ARGS_NAME PROJECT_NAME)
+  set_first_defined(_sources _ARGS_SOURCES ${_name}_SOURCES)
+
+  set(module_proj ${_name}_module)
+  set(${module_proj}_SOURCES ${_sources})
+  pybind11_add_module(${module_proj} SHARED ${_sources})
+  target_link_libraries(
+    ${module_proj}
+    PUBLIC
+      pybind11::embed
+      spark
+      ${_name}
+  )
+  target_include_directories(
+    ${module_proj}
+    PRIVATE
+      $<BUILD_INTERFACE:${pybind11_INCLUDE_DIRS}>
+  )
+  target_include_directories(
+    ${module_proj}
+    PUBLIC
+      ${Python3_INCLUDE_DIRS}
+      $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}>
+      $<INSTALL_INTERFACE:${CMAKE_INSTALL_INCLUDEDIR}>
+  )
+  add_custom_command(
+    OUTPUT ${CMAKE_CURRENT_BINARY_DIR}/stubs/${_name}.pyi
+    DEPENDS ${_name} ${module_proj}
+    COMMAND PYTHONPATH=${CMAKE_CURRENT_BINARY_DIR} pybind11-stubgen --numpy-array-use-type-var --ignore-invalid-expressions '.*' ${_name}_module
+    COMMENT "Generating py stubs"
+  )
+  add_custom_target(
+    "${_name}_py_stubs"
+    ALL
+    DEPENDS
+      ${CMAKE_CURRENT_BINARY_DIR}/stubs/${_name}.pyi
+  )
+  install(
+    TARGETS ${module_proj}
+    EXPORT ${module_proj}Targets
+    RUNTIME
+      DESTINATION ${CMAKE_INSTALL_BINDIR}
+    LIBRARY
+      DESTINATION ${CMAKE_INSTALL_LIBDIR}
+    ARCHIVE
+      DESTINATION ${CMAKE_INSTALL_LIBDIR}
+  )
+  foreach(dir ${pybind11_INCLUDE_DIRS})
+    install(DIRECTORY ${dir} DESTINATION ${CMAKE_INSTALL_INCLUDEDIR})
+  endforeach()
+  install(
+    EXPORT ${module_proj}Targets
+    FILE ${module_proj}Targets.cmake
+    NAMESPACE ${_name}::
+    DESTINATION ${CMAKE_INSTALL_LIBDIR}/cmake/${_name}
+  )
+endfunction()

cmake/InitPythonPlugin.cmake

@@ -0,0 +1,65 @@
+include_guard(DIRECTORY)
+include(GNUInstallDirs)
+
+function(init_python_plugin)
+  set(_options)
+  set(_one_value_args NAME PLUGIN_NAME)
+  set(_multi_value_args SOURCES)
+  cmake_parse_arguments(_ARGS "${_options}" "${_one_value_args}" "${_multi_value_args}" ${ARGN})
+  set_first_defined(_name _ARGS_NAME PROJECT_NAME)
+  set(_plugin_name ${_ARGS_PLUGIN_NAME})
+  if (NOT DEFINED _ARGS_PLUGIN_NAME)
+    set(_plugin_name ${_name}_plugins)
+  endif()
+  set(${_plugin_name}_SOURCES ${_ARGS_SOURCES})
+  if (NOT DEFINED _ARGS_SOURCES)
+    file(
+      GLOB_RECURSE ${_plugin_name}_SOURCES
+      RELATIVE ${CMAKE_CURRENT_SOURCE_DIR}
+      ${_plugin_name}/*.py)
+  endif()
+  set(src_outs)
+  foreach(_src ${${_plugin_name}_SOURCES})
+    set(_src_in ${CMAKE_CURRENT_SOURCE_DIR}/${_src})
+    set(_src_out ${CMAKE_CURRENT_BINARY_DIR}/${_src})
+    add_custom_command(
+      OUTPUT ${_src_out}
+      COMMAND ${CMAKE_COMMAND} -E copy ${_src_in} ${_src_out}
+      DEPENDS ${_src_in}
+      COMMENT "Copying ${_plugin_name} plugin ${_src} to ${_src_out}"
+    )
+    list(APPEND src_outs ${_src_out})
+  endforeach()
+  
+  add_custom_target(copy_${_plugin_name} ALL DEPENDS ${src_outs})
+  
+  add_library(${_plugin_name} INTERFACE)
+  add_library(${_name}::${_plugin_name} ALIAS ${_plugin_name})
+  
+  set_target_properties(
+    ${_plugin_name}
+    PROPERTIES
+      INTERFACE_PYTHON_SCRIPTS_DIR
+        $<BUILD_INTERFACE:${CMAKE_CURRENT_BINARY_DIR}/${_plugin_name}>$<INSTALL_INTERFACE:${CMAKE_INSTALL_BINDIR}/${_plugin_name}>
+  )
+  
+  set_target_properties(
+    ${_plugin_name}
+    PROPERTIES
+    INTERFACE_PYTHON_SCRIPTS $<BUILD_INTERFACE:${CMAKE_CURRENT_BINARY_DIR}/${${_plugin_name}_SOURCES}>$<INSTALL_INTERFACE:${CMAKE_INSTALL_BINDIR}/${${_plugin_name}_SOURCES}>
+  )
+  
+  install(
+    TARGETS ${_plugin_name}
+    EXPORT ${_plugin_name}Targets
+    DESTINATION ${CMAKE_INSTALL_BINDIR}
+  )
+  
+  install(
+    EXPORT ${_plugin_name}Targets
+    FILE ${_plugin_name}Targets.cmake
+    NAMESPACE ${_name}::
+    DESTINATION ${CMAKE_INSTALL_LIBDIR}/cmake/${_name}
+  )
+  install(DIRECTORY ${_plugin_name} DESTINATION ${CMAKE_INSTALL_BINDIR})
+endfunction()

cmake/Protobuf.cmake

@@ -0,0 +1,182 @@
+set(Protobuf_PROTOC_EXECUTABLE "${EXTERNALS_BIN}/protoc")
+set(PROTOBUF_INCLUDE_DIRS "${EXTERNALS_INCLUDE}")
+set(PROTOBUF_STATIC_LIB "libprotobuf")
+set(_protobuf_src_dir ${CMAKE_SOURCE_DIR}/src/lib_spark)
+set(_protobuf_bin_dir ${CMAKE_BINARY_DIR}/src/lib_spark)
+
+function(Protobuf_Generate_Cpp)
+  set(_options)
+  set(_one_value_args CXX_SOURCES CXX_HEADERS CXX_OUT_DIR)
+  set(_multi_value_args PROTO_INCLUDE_DIRS PROTO_SOURCES)
+  cmake_parse_arguments(_ARGS "${_options}" "${_one_value_args}" "${_multi_value_args}" ${ARGN})
+  set(${_ARGS_CXX_SOURCES})
+  set(${_ARGS_CXX_HEADERS})
+
+  set(_proto_include_cli_args)
+  foreach(_dir ${_ARGS_PROTO_INCLUDE_DIRS})
+    list(APPEND _proto_include_cli_args -I ${_dir})
+  endforeach()
+
+  foreach(FIL ${_ARGS_PROTO_SOURCES})
+    get_filename_component(ABS_FIL ${FIL} ABSOLUTE)
+    get_filename_component(FIL_WE ${FIL} NAME_WE)
+    get_filename_component(FIL_DIR ${FIL} DIRECTORY)
+    if(FIL_DIR)
+      set(FIL_WE "${FIL_DIR}/${FIL_WE}")
+    endif()
+
+    list(APPEND ${_ARGS_CXX_SOURCES} "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.pb.cc")
+    list(APPEND ${_ARGS_CXX_HEADERS} "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.pb.h")
+
+    add_custom_command(
+      OUTPUT
+        "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.pb.cc"
+        "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.pb.h"
+      COMMAND
+        ${Protobuf_PROTOC_EXECUTABLE}
+      ARGS
+        --cpp_out=${_ARGS_CXX_OUT_DIR} ${_proto_include_cli_args} ${ABS_FIL}
+      DEPENDS ${ABS_FIL}
+      COMMENT "Running C++ protocol buffer compiler on ${FIL}"
+      VERBATIM
+    )
+  endforeach()
+
+  set_source_files_properties(
+    ${${_ARGS_CXX_SOURCES}}
+    ${${_ARGS_CXX_HEADERS}}
+    PROPERTIES
+      GENERATED TRUE
+  )
+
+  # remove some warnings from protobuf generated files
+  if(WIN32)
+    set_source_files_properties(
+      ${${_ARGS_CXX_SOURCES}}
+      PROPERTIES
+        COMPILE_FLAGS "/wd4309 /wd4838"
+    )
+  else()
+    set_source_files_properties(
+      ${${_ARGS_CXX_SOURCES}}
+      PROPERTIES
+        COMPILE_FLAGS "-Wno-unused-parameter -Wno-effc++ -Wno-pedantic -Wno-sign-compare -Wno-uninitialized"
+    )
+  endif()
+
+  set(${_ARGS_CXX_SOURCES} ${${_ARGS_CXX_SOURCES}} PARENT_SCOPE)
+  set(${_ARGS_CXX_HEADERS} ${${_ARGS_CXX_HEADERS}} PARENT_SCOPE)
+endfunction()
+
+function(Service_Compiler_Generate_Cpp)
+  set(_options)
+  set(_one_value_args CXX_SOURCES CXX_HEADERS CXX_OUT_DIR)
+  set(_multi_value_args PROTO_INCLUDE_DIRS PROTO_SOURCES)
+  cmake_parse_arguments(_ARGS "${_options}" "${_one_value_args}" "${_multi_value_args}" ${ARGN})
+  set(${_ARGS_CXX_SOURCES})
+  set(${_ARGS_CXX_HEADERS})
+
+  set(_proto_include_cli_args)
+  foreach(_dir ${_ARGS_PROTO_INCLUDE_DIRS})
+    list(APPEND _proto_include_cli_args -I ${_dir})
+  endforeach()
+
+  foreach(FIL ${_ARGS_PROTO_SOURCES})
+    get_filename_component(ABS_FIL ${FIL} ABSOLUTE)
+    get_filename_component(FIL_WE ${FIL} NAME_WE)
+    get_filename_component(FIL_DIR ${FIL} DIRECTORY)
+    if(FIL_DIR)
+      set(FIL_WE "${FIL_DIR}/${FIL_WE}")
+    endif()
+
+    list(APPEND ${_ARGS_CXX_SOURCES} "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.pb.cc")
+    list(APPEND ${_ARGS_CXX_HEADERS} "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.pb.h")
+
+    list(APPEND ${_ARGS_CXX_SOURCES} "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.client.pb.cc")
+    list(APPEND ${_ARGS_CXX_HEADERS} "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.client.pb.h")
+
+    list(APPEND ${_ARGS_CXX_SOURCES} "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.server.pb.cc")
+    list(APPEND ${_ARGS_CXX_HEADERS} "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.server.pb.h")
+
+    set(TEMPLATES
+      ${CMAKE_SOURCE_DIR}/src/lib_spark/spark/api/service_compiler/templates/cpp/client.pb.cc
+      ${CMAKE_SOURCE_DIR}/src/lib_spark/spark/api/service_compiler/templates/cpp/client.pb.h
+      ${CMAKE_SOURCE_DIR}/src/lib_spark/spark/api/service_compiler/templates/cpp/server.pb.cc
+      ${CMAKE_SOURCE_DIR}/src/lib_spark/spark/api/service_compiler/templates/cpp/server.pb.h
+    )
+
+    set(_plugin_py ${CMAKE_SOURCE_DIR}/src/lib_spark/spark/api/service_compiler/plugin.py)
+    if(WIN32)
+      set(_plugin_cmd ${CMAKE_SOURCE_DIR}/src/lib_spark/spark/api/service_compiler/plugin.cmd)
+      add_custom_command(
+        OUTPUT
+          "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.pb.cc"
+          "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.pb.h"
+          "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.client.pb.cc"
+          "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.client.pb.h"
+          "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.server.pb.cc"
+          "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.server.pb.h"
+        COMMAND SET PYTHONPATH=${SPARK_PYTHON}
+        COMMAND ${CMAKE_SOURCE_DIR}/.venv/Scripts/activate.bat
+        COMMAND
+          ${Protobuf_PROTOC_EXECUTABLE}
+          ARGS
+            --cpp_out=${_ARGS_CXX_OUT_DIR}
+            --plugin=protoc-gen-service=${_plugin_cmd}
+            --service_out=lang=cpp:${_ARGS_CXX_OUT_DIR}
+            -I ${PROTOBUF_INCLUDE_DIRS}
+            -I ${_protobuf_src_dir}
+            ${ABS_FIL}
+        DEPENDS
+          ${ABS_FIL}
+          ${TEMPLATES}
+          ${_plugin_cmd}
+          ${_plugin_py}
+        COMMENT "Running C++ service compiler on ${FIL}"
+        VERBATIM
+      )
+    else()
+      add_custom_command(
+        OUTPUT
+          "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.pb.cc"
+          "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.pb.h"
+          "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.client.pb.cc"
+          "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.client.pb.h"
+          "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.server.pb.cc"
+          "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.server.pb.h"
+        COMMAND ${CMAKE_SOURCE_DIR}/tools/venv_exec.sh
+        ARGS
+          ${Protobuf_PROTOC_EXECUTABLE}
+          --cpp_out=${_ARGS_CXX_OUT_DIR}
+          --plugin=protoc-gen-service=${_plugin_py}
+          --service_out=lang=cpp,make_file_name=${ABS_FIL}:${_ARGS_CXX_OUT_DIR}
+          ${_proto_include_cli_args} ${ABS_FIL}
+        DEPENDS
+          ${ABS_FIL}
+          ${TEMPLATES}
+          ${_plugin_py}
+        COMMENT "Running C++ service compiler on ${FIL}"
+        VERBATIM
+      )
+    endif()
+  endforeach()
+
+  # remove some warnings from protobuf generated files
+  if(WIN32)
+    set_source_files_properties(
+      ${${_ARGS_CXX_SOURCES}}
+      PROPERTIES
+        COMPILE_FLAGS "/wd4309 /wd4838"
+    )
+  else()
+    set_source_files_properties(
+      ${${_ARGS_CXX_SOURCES}}
+      PROPERTIES
+        COMPILE_FLAGS "-Wno-unused-parameter -Wno-effc++ -Wno-pedantic -Wno-sign-compare -Wno-uninitialized"
+    )
+  endif()
+
+  set(${_ARGS_CXX_SOURCES} ${${_ARGS_CXX_SOURCES}} PARENT_SCOPE)
+  set(${_ARGS_CXX_HEADERS} ${${_ARGS_CXX_HEADERS}} PARENT_SCOPE)
+endfunction()
+

cmake/ProtobufGenerateCpp.cmake

@@ -39,6 +39,8 @@ function(protobuf_generate_cpp)
     list(APPEND ${_ARGS_CXX_SOURCES} "${_ARGS_CXX_OUT_DIR}/${_fil_we}.pb.cc")
     list(APPEND ${_ARGS_CXX_HEADERS} "${_ARGS_CXX_OUT_DIR}/${_fil_we}.pb.h")
     message(STATUS "add_custom_command for ${_fil} -> ${_ARGS_CXX_OUT_DIR}/${_fil_we}.pb.h")
+    # TODO: Pass proto sources by relative path (so this fn can be called from
+    # arbitrary subdirs).
     add_custom_command(
       OUTPUT
         "${_ARGS_CXX_OUT_DIR}/${_fil_we}.pb.cc"
@@ -46,7 +48,7 @@ function(protobuf_generate_cpp)
       COMMAND
         ${_ARGS_PROTOC}
       ARGS
-        --cpp_out=${_ARGS_CXX_OUT_DIR} ${_proto_include_cli_args} ${_fil}
+        --cpp_out=${_ARGS_CXX_OUT_DIR} ${_proto_include_cli_args} ${_abs_fil}
       DEPENDS ${_abs_fil}
       COMMENT "Running C++ protocol buffer compiler on ${_fil}"
       VERBATIM
@@ -78,3 +80,4 @@ function(protobuf_generate_cpp)
   set(${_ARGS_CXX_SOURCES} ${${_ARGS_CXX_SOURCES}} PARENT_SCOPE)
   set(${_ARGS_CXX_HEADERS} ${${_ARGS_CXX_HEADERS}} PARENT_SCOPE)
 endfunction()
+

cmake/ProtobufGenerateCppConfig.cmake

@@ -1,3 +1,5 @@
+include_guard(GLOBAL)
+
 function(protobuf_generate_cpp_config)
   set(_options)
   set(_one_value_args
@@ -56,3 +58,4 @@ function(protobuf_generate_cpp_config)
     file(WRITE "${_ARGS_PROTO_DEFAULTS_HEADER}" "${_cfg_defaults_str}")
   endif()
 endfunction()
+

cmake/SymLinkPythonModule.cmake

@@ -0,0 +1,39 @@
+include_guard(DIRECTORY)
+include(${CMAKE_SOURCE_DIR}/cmake/SymLink.cmake)
+
+function(sym_link_python_module)
+  set(_options)
+  set(_one_value_args LIB_NAME MODULE_NAME PLUGIN_NAME)
+  set(_multi_value_args)
+  cmake_parse_arguments(_ARGS "${_options}" "${_one_value_args}"
+                        "${_multi_value_args}" ${ARGN})
+  if(NOT DEFINED _ARGS_LIB_NAME)
+    message(
+      FATAL_ERROR "LIB_NAME must be specified for sym_link_python_module()")
+  endif()
+  set(_lib_name ${_ARGS_LIB_NAME})
+  set(_module_name ${_ARGS_MODULE_NAME})
+  if(NOT DEFINED _ARGS_MODULE_NAME)
+    set(_module_name ${_lib_name}_module)
+  endif()
+  set(_plugin_name ${_ARGS_PLUGIN_NAME})
+  set(_deps ${_lib_name} ${_module_name})
+  if(DEFINED _ARGS_PLUGIN_NAME)
+    list(APPEND _deps ${_plugin_name} copy_${_plugin_name})
+  endif()
+  add_dependencies(${PROJECT_NAME} ${_deps})
+  sym_link_local_lib(${_lib_name})
+  set(_module_suffix cpython-310-x86_64-linux-gnu.so)
+  sym_link_file(
+    SOURCE
+    ${CMAKE_BINARY_DIR}/src/lib_${_lib_name}/${_module_name}.${_module_suffix}
+    DEST
+    ${CMAKE_CURRENT_BINARY_DIR}/${_module_name}.${_module_suffix}
+    INSTALL_DEST
+    ${CMAKE_INSTALL_BINDIR}/)
+  if(DEFINED _ARGS_PLUGIN_NAME)
+    get_target_property(plugins ${_plugin_name} INTERFACE_PYTHON_SCRIPTS_DIR)
+    sym_link_dir(TARGET_NAME ${PROJECT_NAME}_link_${_plugin_name} SOURCE
+                 ${plugins} DEST ${CMAKE_CURRENT_BINARY_DIR}/${_plugin_name})
+  endif()
+endfunction()

cmake/version.h.in

@@ -0,0 +1,11 @@
+#ifndef BTS_SPARK_VERSION_H
+#define BTS_SPARK_VERSION_H
+
+#define BTS_SPARK_VERSION_MAJOR @PROJECT_VERSION_MAJOR@
+#define BTS_SPARK_VERSION_MINOR @PROJECT_VERSION_MINOR@
+#define BTS_SPARK_VERSION_PATCH @PROJECT_VERSION_PATCH@
+#define BTS_SPARK_VERSION_BUILD_METADATA @PROJECT_VERSION_BUILD_METADATA@
+#define BTS_SPARK_VERSION "@PROJECT_VERSION@"
+#define BTS_SPARK_VERSION_FULL "@PROJECT_VERSION_FULL@"
+
+#endif

cmake/vs_filters.cmake

@@ -1,24 +0,0 @@
-cmake_minimum_required(VERSION 3.5)
-
-# Explicitly set policy to convert relative paths to absolute (although this is a default policy,
-# setting it explicitly prevents a compile-warning)
-cmake_policy(SET CMP0076 NEW)
-
-# add protos to the library to facilitate source_group
-target_sources(spark PUBLIC ${Config_Protos} ${NonConfig_Protos} ${Service_Protos})
-
-function(assign_source_group)
-  foreach(_source IN ITEMS ${ARGN})
-    if(IS_ABSOLUTE "${_source}")
-      file(RELATIVE_PATH _source_rel "${CMAKE_CURRENT_SOURCE_DIR}" "${_source}")
-    else()
-      set(_source_rel "${_source}")
-    endif()
-    get_filename_component(_source_path "${_source_rel}" PATH)
-    string(REPLACE "/" "\\" _source_path_msvc "${_source_path}")
-    source_group("${_source_path_msvc}" FILES "${_source}")
-    set(SOURCE_FILES "${_source}")
-  endforeach()
-endfunction(assign_source_group)
-
-assign_source_group(${SparkLib_Sources} ${Config_Protos} ${NonConfig_Protos} ${Service_Protos})

compose.production.yaml

@@ -1,51 +0,0 @@
-services:
-  sparkd:
-    build:
-      dockerfile: Dockerfile
-      context: ./sparkd
-    command: [ "./sparkd", "run" ]
-    ports: [ "8000:8000", "8001:8001", "8002:8002" ]
-
-  pubsub:
-    build:
-      dockerfile: pubsub.Dockerfile
-      context: ./DataStream
-    command: [ "./PubSubServer" ]
-    environment:
-      - DOTNET_ENVIRONMENT=Production
-    ports:
-      - 22019:22019
-
-  md-publisher:
-    build:
-      dockerfile: md-publisher.Dockerfile
-      context: ./DataStream
-    depends_on:
-      - pubsub
-      - sparkd
-    command:
-      [
-        "dockerize",
-        "-wait",
-        "tcp://pubsub:22019",
-        "-wait",
-        "tcp://sparkd:8000",
-        "-timeout",
-        "10s",
-        "./DataStreamPublisher"
-      ]
-    environment:
-      - DOTNET_ENVIRONMENT=MarketData
-
-  webapp:
-    ports:
-      - 8080:8080
-      - 8443:8443
-    build:
-      dockerfile: webapp.Dockerfile
-      context: ./DataStream
-    command: [ "./Bts.DataStreamWebApp.Server" ]
-    environment:
-      - ASPNETCORE_HTTP_PORT=8080
-      - ASPNETCORE_HTTPS_PORT=8443
-      - ASPNETCORE_ENVIRONMENT=Staging

compose.yaml

@@ -1,69 +1,100 @@
 services:
   sparkd:
+    volumes:
+      - ${SPARKD__DATA_DIR:-sparkd_data}:/home/bts/sparkdata/data
     build:
       dockerfile: src/app_sparkd/Dockerfile
       args:
         - IMAGE=${SPARKD__IMAGE}
         - SOURCE_DIR=${SPARKD__SOURCE_DIR}
-        - CONF_FILE=${SPARKD__CONF_FILE}
-        - MD_RECORDING_DIR=${SPARKD__MD_RECORDING_DIR}
-    command: [ "./sparkd", "run" ]
+        - CONFIG_DIR=${SPARKD__CONFIG_DIR}
+        - PYTHON_REQUIREMENTS=${SPARKD__PYTHON_REQUIREMENTS}
+    shm_size: "${SPARKD__SHM_SIZE:-512mb}"
+    command: >
+      sh -c "
+        echo bts | sudo -S chown -R bts:bts /home/bts/sparkdata/data
+        if [ -e "/home/bts/sparkdata/data/logging.sparkd/log" ]; then
+          mv "/home/bts/sparkdata/data/logging.sparkd/log" "/home/bts/sparkdata/data/logging.sparkd/log_$(date '+%Y%m%d-%H%M')"
+        fi
+        if [ -e "/home/bts/sparkdata/conf/.env.local" ]; then
+          echo 'Sourcing /home/bts/sparkdata/conf/.env.local'
+          set -a
+          . /home/bts/sparkdata/conf/.env.local
+          set +a
+        fi
+        if [ -d /home/bts/sparkdata/conf/plugins ]; then
+          echo 'Copying /home/bts/sparkdata/conf/plugins to runtime dir (/home/bts/bin)'
+          cp -r /home/bts/sparkdata/conf/plugins /home/bts/bin/
+        fi
+        ./sparkd run"
+    env_file:
+      - .env
+      - ${SPARKD__CONFIG_DIR}/.env
     ports:
-      - "8000:8000"
-      - "8001:8001"
-      - "8002:8002"
+      - "${SPARKD__CONFIG_PORT}:${SPARKD__CONFIG_PORT}"
+      - "${SPARKD__QUICKBUS_PORT}:${SPARKD__QUICKBUS_PORT}"
+      - "${SPARKD__SIM_FIX_PORT}:${SPARKD__SIM_FIX_PORT}"
+    deploy:
+      resources:
+        limits:
+          memory: "${SPARKD__MEMORY:-4gb}"
 
-  pubsub:
-    environment:
-      - DOTNET_ENVIRONMENT=${PUBSUB__DOTNET_ENVIRONMENT}
+  sparkd-symbols:
+    volumes:
+      - ${SPARKD__DATA_DIR:-sparkd_data}:/home/bts/sparkdata/data
+      - ${SPARKD__RUNTIME_CONF_DIR:-sparkd_conf}:/home/bts/sparkdata/conf
     build:
-      dockerfile: src/DataStreamSuite/PubSubServer/Dockerfile
+      dockerfile: src/app_sparkd/Dockerfile
       args:
-        - CS_SOURCE_DIR=${PUBSUB__CS_SOURCE_DIR}
-        - CPP_SOURCE_DIR=${PUBSUB__CPP_SOURCE_DIR}
-        - CONF_FILE=${PUBSUB__CONF_FILE}
+        - IMAGE=${SPARKD__IMAGE}
+        - SOURCE_DIR=${SPARKD__SOURCE_DIR}
+        - CONFIG_DIR=${SPARKD__CONFIG_DIR}
+        - PYTHON_REQUIREMENTS=${SPARKD__PYTHON_REQUIREMENTS}
+    shm_size: "${SPARKD__SHM_SIZE:-512mb}"
+    command: >
+      sh -c "
+        echo bts | sudo -S chown -R bts:bts /home/bts/sparkdata
+        if [ -e "/home/bts/sparkdata/data/logging.sparkd/log" ]; then
+          mv "/home/bts/sparkdata/data/logging.sparkd/log" "/home/bts/sparkdata/data/logging.sparkd/log_$(date '+%Y%m%d-%H%M')"
+        fi
+        ls -alh /home/bts/sparkdata/conf/
+        cat /home/bts/sparkdata/conf/.env.local
+        if [ -e "/home/bts/sparkdata/conf/.env.local" ]; then
+          echo 'Sourcing /home/bts/sparkdata/conf/.env.local'
+          set -a
+          . /home/bts/sparkdata/conf/.env.local
+          set +a
+        fi
+        if [ -d /home/bts/sparkdata/conf/plugins ]; then
+          echo 'Copying /home/bts/sparkdata/conf/plugins to runtime dir (/home/bts/bin)'
+          cp -r /home/bts/sparkdata/conf/plugins /home/bts/bin/
+        fi
+        ./sparkd run"
+    env_file:
+      - .env
+      - ${SPARKD__CONFIG_DIR}/.env
     ports:
-      - "22019:22019"
-    command: [ "./PubSubServer" ]
+      - "${SPARKD__CONFIG_PORT}:${SPARKD__CONFIG_PORT}"
+      - "${SPARKD__QUICKBUS_PORT}:${SPARKD__QUICKBUS_PORT}"
+      - "${SPARKD__SIM_FIX_PORT}:${SPARKD__SIM_FIX_PORT}"
+    deploy:
+      restart_policy:
+        condition: any
+        delay: 3s
+      resources:
+        limits:
+          memory: "${SPARKD__MEMORY:-4gb}"
 
-  md-publisher:
-    build:
-      dockerfile: src/DataStreamSuite/DataStreamPublisher/Dockerfile
-      args:
-        - SOURCE_DIR=${MD_PUBLISHER__SOURCE_DIR}
-        - CONF_FILE=${MD_PUBLISHER__CONF_FILE}
-    depends_on:
-      - pubsub
-      - sparkd
-    command: [ "dockerize", "-wait", "tcp://pubsub:22019", "-wait", "tcp://sparkd:8000", "-timeout", "10s", "./DataStreamPublisher" ]
-    environment:
-      - DOTNET_ENVIRONMENT=${MD_PUBLISHER__DOTNET_ENVIRONMENT}
-
-  webapp:
-    ports:
-      - "8080:8080"
-      - "8443:8443"
-    build:
-      dockerfile: src/DataStreamSuite/Bts.DataStreamWebApp.Server/Dockerfile
-      args:
-        - SOURCE_DIR=${WEBAPP__SOURCE_DIR}
-        - WEB_ROOT=${WEBAPP__WEB_ROOT}
-        - CONF_FILE=${WEBAPP__CONF_FILE}
-    command: [ "./Bts.DataStreamWebApp.Server" ]
-    environment:
-      - ASPNETCORE_ENVIRONMENT=${WEBAPP__ASPNETCORE_ENVIRONMENT}
-
-  api-server:
-    depends_on:
-      - sparkd
-    ports:
-      - 8008:8008
-      - 9443:9443
-    build:
-      dockerfile: src/client/spark_api_server/Dockerfile
-    command: [ "dockerize", "-wait", "tcp://sparkd:8000", "-timeout", "10s", "./spark_api_server", "run" ]
-    environment:
-      - ASPNETCORE_ENVIRONMENT=${API_SERVER__ASPNETCORE_ENVIRONMENT}
+  sparkd-tail:
+    image: bluetradingsystems/linux_dotnet7
+    volumes:
+      - type: volume
+        read_only: true
+        source: sparkd_data
+        target: /home/bts/sparkdata/data/logging.sparkd
+        volume:
+          subpath: logging.sparkd
+    command: ["tail", "-f", "/home/bts/sparkdata/data/logging.sparkd/log"]
 
   sparkd-ping-test:
     depends_on:
@@ -77,3 +108,56 @@ services:
         USER "root"
         RUN apt-get update && apt-get -y install iputils-ping
     command: [ "ping", "sparkd" ]
+
+  pubsub:
+    environment:
+      - DOTNET_ENVIRONMENT=Staging
+    build:
+      dockerfile: src/DataStreamSuite/PubSubServer/Dockerfile
+      args:
+        - SOURCE_DIR=build/Release/bin/net8.0/linux-x64/publish
+        - CONF_FILE=src/DataStreamSuite/PubSubServer/spark.conf
+    ports:
+      - 22019:22019
+    command: [ "./PubSubServer" ]
+
+  md-publisher:
+    build:
+      dockerfile: src/DataStreamSuite/DataStreamPublisher/Dockerfile
+      args:
+        - SOURCE_DIR=build/Release/bin/net8.0/linux-x64/publish
+        - CONF_FILE=src/DataStreamSuite/DataStreamPublisher/spark.conf
+    depends_on:
+      - pubsub
+      - sparkd
+    command: [ "dockerize", "-wait", "tcp://pubsub:22019", "-wait", "tcp://sparkd:8000", "-timeout", "10s", "./DataStreamPublisher" ]
+    environment:
+      - DOTNET_ENVIRONMENT=MarketData
+
+  api-server:
+    depends_on:
+      - sparkd
+    ports:
+      - 8008:8008
+      - 9443:9443
+    build:
+      dockerfile: src/client/spark_api_server/Dockerfile
+    command: [ "dockerize", "-wait", "tcp://sparkd:8000", "-timeout", "10s", "./spark_api_server", "run" ]
+
+  webapp:
+    ports:
+      - 8080:8080
+      - 8443:8443
+    build:
+      dockerfile: src/DataStreamSuite/Bts.DataStreamWebApp.Server/Dockerfile
+      args:
+        - SOURCE_DIR=build/Release/bin/net8.0/linux-x64/publish
+        - WEB_ROOT=build/Bts.DataStreamWebApp.Client/production/dist
+        - CONF_FILE=src/DataStreamSuite/Bts.DataStreamWebApp.Server/spark.conf
+    command: [ "./Bts.DataStreamWebApp.Server" ]
+    environment:
+      - ASPNETCORE_ENVIRONMENT=Staging
+
+volumes:
+  sparkd_data:
+    name: "sparkd_${SPARKD__CONFIG_PORT:-8000}"

doc/api/content/client/getting-started/services.md

@@ -124,8 +124,8 @@ service OrderService {
 ```
 ### C++ Example
 ```cpp
-#include "lib/api/public/ServiceContext.h"
-#include "lib/api/services/OrderService.client.pb.h"
+#include "spark/api/public/ServiceContext.h"
+#include "spark/api/services/OrderService.client.pb.h"
 using namespace bts::api;
 ServiceContext ctx;
 auto orders = ctx.locate<orderman::OrderService>();

doc/api/content/client/services/marketdata.md

@@ -29,9 +29,9 @@ namespace sample {
 ```
 
 ```cpp
-#include "lib/api/public/ServiceContext.h"
-#include "spark/lib/domain_model/Security.h"
-#include "lib/api/services/MarketDataService.client.pb.h"
+#include "spark/api/public/ServiceContext.h"
+#include "spark/domain_model/Security.h"
+#include "spark/api/services/MarketDataService.client.pb.h"
 #include <thread>
 
 using namespace bts::api;

doc/api/content/server/ref/subscriptions.md

@@ -53,7 +53,7 @@ The status callback is called when the market for this security opens, closes, o
 // Subscribe to security trading status
 void IAlgoServices::subscribe_status(SecurityKey);
 // You implement
-virtual void on_status(SecurityKey, domain::protobufs::TradingStatus) override;
+virtual void on_status(SecurityKey, co2::TradingStatus) override;
 ```
 
 # Child Orders

docker/images/bluetradingsystems/linux_dotnet7/Dockerfile

@@ -0,0 +1,38 @@
+FROM ubuntu:22.04
+
+ARG DEBIAN_FRONTEND=noninteractive
+
+RUN apt-get update && apt-get install -y pkg-config
+RUN apt-get update && apt-get install -y locales
+RUN apt-get update && apt-get install -y ca-certificates
+RUN apt-get update && apt-get install -y libssh2-1-dev
+RUN apt-get update && apt-get install -y software-properties-common
+RUN apt-get update && apt-get install -y sudo
+
+# GCC
+ARG gcc_major=13
+RUN add-apt-repository ppa:ubuntu-toolchain-r/test \
+  && apt-get update \
+  && apt-get install -y g++-$gcc_major
+
+RUN apt-get update \
+  && apt-get install -y dotnet7
+
+RUN locale-gen en_US.UTF-8
+
+ARG docker_user="bts"
+ARG docker_uid="1000"
+ARG docker_pass="bts"
+
+RUN \
+  groupadd -g "$docker_uid" "$docker_user" \
+  && useradd \
+  -m \
+  -u "$docker_uid" \
+  -g "$docker_uid" \
+  -G sudo \
+  --password "$(openssl passwd -1 "$docker_pass")" \
+  "$docker_user"
+
+USER bts
+WORKDIR /home/bts

global.json

@@ -1,6 +0,0 @@
-{
-  "sdk": {
-    "version": "6.0.200",
-    "rollForward": "latestMajor"
-  }
-}

requirements.txt

@@ -1,22 +1,42 @@
 astroid==2.7.3
 autopep8==1.5.7
+blinker==1.9.0
+boto3==1.37.9
+botocore==1.37.9
 certifi==2023.5.7
 charset-normalizer==3.1.0
+click==8.1.8
 colorama==0.4.4
+Flask==3.1.0
 idna==3.4
 isort==5.9.3
+itsdangerous==2.2.0
+Jinja2==3.1.6
+jmespath==1.0.1
 lazy-object-proxy==1.6.0
+MarkupSafe==3.0.2
 mccabe==0.6.1
+numpy==2.2.6
 platformdirs==2.3.0
 protobuf==4.21.12
 pybars3==0.9.7
+pybind11==2.13.6
+pybind11-stubgen==2.5.3
+pybit==5.11.0
 pycodestyle==2.7.0
 pycparser==2.20
+pycryptodome==3.23.0
 pylint==2.10.2
 PyMeta3==0.5.1
+python-dateutil==2.9.0.post0
+python-dotenv==1.0.1
 requests==2.31.0
+s3transfer==0.11.4
+six==1.17.0
 termcolor==1.1.0
 toml==0.10.2
 urllib3==2.0.3
+websocket-client==1.8.0
+Werkzeug==3.1.3
 wrapt==1.12.1
 yapf==0.31.0

spark_client.sln

@@ -1,103 +1,164 @@
-Microsoft Visual Studio Solution File, Format Version 12.00
-# Visual Studio Version 17
-VisualStudioVersion = 17.3.32922.545
-MinimumVisualStudioVersion = 15.0.26124.0
-Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "spark_api", "src\client\spark_api\spark_api.csproj", "{DC9A8975-8211-4FC9-A9BA-C8B9D26AD481}"
-EndProject
-Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Spark", "src\client\spark\Spark.csproj", "{9A4D4D07-1AB6-4B2B-B111-A4F34D6C267E}"
-EndProject
-Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Spark.Console", "src\client\Spark.Console\Spark.Console.csproj", "{387566C6-C6F6-412C-A30C-9855BBCB4816}"
-EndProject
-Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "spark_core", "src\client\spark_core\spark_core.csproj", "{6E449EB6-20AC-4FA6-B8D9-10F4BB7E1FEE}"
-EndProject
-Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "spark_sdk", "src\client\spark_sdk\spark_sdk.csproj", "{6963C314-83C4-4667-B0D0-C2F7AE208C29}"
-EndProject
-Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "spark_test", "src\client\spark_test\spark_test.csproj", "{ECD78CC5-138F-4809-A764-2358C2AC2E82}"
-EndProject
-Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "spark_websocket", "src\client\spark_websocket\spark_websocket.csproj", "{3201359D-C071-4089-9EDE-B04D759F1508}"
-EndProject
-Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "spark_api_server", "src\client\spark_api_server\spark_api_server.csproj", "{2B8C5D01-FF78-4BFA-B332-5E430586DAF9}"
-EndProject
-Global
-	GlobalSection(SolutionConfigurationPlatforms) = preSolution
-		Debug|x64 = Debug|x64
-		MinSizeRel|x64 = MinSizeRel|x64
-		Release|x64 = Release|x64
-		RelWithDebInfo|x64 = RelWithDebInfo|x64
-	EndGlobalSection
-	GlobalSection(ProjectConfigurationPlatforms) = postSolution
-		{DC9A8975-8211-4FC9-A9BA-C8B9D26AD481}.Debug|x64.ActiveCfg = Debug|x64
-		{DC9A8975-8211-4FC9-A9BA-C8B9D26AD481}.Debug|x64.Build.0 = Debug|x64
-		{DC9A8975-8211-4FC9-A9BA-C8B9D26AD481}.MinSizeRel|x64.ActiveCfg = Release|x64
-		{DC9A8975-8211-4FC9-A9BA-C8B9D26AD481}.MinSizeRel|x64.Build.0 = Release|x64
-		{DC9A8975-8211-4FC9-A9BA-C8B9D26AD481}.Release|x64.ActiveCfg = Release|x64
-		{DC9A8975-8211-4FC9-A9BA-C8B9D26AD481}.Release|x64.Build.0 = Release|x64
-		{DC9A8975-8211-4FC9-A9BA-C8B9D26AD481}.RelWithDebInfo|x64.ActiveCfg = Release|x64
-		{DC9A8975-8211-4FC9-A9BA-C8B9D26AD481}.RelWithDebInfo|x64.Build.0 = Release|x64
-		{9A4D4D07-1AB6-4B2B-B111-A4F34D6C267E}.Debug|x64.ActiveCfg = Debug|x64
-		{9A4D4D07-1AB6-4B2B-B111-A4F34D6C267E}.Debug|x64.Build.0 = Debug|x64
-		{9A4D4D07-1AB6-4B2B-B111-A4F34D6C267E}.MinSizeRel|x64.ActiveCfg = Release|x64
-		{9A4D4D07-1AB6-4B2B-B111-A4F34D6C267E}.MinSizeRel|x64.Build.0 = Release|x64
-		{9A4D4D07-1AB6-4B2B-B111-A4F34D6C267E}.Release|x64.ActiveCfg = Release|x64
-		{9A4D4D07-1AB6-4B2B-B111-A4F34D6C267E}.Release|x64.Build.0 = Release|x64
-		{9A4D4D07-1AB6-4B2B-B111-A4F34D6C267E}.RelWithDebInfo|x64.ActiveCfg = Release|x64
-		{9A4D4D07-1AB6-4B2B-B111-A4F34D6C267E}.RelWithDebInfo|x64.Build.0 = Release|x64
-		{387566C6-C6F6-412C-A30C-9855BBCB4816}.Debug|x64.ActiveCfg = Debug|x64
-		{387566C6-C6F6-412C-A30C-9855BBCB4816}.Debug|x64.Build.0 = Debug|x64
-		{387566C6-C6F6-412C-A30C-9855BBCB4816}.MinSizeRel|x64.ActiveCfg = Debug|x64
-		{387566C6-C6F6-412C-A30C-9855BBCB4816}.MinSizeRel|x64.Build.0 = Debug|x64
-		{387566C6-C6F6-412C-A30C-9855BBCB4816}.Release|x64.ActiveCfg = Release|x64
-		{387566C6-C6F6-412C-A30C-9855BBCB4816}.Release|x64.Build.0 = Release|x64
-		{387566C6-C6F6-412C-A30C-9855BBCB4816}.RelWithDebInfo|x64.ActiveCfg = Release|x64
-		{387566C6-C6F6-412C-A30C-9855BBCB4816}.RelWithDebInfo|x64.Build.0 = Release|x64
-		{6E449EB6-20AC-4FA6-B8D9-10F4BB7E1FEE}.Debug|x64.ActiveCfg = Debug|x64
-		{6E449EB6-20AC-4FA6-B8D9-10F4BB7E1FEE}.Debug|x64.Build.0 = Debug|x64
-		{6E449EB6-20AC-4FA6-B8D9-10F4BB7E1FEE}.MinSizeRel|x64.ActiveCfg = Release|x64
-		{6E449EB6-20AC-4FA6-B8D9-10F4BB7E1FEE}.MinSizeRel|x64.Build.0 = Release|x64
-		{6E449EB6-20AC-4FA6-B8D9-10F4BB7E1FEE}.Release|x64.ActiveCfg = Release|x64
-		{6E449EB6-20AC-4FA6-B8D9-10F4BB7E1FEE}.Release|x64.Build.0 = Release|x64
-		{6E449EB6-20AC-4FA6-B8D9-10F4BB7E1FEE}.RelWithDebInfo|x64.ActiveCfg = Release|x64
-		{6E449EB6-20AC-4FA6-B8D9-10F4BB7E1FEE}.RelWithDebInfo|x64.Build.0 = Release|x64
-		{6963C314-83C4-4667-B0D0-C2F7AE208C29}.Debug|x64.ActiveCfg = Debug|x64
-		{6963C314-83C4-4667-B0D0-C2F7AE208C29}.Debug|x64.Build.0 = Debug|x64
-		{6963C314-83C4-4667-B0D0-C2F7AE208C29}.MinSizeRel|x64.ActiveCfg = Release|x64
-		{6963C314-83C4-4667-B0D0-C2F7AE208C29}.MinSizeRel|x64.Build.0 = Release|x64
-		{6963C314-83C4-4667-B0D0-C2F7AE208C29}.Release|x64.ActiveCfg = Release|x64
-		{6963C314-83C4-4667-B0D0-C2F7AE208C29}.Release|x64.Build.0 = Release|x64
-		{6963C314-83C4-4667-B0D0-C2F7AE208C29}.RelWithDebInfo|x64.ActiveCfg = Release|x64
-		{6963C314-83C4-4667-B0D0-C2F7AE208C29}.RelWithDebInfo|x64.Build.0 = Release|x64
-		{ECD78CC5-138F-4809-A764-2358C2AC2E82}.Debug|x64.ActiveCfg = Debug|x64
-		{ECD78CC5-138F-4809-A764-2358C2AC2E82}.Debug|x64.Build.0 = Debug|x64
-		{ECD78CC5-138F-4809-A764-2358C2AC2E82}.MinSizeRel|x64.ActiveCfg = Debug|x64
-		{ECD78CC5-138F-4809-A764-2358C2AC2E82}.MinSizeRel|x64.Build.0 = Debug|x64
-		{ECD78CC5-138F-4809-A764-2358C2AC2E82}.Release|x64.ActiveCfg = Release|x64
-		{ECD78CC5-138F-4809-A764-2358C2AC2E82}.Release|x64.Build.0 = Release|x64
-		{ECD78CC5-138F-4809-A764-2358C2AC2E82}.RelWithDebInfo|x64.ActiveCfg = Release|x64
-		{ECD78CC5-138F-4809-A764-2358C2AC2E82}.RelWithDebInfo|x64.Build.0 = Release|x64
-		{3201359D-C071-4089-9EDE-B04D759F1508}.Debug|x64.ActiveCfg = Debug|x64
-		{3201359D-C071-4089-9EDE-B04D759F1508}.Debug|x64.Build.0 = Debug|x64
-		{3201359D-C071-4089-9EDE-B04D759F1508}.MinSizeRel|x64.ActiveCfg = Debug|x64
-		{3201359D-C071-4089-9EDE-B04D759F1508}.MinSizeRel|x64.Build.0 = Debug|x64
-		{3201359D-C071-4089-9EDE-B04D759F1508}.Release|x64.ActiveCfg = Release|x64
-		{3201359D-C071-4089-9EDE-B04D759F1508}.Release|x64.Build.0 = Release|x64
-		{3201359D-C071-4089-9EDE-B04D759F1508}.RelWithDebInfo|x64.ActiveCfg = Release|x64
-		{3201359D-C071-4089-9EDE-B04D759F1508}.RelWithDebInfo|x64.Build.0 = Release|x64
-		{2B8C5D01-FF78-4BFA-B332-5E430586DAF9}.Debug|x64.ActiveCfg = Debug|x64
-		{2B8C5D01-FF78-4BFA-B332-5E430586DAF9}.Debug|x64.Build.0 = Debug|x64
-		{2B8C5D01-FF78-4BFA-B332-5E430586DAF9}.MinSizeRel|x64.ActiveCfg = Debug|x64
-		{2B8C5D01-FF78-4BFA-B332-5E430586DAF9}.MinSizeRel|x64.Build.0 = Debug|x64
-		{2B8C5D01-FF78-4BFA-B332-5E430586DAF9}.Release|x64.ActiveCfg = Release|x64
-		{2B8C5D01-FF78-4BFA-B332-5E430586DAF9}.Release|x64.Build.0 = Release|x64
-		{2B8C5D01-FF78-4BFA-B332-5E430586DAF9}.RelWithDebInfo|x64.ActiveCfg = Release|x64
-		{2B8C5D01-FF78-4BFA-B332-5E430586DAF9}.RelWithDebInfo|x64.Build.0 = Release|x64
-	EndGlobalSection
-	GlobalSection(SolutionProperties) = preSolution
-		HideSolutionNode = FALSE
-	EndGlobalSection
-	GlobalSection(ExtensibilityGlobals) = postSolution
-		SolutionGuid = {422D1620-0166-4C26-9A81-CA8A2EB1AB58}
-	EndGlobalSection
-	GlobalSection(Performance) = preSolution
-		HasPerformanceSessions = true
-	EndGlobalSection
-EndGlobal
+Microsoft Visual Studio Solution File, Format Version 12.00
+# Visual Studio Version 17
+VisualStudioVersion = 17.3.32922.545
+MinimumVisualStudioVersion = 15.0.26124.0
+Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "spark_api", "src\client\spark_api\spark_api.csproj", "{DC9A8975-8211-4FC9-A9BA-C8B9D26AD481}"
+EndProject
+Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Spark", "src\client\spark\Spark.csproj", "{9A4D4D07-1AB6-4B2B-B111-A4F34D6C267E}"
+EndProject
+Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Spark.Console", "src\client\Spark.Console\Spark.Console.csproj", "{387566C6-C6F6-412C-A30C-9855BBCB4816}"
+EndProject
+Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "spark_core", "src\client\spark_core\spark_core.csproj", "{6E449EB6-20AC-4FA6-B8D9-10F4BB7E1FEE}"
+EndProject
+Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "spark_sdk", "src\client\spark_sdk\spark_sdk.csproj", "{6963C314-83C4-4667-B0D0-C2F7AE208C29}"
+EndProject
+Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "spark_test", "src\client\spark_test\spark_test.csproj", "{ECD78CC5-138F-4809-A764-2358C2AC2E82}"
+EndProject
+Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "spark_websocket", "src\client\spark_websocket\spark_websocket.csproj", "{3201359D-C071-4089-9EDE-B04D759F1508}"
+EndProject
+Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "spark_api_server", "src\client\spark_api_server\spark_api_server.csproj", "{2B8C5D01-FF78-4BFA-B332-5E430586DAF9}"
+EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "DataStreamCore", "src\DataStreamSuite\DataStreamCore\DataStreamCore.csproj", "{4E4F491F-C688-43C4-88DA-AC92BC5FA1B8}"
+EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "DataStream", "src\DataStreamSuite\DataStream\DataStream.csproj", "{2AC54ADC-3246-427E-9CA9-FEA2DBCB9432}"
+EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "SharedStreams", "src\DataStreamSuite\SharedStreams\SharedStreams.csproj", "{4E96E5CC-7791-40B3-A8A1-C34FBB736B3B}"
+EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "PubSub", "src\DataStreamSuite\PubSub\PubSub.csproj", "{51C40BA9-F226-4A35-939D-1ED6F5FF7299}"
+EndProject
+Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "src", "src", "{4564E677-343A-46CB-AF02-AFC4E38BF94B}"
+EndProject
+Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "client", "client", "{56A8A70A-DA03-4CED-A144-EC10F1379FD8}"
+EndProject
+Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "DataStreamSuite", "DataStreamSuite", "{915EF238-CA0D-4E73-AF50-88CB3F29C7AD}"
+EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Bts.Extensions.Hosting", "src\DataStreamSuite\Bts.Extensions.Hosting\Bts.Extensions.Hosting.csproj", "{4E0E51AA-7B59-4808-B7CC-9EAEDF397460}"
+EndProject
+Global
+	GlobalSection(SolutionConfigurationPlatforms) = preSolution
+		Debug|x64 = Debug|x64
+		MinSizeRel|x64 = MinSizeRel|x64
+		Release|x64 = Release|x64
+		RelWithDebInfo|x64 = RelWithDebInfo|x64
+	EndGlobalSection
+	GlobalSection(ProjectConfigurationPlatforms) = postSolution
+		{DC9A8975-8211-4FC9-A9BA-C8B9D26AD481}.Debug|x64.ActiveCfg = Debug|x64
+		{DC9A8975-8211-4FC9-A9BA-C8B9D26AD481}.Debug|x64.Build.0 = Debug|x64
+		{DC9A8975-8211-4FC9-A9BA-C8B9D26AD481}.MinSizeRel|x64.ActiveCfg = Release|x64
+		{DC9A8975-8211-4FC9-A9BA-C8B9D26AD481}.MinSizeRel|x64.Build.0 = Release|x64
+		{DC9A8975-8211-4FC9-A9BA-C8B9D26AD481}.Release|x64.ActiveCfg = Release|x64
+		{DC9A8975-8211-4FC9-A9BA-C8B9D26AD481}.Release|x64.Build.0 = Release|x64
+		{DC9A8975-8211-4FC9-A9BA-C8B9D26AD481}.RelWithDebInfo|x64.ActiveCfg = Release|x64
+		{DC9A8975-8211-4FC9-A9BA-C8B9D26AD481}.RelWithDebInfo|x64.Build.0 = Release|x64
+		{9A4D4D07-1AB6-4B2B-B111-A4F34D6C267E}.Debug|x64.ActiveCfg = Debug|x64
+		{9A4D4D07-1AB6-4B2B-B111-A4F34D6C267E}.Debug|x64.Build.0 = Debug|x64
+		{9A4D4D07-1AB6-4B2B-B111-A4F34D6C267E}.MinSizeRel|x64.ActiveCfg = Release|x64
+		{9A4D4D07-1AB6-4B2B-B111-A4F34D6C267E}.MinSizeRel|x64.Build.0 = Release|x64
+		{9A4D4D07-1AB6-4B2B-B111-A4F34D6C267E}.Release|x64.ActiveCfg = Release|x64
+		{9A4D4D07-1AB6-4B2B-B111-A4F34D6C267E}.Release|x64.Build.0 = Release|x64
+		{9A4D4D07-1AB6-4B2B-B111-A4F34D6C267E}.RelWithDebInfo|x64.ActiveCfg = Release|x64
+		{9A4D4D07-1AB6-4B2B-B111-A4F34D6C267E}.RelWithDebInfo|x64.Build.0 = Release|x64
+		{387566C6-C6F6-412C-A30C-9855BBCB4816}.Debug|x64.ActiveCfg = Debug|x64
+		{387566C6-C6F6-412C-A30C-9855BBCB4816}.Debug|x64.Build.0 = Debug|x64
+		{387566C6-C6F6-412C-A30C-9855BBCB4816}.MinSizeRel|x64.ActiveCfg = Debug|x64
+		{387566C6-C6F6-412C-A30C-9855BBCB4816}.MinSizeRel|x64.Build.0 = Debug|x64
+		{387566C6-C6F6-412C-A30C-9855BBCB4816}.Release|x64.ActiveCfg = Release|x64
+		{387566C6-C6F6-412C-A30C-9855BBCB4816}.Release|x64.Build.0 = Release|x64
+		{387566C6-C6F6-412C-A30C-9855BBCB4816}.RelWithDebInfo|x64.ActiveCfg = Release|x64
+		{387566C6-C6F6-412C-A30C-9855BBCB4816}.RelWithDebInfo|x64.Build.0 = Release|x64
+		{6E449EB6-20AC-4FA6-B8D9-10F4BB7E1FEE}.Debug|x64.ActiveCfg = Debug|x64
+		{6E449EB6-20AC-4FA6-B8D9-10F4BB7E1FEE}.Debug|x64.Build.0 = Debug|x64
+		{6E449EB6-20AC-4FA6-B8D9-10F4BB7E1FEE}.MinSizeRel|x64.ActiveCfg = Release|x64
+		{6E449EB6-20AC-4FA6-B8D9-10F4BB7E1FEE}.MinSizeRel|x64.Build.0 = Release|x64
+		{6E449EB6-20AC-4FA6-B8D9-10F4BB7E1FEE}.Release|x64.ActiveCfg = Release|x64
+		{6E449EB6-20AC-4FA6-B8D9-10F4BB7E1FEE}.Release|x64.Build.0 = Release|x64
+		{6E449EB6-20AC-4FA6-B8D9-10F4BB7E1FEE}.RelWithDebInfo|x64.ActiveCfg = Release|x64
+		{6E449EB6-20AC-4FA6-B8D9-10F4BB7E1FEE}.RelWithDebInfo|x64.Build.0 = Release|x64
+		{6963C314-83C4-4667-B0D0-C2F7AE208C29}.Debug|x64.ActiveCfg = Debug|x64
+		{6963C314-83C4-4667-B0D0-C2F7AE208C29}.Debug|x64.Build.0 = Debug|x64
+		{6963C314-83C4-4667-B0D0-C2F7AE208C29}.MinSizeRel|x64.ActiveCfg = Release|x64
+		{6963C314-83C4-4667-B0D0-C2F7AE208C29}.MinSizeRel|x64.Build.0 = Release|x64
+		{6963C314-83C4-4667-B0D0-C2F7AE208C29}.Release|x64.ActiveCfg = Release|x64
+		{6963C314-83C4-4667-B0D0-C2F7AE208C29}.Release|x64.Build.0 = Release|x64
+		{6963C314-83C4-4667-B0D0-C2F7AE208C29}.RelWithDebInfo|x64.ActiveCfg = Release|x64
+		{6963C314-83C4-4667-B0D0-C2F7AE208C29}.RelWithDebInfo|x64.Build.0 = Release|x64
+		{ECD78CC5-138F-4809-A764-2358C2AC2E82}.Debug|x64.ActiveCfg = Debug|x64
+		{ECD78CC5-138F-4809-A764-2358C2AC2E82}.Debug|x64.Build.0 = Debug|x64
+		{ECD78CC5-138F-4809-A764-2358C2AC2E82}.MinSizeRel|x64.ActiveCfg = Debug|x64
+		{ECD78CC5-138F-4809-A764-2358C2AC2E82}.MinSizeRel|x64.Build.0 = Debug|x64
+		{ECD78CC5-138F-4809-A764-2358C2AC2E82}.Release|x64.ActiveCfg = Release|x64
+		{ECD78CC5-138F-4809-A764-2358C2AC2E82}.Release|x64.Build.0 = Release|x64
+		{ECD78CC5-138F-4809-A764-2358C2AC2E82}.RelWithDebInfo|x64.ActiveCfg = Release|x64
+		{ECD78CC5-138F-4809-A764-2358C2AC2E82}.RelWithDebInfo|x64.Build.0 = Release|x64
+		{3201359D-C071-4089-9EDE-B04D759F1508}.Debug|x64.ActiveCfg = Debug|x64
+		{3201359D-C071-4089-9EDE-B04D759F1508}.Debug|x64.Build.0 = Debug|x64
+		{3201359D-C071-4089-9EDE-B04D759F1508}.MinSizeRel|x64.ActiveCfg = Debug|x64
+		{3201359D-C071-4089-9EDE-B04D759F1508}.MinSizeRel|x64.Build.0 = Debug|x64
+		{3201359D-C071-4089-9EDE-B04D759F1508}.Release|x64.ActiveCfg = Release|x64
+		{3201359D-C071-4089-9EDE-B04D759F1508}.Release|x64.Build.0 = Release|x64
+		{3201359D-C071-4089-9EDE-B04D759F1508}.RelWithDebInfo|x64.ActiveCfg = Release|x64
+		{3201359D-C071-4089-9EDE-B04D759F1508}.RelWithDebInfo|x64.Build.0 = Release|x64
+		{2B8C5D01-FF78-4BFA-B332-5E430586DAF9}.Debug|x64.ActiveCfg = Debug|x64
+		{2B8C5D01-FF78-4BFA-B332-5E430586DAF9}.Debug|x64.Build.0 = Debug|x64
+		{2B8C5D01-FF78-4BFA-B332-5E430586DAF9}.MinSizeRel|x64.ActiveCfg = Debug|x64
+		{2B8C5D01-FF78-4BFA-B332-5E430586DAF9}.MinSizeRel|x64.Build.0 = Debug|x64
+		{2B8C5D01-FF78-4BFA-B332-5E430586DAF9}.Release|x64.ActiveCfg = Release|x64
+		{2B8C5D01-FF78-4BFA-B332-5E430586DAF9}.Release|x64.Build.0 = Release|x64
+		{2B8C5D01-FF78-4BFA-B332-5E430586DAF9}.RelWithDebInfo|x64.ActiveCfg = Release|x64
+		{2B8C5D01-FF78-4BFA-B332-5E430586DAF9}.RelWithDebInfo|x64.Build.0 = Release|x64
+		{4E4F491F-C688-43C4-88DA-AC92BC5FA1B8}.Debug|x64.ActiveCfg = Debug|Any CPU
+		{4E4F491F-C688-43C4-88DA-AC92BC5FA1B8}.Debug|x64.Build.0 = Debug|Any CPU
+		{4E4F491F-C688-43C4-88DA-AC92BC5FA1B8}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
+		{4E4F491F-C688-43C4-88DA-AC92BC5FA1B8}.MinSizeRel|x64.Build.0 = Debug|Any CPU
+		{4E4F491F-C688-43C4-88DA-AC92BC5FA1B8}.Release|x64.ActiveCfg = Release|Any CPU
+		{4E4F491F-C688-43C4-88DA-AC92BC5FA1B8}.Release|x64.Build.0 = Release|Any CPU
+		{4E4F491F-C688-43C4-88DA-AC92BC5FA1B8}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
+		{4E4F491F-C688-43C4-88DA-AC92BC5FA1B8}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
+		{2AC54ADC-3246-427E-9CA9-FEA2DBCB9432}.Debug|x64.ActiveCfg = Debug|Any CPU
+		{2AC54ADC-3246-427E-9CA9-FEA2DBCB9432}.Debug|x64.Build.0 = Debug|Any CPU
+		{2AC54ADC-3246-427E-9CA9-FEA2DBCB9432}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
+		{2AC54ADC-3246-427E-9CA9-FEA2DBCB9432}.MinSizeRel|x64.Build.0 = Debug|Any CPU
+		{2AC54ADC-3246-427E-9CA9-FEA2DBCB9432}.Release|x64.ActiveCfg = Release|Any CPU
+		{2AC54ADC-3246-427E-9CA9-FEA2DBCB9432}.Release|x64.Build.0 = Release|Any CPU
+		{2AC54ADC-3246-427E-9CA9-FEA2DBCB9432}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
+		{2AC54ADC-3246-427E-9CA9-FEA2DBCB9432}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
+		{4E96E5CC-7791-40B3-A8A1-C34FBB736B3B}.Debug|x64.ActiveCfg = Debug|Any CPU
+		{4E96E5CC-7791-40B3-A8A1-C34FBB736B3B}.Debug|x64.Build.0 = Debug|Any CPU
+		{4E96E5CC-7791-40B3-A8A1-C34FBB736B3B}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
+		{4E96E5CC-7791-40B3-A8A1-C34FBB736B3B}.MinSizeRel|x64.Build.0 = Debug|Any CPU
+		{4E96E5CC-7791-40B3-A8A1-C34FBB736B3B}.Release|x64.ActiveCfg = Release|Any CPU
+		{4E96E5CC-7791-40B3-A8A1-C34FBB736B3B}.Release|x64.Build.0 = Release|Any CPU
+		{4E96E5CC-7791-40B3-A8A1-C34FBB736B3B}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
+		{4E96E5CC-7791-40B3-A8A1-C34FBB736B3B}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
+		{51C40BA9-F226-4A35-939D-1ED6F5FF7299}.Debug|x64.ActiveCfg = Debug|Any CPU
+		{51C40BA9-F226-4A35-939D-1ED6F5FF7299}.Debug|x64.Build.0 = Debug|Any CPU
+		{51C40BA9-F226-4A35-939D-1ED6F5FF7299}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
+		{51C40BA9-F226-4A35-939D-1ED6F5FF7299}.MinSizeRel|x64.Build.0 = Debug|Any CPU
+		{51C40BA9-F226-4A35-939D-1ED6F5FF7299}.Release|x64.ActiveCfg = Release|Any CPU
+		{51C40BA9-F226-4A35-939D-1ED6F5FF7299}.Release|x64.Build.0 = Release|Any CPU
+		{51C40BA9-F226-4A35-939D-1ED6F5FF7299}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
+		{51C40BA9-F226-4A35-939D-1ED6F5FF7299}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
+		{4E0E51AA-7B59-4808-B7CC-9EAEDF397460}.Debug|x64.ActiveCfg = Debug|Any CPU
+		{4E0E51AA-7B59-4808-B7CC-9EAEDF397460}.Debug|x64.Build.0 = Debug|Any CPU
+		{4E0E51AA-7B59-4808-B7CC-9EAEDF397460}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
+		{4E0E51AA-7B59-4808-B7CC-9EAEDF397460}.MinSizeRel|x64.Build.0 = Debug|Any CPU
+		{4E0E51AA-7B59-4808-B7CC-9EAEDF397460}.Release|x64.ActiveCfg = Release|Any CPU
+		{4E0E51AA-7B59-4808-B7CC-9EAEDF397460}.Release|x64.Build.0 = Release|Any CPU
+		{4E0E51AA-7B59-4808-B7CC-9EAEDF397460}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
+		{4E0E51AA-7B59-4808-B7CC-9EAEDF397460}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
+	EndGlobalSection
+	GlobalSection(SolutionProperties) = preSolution
+		HideSolutionNode = FALSE
+	EndGlobalSection
+	GlobalSection(ExtensibilityGlobals) = postSolution
+		SolutionGuid = {422D1620-0166-4C26-9A81-CA8A2EB1AB58}
+	EndGlobalSection
+	GlobalSection(Performance) = preSolution
+		HasPerformanceSessions = true
+	EndGlobalSection
+	GlobalSection(NestedProjects) = preSolution
+		{56A8A70A-DA03-4CED-A144-EC10F1379FD8} = {4564E677-343A-46CB-AF02-AFC4E38BF94B}
+		{915EF238-CA0D-4E73-AF50-88CB3F29C7AD} = {4564E677-343A-46CB-AF02-AFC4E38BF94B}
+		{4E0E51AA-7B59-4808-B7CC-9EAEDF397460} = {915EF238-CA0D-4E73-AF50-88CB3F29C7AD}
+	EndGlobalSection
+EndGlobal

src/DataStreamSuite/ArithmeticExpressions/ArithmeticExpressions.csproj

@@ -1,7 +1,7 @@
 <Project Sdk="Microsoft.NET.Sdk">
 
   <PropertyGroup>
-    <TargetFramework>net7.0</TargetFramework>
+    <TargetFramework>net8.0</TargetFramework>
     <ImplicitUsings>enable</ImplicitUsings>
     <Nullable>enable</Nullable>
   </PropertyGroup>

src/DataStreamSuite/BaseUtilities/BaseUtilities.csproj

@@ -1,7 +1,7 @@
 <Project Sdk="Microsoft.NET.Sdk">
 
   <PropertyGroup>
-    <TargetFramework>net7.0</TargetFramework>
+    <TargetFramework>net8.0</TargetFramework>
     <ImplicitUsings>enable</ImplicitUsings>
     <Nullable>enable</Nullable>
   </PropertyGroup>

src/DataStreamSuite/BlueRegistry/BlueRegistry.csproj

@@ -1,7 +1,7 @@
 <Project Sdk="Microsoft.NET.Sdk">
 
   <PropertyGroup>
-    <TargetFramework>net7.0</TargetFramework>
+    <TargetFramework>net8.0</TargetFramework>
     <ImplicitUsings>enable</ImplicitUsings>
     <Nullable>enable</Nullable>
   </PropertyGroup>

src/DataStreamSuite/BooleanExpressions/Arguments/ArgumentComparer.cs

@@ -6,23 +6,80 @@ using Spark.Core.Collections;
 
 static class ArgumentComparer {
    public static int Compare(
-      IArgument value1, IArgument value2, IDictionary<string,object?> env) {
+      IArgument value1, IArgument value2, IDictionary<string, object?> env) {
       if (value1 is null) {
          throw new ArgumentNullException(nameof(value1));
       }
       if (value2 is null) {
          throw new ArgumentNullException(nameof(value2));
       }
-      var cmp1 = value1.Cast<IComparable>();
-      var cmp2 = value2.Cast<IComparable>();
+      var obj1 = value1.GetValue(env);
+      var obj2 = value2.GetValue(env);
+      if (obj1 is null) {
+         if (obj2 is null) {
+            return 0;
+         }
+         return -1;
+      }
+      if (obj2 is null) {
+         return 1;
+      }
+      return compareObjs(obj1, obj2);
+   }
+
+   static int compareObjs(object obj1, object obj2) {
       try {
-         var v1 = cmp1.GetValue(env);
-         var v2 = cmp2.GetValue(env);
-         return v1.GetType().IsNumeric() ?
-            new NumericComparer().Compare(v1, v2) : v1.CompareTo(v2);
+         var typ1 = obj1.GetType();
+         var typ2 = obj2.GetType();
+         bool areSameTypes = typ1 == typ2;
+         if (areSameTypes) {
+            return ((IComparable)obj1).CompareTo(obj2);
+         }
+         if (typ1.IsNumeric()) {
+            return tryCompareNumeric(obj1, obj2);
+         } else if (typ2.IsNumeric()) {
+            return -tryCompareNumeric(obj2, obj1);
+         } else if (typ1.IsEnum) {
+            if (typ2 == typeof(string)) {
+               return -((string)obj2).CompareTo(obj1.ToString());
+            }
+         } else if (typ1 == typeof(string)) {
+            return ((string)obj1).CompareTo(obj2.ToString());
+         } else if (typ1 == typeof(DateTime)) {
+            if (typ2 == typeof(TimeSpan)) {
+               return ((DateTime)obj1).Ticks.CompareTo(((TimeSpan)obj2).Ticks);
+            }
+         } else if (typ1 == typeof(TimeSpan)) {
+            if (typ2 == typeof(DateTime)) {
+               return ((TimeSpan)obj1).Ticks.CompareTo(((DateTime)obj2).Ticks);
+            }
+         }
       } catch (Exception ex) {
          Log.Ui.error($"Comparison failed: {ex}");
          throw;
       }
+      throw new ArgumentException("Failed to compare obj1 and obj2", nameof(obj1));
+   }
+
+   static int tryCompareNumeric(object numericObj1, object obj2) {
+      var typ2 = obj2.GetType();
+      if (typ2.IsNumeric()) {
+         return new NumericComparer().Compare(numericObj1, obj2);
+      } else if (typ2.IsEnum) {
+         var undTyp2 = typ2.GetEnumUnderlyingType();
+         if (undTyp2 == typeof(byte)) {
+            return new NumericComparer().Compare(numericObj1, (byte)obj2);
+         }
+         if (undTyp2 == typeof(short)) {
+            return new NumericComparer().Compare(numericObj1, (short)obj2);
+         }
+         if (undTyp2 == typeof(int)) {
+            return new NumericComparer().Compare(numericObj1, (int)obj2);
+         }
+         if (undTyp2 == typeof(long)) {
+            return new NumericComparer().Compare(numericObj1, (long)obj2);
+         }
+      }
+      throw new ArgumentException("Failed to compare numeric obj1 and obj2", nameof(obj2));
    }
 }

src/DataStreamSuite/BooleanExpressions/BooleanExpressionTokens.lex

@@ -41,7 +41,6 @@ Identifier               {Letter}({Letter}*[0-9]*_*)*
 OpenParenthesis          \(
 CloseParenthesis         \)
 Comma                    ,
-Negate                   '!'
 MapsTo                   ':'
 IsOfType                 '/'
 
@@ -94,10 +93,6 @@ IsOfType                 '/'
    return (int)Token.COMMA;
 }
 
-{Negate} {
-   return (int)Token.NEGATE;
-}
-
 {MapsTo} {
    return (int)Token.MAPS_TO;
 }

src/DataStreamSuite/BooleanExpressions/BooleanExpressions.csproj

@@ -1,14 +1,14 @@
 <Project Sdk="Microsoft.NET.Sdk">
 
   <PropertyGroup>
-    <TargetFramework>net7.0</TargetFramework>
+    <TargetFramework>net8.0</TargetFramework>
     <ImplicitUsings>enable</ImplicitUsings>
     <Nullable>enable</Nullable>
   </PropertyGroup>
 
   <ItemGroup>
     <PackageReference Include="Google.Protobuf" Version="3.24.3" />
-    <PackageReference Include="YaccLexTools" Version="1.2.0" />
+    <PackageReference Include="YaccLexTools" Version="1.2.3" />
   </ItemGroup>
 
   <ItemGroup>

src/DataStreamSuite/BooleanExpressions/BoundBooleanExpression.cs

@@ -2,6 +2,8 @@ namespace Bts.BooleanExpressions;
 
 using Bts.BooleanExpressions.Arguments;
 
+using Spark.Core.Collections;
+
 class BoundBooleanExpression : IBooleanExpression {
    readonly IBooleanExpression expression;
    readonly IDictionary<string, FunctionDefinition> definitions;
@@ -17,10 +19,10 @@ class BoundBooleanExpression : IBooleanExpression {
    }
 
    public bool Evaluate(IDictionary<string, object?> env) {
-      foreach (var kv in env) {
-         identifiers[kv.Key] = kv.Value;
+      if (identifiers.Count > 0) {
+         env = AggregateDictionary.Create(env, identifiers);
       }
-      env = new EvaluationEnvironment(identifiers, definitions);
+      env = new EvaluationEnvironment(env, definitions);
       return expression.Evaluate(env);
    }
 

src/DataStreamSuite/BooleanExpressions/GplexBuffers.cs

@@ -24,8 +24,6 @@ namespace QUT.GplexBuffers
         public BufferException(string message) : base(message) { }
         public BufferException(string message, Exception innerException)
             : base(message, innerException) { }
-        protected BufferException(SerializationInfo info, StreamingContext context)
-            : base(info, context) { }
     }
 
     internal abstract class ScanBuff
@@ -60,12 +58,10 @@ namespace QUT.GplexBuffers
             return new BuildBuffer(source);
         }
 
-#if (!BYTEMODE)
         public static ScanBuff GetBuffer(Stream source, int fallbackCodePage)
         {
             return new BuildBuffer(source, fallbackCodePage);
         }
-#endif // !BYTEMODE
 #endif // !NOFILES
     }
 
@@ -353,17 +349,15 @@ namespace QUT.GplexBuffers
         {
             FileStream fStrm = (stream as FileStream);
             if (fStrm != null) FileName = fStrm.Name;
-            NextBlk = BlockReaderFactory.Raw(stream);
+            NextBlk = BlockReaderFactory.Get(stream);
         }
 
-#if (!BYTEMODE)
         public BuildBuffer(Stream stream, int fallbackCodePage)
         {
             FileStream fStrm = (stream as FileStream);
             if (fStrm != null) FileName = fStrm.Name;
             NextBlk = BlockReaderFactory.Get(stream, fallbackCodePage);
         }
-#endif
 
         /// <summary>
         /// Marks a conservative lower bound for the buffer,
@@ -434,43 +428,19 @@ namespace QUT.GplexBuffers
     //
     internal static class BlockReaderFactory
     {
-        public static BlockReader Raw(Stream stream)
-        {
-            return delegate(char[] block, int index, int number)
-            {
-                byte[] b = new byte[number];
-                int count = stream.Read(b, 0, number);
-                int i = 0;
-                int j = index;
-                for (; i < count; i++, j++)
-                    block[j] = (char)b[i];
-                return count;
-            };
-        }
 
-#if (!BYTEMODE)
-        public static BlockReader Get(Stream stream, int fallbackCodePage)
+        public static BlockReader Get(Stream stream, int fallbackCodePage = -2)
         {
             Encoding encoding;
             int preamble = Preamble(stream);
 
             if (preamble != 0)  // There is a valid BOM here!
                 encoding = Encoding.GetEncoding(preamble);
-            else if (fallbackCodePage == -1) // Fallback is "raw" bytes
-                return Raw(stream);
-            else if (fallbackCodePage != -2) // Anything but "guess"
+            else if (fallbackCodePage >= 0) // Anything but "guess"
                 encoding = Encoding.GetEncoding(fallbackCodePage);
             else // This is the "guess" option
-            {
-                int guess = new Guesser(stream).GuessCodePage();
-                stream.Seek(0, SeekOrigin.Begin);
-                if (guess == -1) // ==> this is a 7-bit file
-                    encoding = Encoding.ASCII;
-                else if (guess == 65001)
-                    encoding = Encoding.UTF8;
-                else             // ==> use the machine default
-                    encoding = Encoding.Default;
-            }
+                encoding = GuessEncoding(stream);
+
             StreamReader reader = new StreamReader(stream, encoding);
             return reader.Read;
         }
@@ -488,6 +458,14 @@ namespace QUT.GplexBuffers
             int b2 = stream.ReadByte();
             if (b0 == 0xef && b1 == 0xbb && b2 == 0xbf)
                 return 65001; // UTF8
+
+            int b3 = stream.ReadByte();
+            // Check for UTF-32 BOM (Byte Order Mark)
+            if (b0 == 0x00 && b1 == 0x00 && b2 == 0xFE && b3 == 0xFF)
+                return 12001; // UTF-32BE
+            if (b0 == 0xFF && b1 == 0xFE && b2 == 0x00 && b3 == 0x00)
+                return 12000; // UTF-32LE
+
             //
             // There is no unicode preamble, so we
             // return denoter for the machine default.
@@ -495,7 +473,150 @@ namespace QUT.GplexBuffers
             stream.Seek(0, SeekOrigin.Begin);
             return 0;
         }
-#endif // !BYTEMODE
+
+        private static Encoding GuessEncoding(Stream stream)
+        {
+            stream.Seek(0, SeekOrigin.Begin);
+            if (IsUtf16(stream, out Encoding encoding))
+            {
+                stream.Seek(0, SeekOrigin.Begin);
+                return encoding;
+            }
+
+            stream.Seek(0, SeekOrigin.Begin);
+            if (IsUtf8(stream))
+            {
+                stream.Seek(0, SeekOrigin.Begin);
+                return Encoding.UTF8;
+            }
+
+            stream.Seek(0, SeekOrigin.Begin);
+            if (IsAscii(stream))
+            {
+                stream.Seek(0, SeekOrigin.Begin);
+                return Encoding.ASCII;
+            }
+
+#if !NET462_OR_GREATER
+            stream.Seek(0, SeekOrigin.Begin);
+            if (IsLatin1(stream))
+            {
+                stream.Seek(0, SeekOrigin.Begin);
+                return Encoding.Latin1;
+            }
+#endif
+            stream.Seek(0, SeekOrigin.Begin);
+            return Encoding.Default;
+        }
+
+        private static bool IsUtf8(Stream stream)
+        {
+            int b;
+            while ((b = stream.ReadByte()) != -1)
+            {
+                if ((b & 0x80) == 0) // 0xxxxxxx
+                {
+                    continue;
+                }
+                else if ((b & 0xE0) == 0xC0) // 110xxxxx 10xxxxxx
+                {
+                    if (!IsValidUtf8ContinuationByte(stream)) return false;
+                }
+                else if ((b & 0xF0) == 0xE0) // 1110xxxx 10xxxxxx 10xxxxxx
+                {
+                    if (!IsValidUtf8ContinuationByte(stream) || !IsValidUtf8ContinuationByte(stream)) return false;
+                }
+                else if ((b & 0xF8) == 0xF0) // 11110xxx 10xxxxxx 10xxxxxx 10xxxxxx
+                {
+                    if (!IsValidUtf8ContinuationByte(stream) || !IsValidUtf8ContinuationByte(stream) || !IsValidUtf8ContinuationByte(stream)) return false;
+                }
+                else
+                {
+                    return false;
+                }
+            }
+            return true;
+        }
+
+        private static bool IsValidUtf8ContinuationByte(Stream stream)
+        {
+            int b = stream.ReadByte();
+            return b != -1 && (b & 0xC0) == 0x80; // 10xxxxxx
+        }
+
+        private static bool IsUtf16(Stream stream, out Encoding encoding)
+        {
+            encoding = Encoding.Default;
+
+            if (stream.Length >= 2)
+            {
+                // Read the first two bytes
+                int b0 = stream.ReadByte();
+                int b1 = stream.ReadByte();
+
+                // Check for UTF-16 BOM (Byte Order Mark)
+                if (b0 == 0xFF && b1 == 0xFE)
+                {
+                    encoding = Encoding.Unicode;
+                    return true; // UTF-16LE
+                }
+                if (b0 == 0xFE && b1 == 0xFF)
+                {
+                    encoding = Encoding.BigEndianUnicode;
+                    return true; // UTF-16BE
+                }
+
+                // Reset the stream position
+                stream.Seek(0, SeekOrigin.Begin);
+
+                // Check for common UTF-16 patterns
+                byte[] buffer = new byte[2];
+                while (stream.Read(buffer, 0, 2) == 2)
+                {
+                    if (buffer[0] == 0x00 && buffer[1] != 0x00)
+                    {
+                        encoding = Encoding.BigEndianUnicode;
+                        return true; // Likely UTF-16BE
+                    }
+                    if (buffer[0] != 0x00 && buffer[1] == 0x00)
+                    {
+                        encoding = Encoding.Unicode;
+                        return true; // Likely UTF-16LE
+                    }
+                }
+            }
+
+            return false;
+        }
+
+        private static bool IsAscii(Stream stream)
+        {
+            int b;
+            while ((b = stream.ReadByte()) != -1)
+            {
+                // Latin1 includes byte values from 0x00 to 0xFF
+                if (b > 0x7F)
+                {
+                    return false;
+                }
+            }
+            return true;
+        }
+
+        private static bool IsLatin1(Stream stream)
+        {
+            int b;
+            while ((b = stream.ReadByte()) != -1)
+            {
+                // Latin1 includes byte values from 0x00 to 0xFF
+                if (b > 0xFF)
+                {
+                    return false;
+                }
+            }
+            return true;
+        }
+
     }
 #endif // !NOFILES
     #endregion Buffer classes
@@ -540,180 +661,6 @@ namespace QUT.GplexBuffers
             return 0;
         }
     }
-#region guesser
-#if (!BYTEMODE)
-    // ==============================================================
-    // ============          Encoding Guesser           =============
-    // ==============================================================
-
-    /// <summary>
-    /// This class provides a simple finite state automaton that
-    /// scans the file looking for (1) valid UTF-8 byte patterns,
-    /// (2) bytes >= 0x80 which are not part of a UTF-8 sequence.
-    /// The method then guesses whether it is UTF-8 or maybe some 
-    /// local machine default encoding.  This works well for the
-    /// various Latin encodings.
-    /// </summary>
-    internal class Guesser
-    {
-        ScanBuff buffer;
-
-        public int GuessCodePage() { return Scan(); }
-
-        const int maxAccept = 10;
-        const int initial = 0;
-        const int eofNum = 0;
-        const int goStart = -1;
-        const int INITIAL = 0;
-        const int EndToken = 0;
-
-        #region user code
-        /* 
-         *  Reads the bytes of a file to determine if it is 
-         *  UTF-8 or a single-byte code page file.
-         */
-        public long utfX;
-        public long uppr;
-        #endregion user code
-
-        int state;
-        int currentStart = startState[0];
-        int code;
-
-        #region ScannerTables
-        static int[] startState = new int[] { 11, 0 };
-
-        #region CharacterMap
-        static sbyte[] map = new sbyte[256] {
-/*     '\0' */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 
-/*   '\x10' */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 
-/*   '\x20' */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 
-/*      '0' */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 
-/*      '@' */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 
-/*      'P' */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 
-/*      '`' */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 
-/*      'p' */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 
-/*   '\x80' */ 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 
-/*   '\x90' */ 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 
-/*   '\xA0' */ 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 
-/*   '\xB0' */ 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 
-/*   '\xC0' */ 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 
-/*   '\xD0' */ 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 
-/*   '\xE0' */ 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 
-/*   '\xF0' */ 4, 4, 4, 4, 4, 4, 4, 4, 5, 5, 5, 5, 5, 5, 5, 5 };
-        #endregion
-
-        static sbyte[][] nextState = new sbyte[][] {
-            new sbyte[] {0, 0, 0, 0, 0, 0},
-            new sbyte[] {-1, -1, 10, -1, -1, -1},
-            new sbyte[] {-1, -1, -1, -1, -1, -1},
-            new sbyte[] {-1, -1, 8, -1, -1, -1},
-            new sbyte[] {-1, -1, 5, -1, -1, -1},
-            new sbyte[] {-1, -1, 6, -1, -1, -1},
-            new sbyte[] {-1, -1, 7, -1, -1, -1},
-            null,
-            new sbyte[] {-1, -1, 9, -1, -1, -1},
-            null,
-            null,
-            new sbyte[] {-1, 1, 2, 3, 4, 2}
-        };
-
-
-        [SuppressMessage("Microsoft.Performance", "CA1810:InitializeReferenceTypeStaticFieldsInline")]
-        // Reason for suppression: cannot have self-reference in array initializer.
-        static Guesser()
-        {
-            nextState[7] = nextState[2];
-            nextState[9] = nextState[2];
-            nextState[10] = nextState[2];
-        }
-
-        int NextState()
-        {
-            if (code == ScanBuff.EndOfFile)
-                return eofNum;
-            else
-                return nextState[state][map[code]];
-        }
-        #endregion
-
-        public Guesser(System.IO.Stream file) { SetSource(file); }
-
-        public void SetSource(System.IO.Stream source)
-        {
-            this.buffer = new BuildBuffer(source);
-            code = buffer.Read();
-        }
-
-        int Scan()
-        {
-            for (; ; )
-            {
-                int next;
-                state = currentStart;
-                while ((next = NextState()) == goStart)
-                    code = buffer.Read();
-
-                state = next;
-                code = buffer.Read();
-
-                while ((next = NextState()) > eofNum)
-                {
-                    state = next;
-                    code = buffer.Read();
-                }
-                if (state <= maxAccept)
-                {
-                    #region ActionSwitch
-#pragma warning disable 162
-                    switch (state)
-                    {
-                        case eofNum:
-                            switch (currentStart)
-                            {
-                                case 11:
-                                    if (utfX == 0 && uppr == 0) return -1; /* raw ascii */
-                                    else if (uppr * 10 > utfX) return 0;   /* default code page */
-                                    else return 65001;                     /* UTF-8 encoding */
-                                    break;
-                            }
-                            return EndToken;
-                        case 1: // Recognized '{Upper128}',	Shortest string "\xC0"
-                        case 2: // Recognized '{Upper128}',	Shortest string "\x80"
-                        case 3: // Recognized '{Upper128}',	Shortest string "\xE0"
-                        case 4: // Recognized '{Upper128}',	Shortest string "\xF0"
-                            uppr++;
-                            break;
-                        case 5: // Recognized '{Utf8pfx4}{Utf8cont}',	Shortest string "\xF0\x80"
-                            uppr += 2;
-                            break;
-                        case 6: // Recognized '{Utf8pfx4}{Utf8cont}{2}',	Shortest string "\xF0\x80\x80"
-                            uppr += 3;
-                            break;
-                        case 7: // Recognized '{Utf8pfx4}{Utf8cont}{3}',	Shortest string "\xF0\x80\x80\x80"
-                            utfX += 3;
-                            break;
-                        case 8: // Recognized '{Utf8pfx3}{Utf8cont}',	Shortest string "\xE0\x80"
-                            uppr += 2;
-                            break;
-                        case 9: // Recognized '{Utf8pfx3}{Utf8cont}{2}',	Shortest string "\xE0\x80\x80"
-                            utfX += 2;
-                            break;
-                        case 10: // Recognized '{Utf8pfx2}{Utf8cont}',	Shortest string "\xC0\x80"
-                            utfX++;
-                            break;
-                        default:
-                            break;
-                    }
-#pragma warning restore 162
-                    #endregion
-                }
-            }
-        }
-    } // end class Guesser
-    
-#endif // !BYTEMODE
-#endregion
 #endif // !NOFILES
 
 // End of code copied from embedded resource

src/DataStreamSuite/Bts.ColumnExpressions/Bts.ColumnExpressions.csproj

@@ -1,7 +1,7 @@
 <Project Sdk="Microsoft.NET.Sdk">
 
   <PropertyGroup>
-    <TargetFramework>net7.0</TargetFramework>
+    <TargetFramework>net8.0</TargetFramework>
     <ImplicitUsings>enable</ImplicitUsings>
     <Nullable>enable</Nullable>
   </PropertyGroup>

src/DataStreamSuite/Bts.ColumnSortExpressions/Bts.ColumnSortExpressions.csproj

@@ -1,7 +1,7 @@
 <Project Sdk="Microsoft.NET.Sdk">
 
   <PropertyGroup>
-    <TargetFramework>net7.0</TargetFramework>
+    <TargetFramework>net8.0</TargetFramework>
     <ImplicitUsings>enable</ImplicitUsings>
     <Nullable>enable</Nullable>
   </PropertyGroup>

src/DataStreamSuite/Bts.DataStreamWebApp.Client/.eslintrc.cjs

@@ -0,0 +1,20 @@
+module.exports = {
+  root: true,
+  env: { browser: true, es2020: true },
+  extends: [
+    'eslint:recommended',
+    'plugin:react/recommended',
+    'plugin:react/jsx-runtime',
+    'plugin:react-hooks/recommended',
+  ],
+  ignorePatterns: ['dist', '.eslintrc.cjs'],
+  parserOptions: { ecmaVersion: 'latest', sourceType: 'module' },
+  settings: { react: { version: '18.3.1' } },
+  plugins: ['react-refresh'],
+  rules: {
+    'react-refresh/only-export-components': [
+      'warn',
+      { allowConstantExport: true },
+    ],
+  },
+}

src/DataStreamSuite/Bts.DataStreamWebApp.Client/eslint.config.js

@@ -1,36 +0,0 @@
-import importPlugin from "eslint-plugin-import";
-import reactPkg from "eslint-plugin-react";
-const { configs: reactConfigs } = reactPkg;
-import js from "@eslint/js";
-import defineConfig from "./vite.config.js";
-
-export default [
-  js.configs.recommended,
-  importPlugin.flatConfigs.recommended,
-  reactConfigs.flat.recommended,
-  reactConfigs.flat["jsx-runtime"],
-  {
-    files: ["**/*.{js,jsx}"],
-    languageOptions: {
-      ecmaVersion: "latest",
-      sourceType: "module",
-      globals: {
-        console: "readonly"
-      }
-    },
-    settings: {
-      "import/resolver": {
-        vite: {
-          viteConfig: defineConfig({ mode: "development" })
-        }
-      }
-    },
-    rules: {
-      semi: "error",
-      "prefer-const": "error",
-      "no-unused-vars": "warn",
-      "import/no-dynamic-require": "warn",
-      "import/no-nodejs-modules": "warn",
-    }
-  }
-];

src/DataStreamSuite/Bts.DataStreamWebApp.Client/package.json

@@ -10,37 +10,34 @@
     "preview": "vite preview"
   },
   "dependencies": {
-    "@angular/material": "^18.2.8",
-    "@emotion/react": "^11.13.3",
-    "@emotion/styled": "^11.13.0",
-    "@fontsource/noto-sans": "^5.1.0",
-    "@material/data-table": "^15.0.0-canary.423edc3dc.0",
-    "@mui/icons-material": "^6.1.4",
-    "@mui/material": "^6.1.4",
-    "@mui/x-data-grid": "^7.20.0",
-    "bulma": "^1.0.2",
+    "@angular/material": "^18.0.4",
+    "@emotion/react": "^11.11.4",
+    "@emotion/styled": "^11.11.5",
+    "@fontsource/noto-sans": "^5.0.22",
+    "@material/data-table": "^15.0.0-canary.7f224ddd4.0",
+    "@mui/icons-material": "^5.15.20",
+    "@mui/material": "^5.15.20",
+    "@mui/x-data-grid": "^7.7.1",
+    "bulma": "^1.0.1",
     "classnames": "^2.5.1",
     "react": "^18.3.1",
-    "react-data-grid": "^7.0.0-beta.47",
+    "react-data-grid": "^7.0.0-beta.44",
     "react-data-table-component": "^7.6.2",
     "react-dom": "^18.3.1",
-    "react-json-view-lite": "^1.5.0",
+    "react-json-view-lite": "^1.4.0",
     "react-pro-sidebar": "^1.1.0",
-    "react-use-websocket": "^4.9.0",
-    "rpc-websockets": "^9.0.4"
+    "react-use-websocket": "^4.8.1",
+    "rpc-websockets": "^7.11.2"
   },
   "devDependencies": {
-    "@types/react": "^18.3.11",
-    "@types/react-dom": "^18.3.1",
-    "@vitejs/plugin-react": "^4.3.2",
-    "eslint": "^9.12.0",
-    "@eslint/js": "^9.12.0",
-    "eslint-plugin-import": "^2.31.0",
-    "eslint-plugin-react": "^7.37.1",
-    "eslint-plugin-react-hooks": "^5.0.0",
-    "eslint-plugin-react-refresh": "^0.4.12",
-    "eslint-import-resolver-vite": "^2.0.1",
-    "react-mui-sidebar": "^1.3.8",
-    "vite": "^5.4.9"
+    "@types/react": "^18.3.3",
+    "@types/react-dom": "^18.3.0",
+    "@vitejs/plugin-react": "^4.3.1",
+    "eslint": "^8.57.0",
+    "eslint-plugin-react": "^7.34.2",
+    "eslint-plugin-react-hooks": "^4.6.2",
+    "eslint-plugin-react-refresh": "^0.4.7",
+    "react-mui-sidebar": "^1.3.5",
+    "vite": "^5.2.13"
   }
 }

src/DataStreamSuite/Bts.DataStreamWebApp.Client/src/QueryStatusView.jsx

@@ -12,7 +12,7 @@ const QueryStatusView = ({ queryState, details, error }) => {
 
   let header = queryState;
   let headerContent = null;
-  const detailElements = [];
+  let detailElements = [];
   let severity = queryState == "Subscribed" ? "success" : "info";
   let cursor = details == null ? "default" : "pointer";
   if (error != null) {
@@ -20,14 +20,14 @@ const QueryStatusView = ({ queryState, details, error }) => {
     cursor = "pointer";
     header = `Error Code: ${error.code}`;
     headerContent = error.message;
-    detailElements.push(<Typography key="error-msg" variant="body1">{error.data.error_message}</Typography>);
-    detailElements.push(<Typography key="error-details" variant="subtitle1">Stack Trace:</Typography>);
-    (error.data.stack_trace ?? []).forEach((line, idx) => detailElements.push(
-      <Typography key={`stacktrace-${idx}`} variant="body2" component="pre" sx={{ whiteSpace: 'pre-wrap' }}>
+    detailElements.push(<Typography variant="body1">{error.data.error_message}</Typography>);
+    detailElements.push(<Typography variant="subtitle1">Stack Trace:</Typography>);
+    (error.data.stack_trace ?? []).forEach(line => detailElements.push(
+      <Typography variant="body2" component="pre" sx={{ whiteSpace: 'pre-wrap' }}>
         {line}
       </Typography>));
   } else if (details != null) {
-    detailElements.push(<Typography key="status-msg" variant="body1">{details}</Typography>);
+    detailElements.push(<Typography variant="body1">{details}</Typography>);
   }
   return (
     <Box sx={{ margin: theme.spacing(2, 0, 0, 0) }}>

src/DataStreamSuite/Bts.DataStreamWebApp.Client/vite.config.js

@@ -29,12 +29,13 @@ if (!fs.existsSync(certFilePath) || !fs.existsSync(keyFilePath)) {
   }
 }
 
+const target = env.ASPNETCORE_HTTPS_PORT
+  ? `https://localhost:${env.ASPNETCORE_HTTPS_PORT}`
+  : env.ASPNETCORE_URLS
+    ? env.ASPNETCORE_URLS.split(';')[0]
+    : 'https://localhost:8443';
+
 export default defineConfig(({ $$command, mode, $$isSsrBuild, $$isPreview }) => {
-  const target = env.ASPNETCORE_HTTPS_PORT
-    ? `https://localhost:${env.ASPNETCORE_HTTPS_PORT}`
-    : env.ASPNETCORE_URLS
-      ? env.ASPNETCORE_URLS.split(';')[0]
-      : 'https://localhost:8443';
   return {
     plugins: [plugin()],
     resolve: {
@@ -67,10 +68,6 @@ export default defineConfig(({ $$command, mode, $$isSsrBuild, $$isPreview }) =>
         '^/ws': {
           target,
           secure: false
-        },
-        '^/browse': {
-          target,
-          secure: false
         }
       },
       port: 5173,

src/DataStreamSuite/Bts.DataStreamWebApp.Server/Bts.DataStreamWebApp.Server.csproj

@@ -1,7 +1,7 @@
 <Project Sdk="Microsoft.NET.Sdk.Web">
 
   <PropertyGroup>
-    <TargetFramework>net7.0</TargetFramework>
+    <TargetFramework>net8.0</TargetFramework>
     <Nullable>enable</Nullable>
     <ImplicitUsings>enable</ImplicitUsings>
     <InvariantGlobalization>true</InvariantGlobalization>
@@ -12,10 +12,10 @@
 
   <ItemGroup>
     <PackageReference Include="CommandLineParser" Version="2.9.1" />
-    <PackageReference Include="Microsoft.AspNetCore.SpaProxy" Version="7.0.20" />
-    <PackageReference Include="Microsoft.AspNetCore.SpaServices.Extensions" Version="7.0.20" />
+    <PackageReference Include="Microsoft.AspNetCore.SpaProxy" Version="8.0.18" />
+    <PackageReference Include="Microsoft.AspNetCore.SpaServices.Extensions" Version="8.0.18" />
     <PackageReference Include="Newtonsoft.Json" Version="13.0.3" />
-    <PackageReference Include="Swashbuckle.AspNetCore" Version="6.4.0" />
+    <PackageReference Include="Swashbuckle.AspNetCore" Version="9.0.3" />
   </ItemGroup>
 
   <ItemGroup>

src/DataStreamSuite/Bts.DataStreamWebApp.Server/Controllers/DataStreamSubscriptionController.cs

@@ -1,37 +1,34 @@
-namespace Bts.DataStreamWebApp.Server.Controllers;
-
-using Bts.DataStream;
-
-using Microsoft.AspNetCore.Mvc;
-
-[Route("/ws")]
-public class DataStreamSubscriptionController : ControllerBase {
-   readonly ILogger logger;
-   readonly IDataStreamFactory streamFactory;
-
-   public DataStreamSubscriptionController(ILogger<DataStreamSubscriptionController> logger, IEnumerable<IDataStreamFactory> streamFactories) {
-      this.logger = logger;
-      logger.LogInformation("Creating {ControllerType}", GetType().Name);
-      var streamFactoryList = streamFactories.ToList();
-      streamFactory = streamFactoryList.Count == 1
-         ? streamFactoryList[0]
-         : new CompositeDataStreamFactory(streamFactoryList);
-   }
-
-   [Route("/ws")]
-   public async Task Get(CancellationToken cancellationToken) {
-      if (!HttpContext.WebSockets.IsWebSocketRequest) {
-         logger.LogError("Expected websocket request on path '{}'", HttpContext.Request.Path);
-         return;
-      }
-      logger.LogInformation("Establishing websocket connection...");
-      try {
-         using var webSocket = await HttpContext.WebSockets.AcceptWebSocketAsync();
-         logger.LogInformation("Websocket connection established.");
-         using var server = new DataStreamWebSocketServer(webSocket, streamFactory, logger);
-         await server.Run(cancellationToken);
-      } catch (Exception ex) {
-         logger.LogError("DataStreamSubscriptionController.Run threw exception: {Exception}", ex);
-      }
-   }
-}
+namespace Bts.DataStreamWebApp.Server.Controllers;
+
+using Bts.DataStreamCore;
+
+using Microsoft.AspNetCore.Mvc;
+
+[Route("/ws")]
+public class DataStreamSubscriptionController : ControllerBase {
+   readonly ILogger logger;
+   readonly IDataStreamFactory streamFactory;
+
+   public DataStreamSubscriptionController(ILogger<DataStreamSubscriptionController> logger, IEnumerable<IDataStreamFactory> streamFactories) {
+      this.logger = logger;
+      logger.LogInformation("Creating {ControllerType}", GetType().Name);
+      streamFactory = DataStreamFactory.Create(streamFactories);
+   }
+
+   [Route("/ws")]
+   public async Task Get(CancellationToken cancellationToken) {
+      if (!HttpContext.WebSockets.IsWebSocketRequest) {
+         logger.LogError("Expected websocket request on path '{}'", HttpContext.Request.Path);
+         return;
+      }
+      logger.LogInformation("Establishing websocket connection...");
+      try {
+         using var webSocket = await HttpContext.WebSockets.AcceptWebSocketAsync();
+         logger.LogInformation("Websocket connection established.");
+         using var server = new DataStreamWebSocketServer(webSocket, streamFactory, logger);
+         await server.Run(cancellationToken);
+      } catch (Exception ex) {
+         logger.LogError("DataStreamSubscriptionController.Run threw exception: {Exception}", ex);
+      }
+   }
+}

src/DataStreamSuite/Bts.DataStreamWebApp.Server/DataStreamWebSocketServer.cs

@@ -2,7 +2,7 @@ namespace Bts.DataStreamWebApp.Server.Controllers;
 
 using bts.websocket_core.json_rpc;
 
-using Bts.DataStream;
+using Bts.DataStreamCore;
 using Bts.DataStreamWebApp.Server.Messages;
 
 using System.Net.WebSockets;

src/DataStreamSuite/Bts.DataStreamWebApp.Server/Dockerfile

@@ -1,13 +1,11 @@
 FROM bluetradingsystems/linux_dotnet7
 
-ARG CS_SOURCE_DIR="build/Release/bin/net7.0/linux-x64/publish"
-ARG CPP_SOURCE_DIR="install/data-stream-suite-linux"
+ARG SOURCE_DIR="build/Release/bin/net7.0/linux-x64/publish"
 ARG WEB_ROOT="build/Bts.DataStreamWebApp.Client/production/dist"
 ARG CONF_FILE="src/DataStreamSuite/Bts.DataStreamWebApp.Server/spark.conf"
 
 RUN dotnet dev-certs https --trust
-COPY --chown=bts:bts "${CPP_SOURCE_DIR}/lib" "bin"
-COPY --chown=bts:bts "${CS_SOURCE_DIR}" "bin"
+COPY --chown=bts:bts "${SOURCE_DIR}" "bin"
 COPY --chown=bts:bts "${WEB_ROOT}" "bin/wwwroot"
-COPY --chown=bts:bts "${CONF_FILE}" ".local/share/bts/Bts.DataStreamWebApp.Server/conf/spark.conf"
+COPY --chown=bts:bts "${CONF_FILE}" ".local/share/DataStream/conf/spark.conf"
 WORKDIR "/home/bts/bin"

src/DataStreamSuite/Bts.DataStreamWebApp.Server/Messages/SubscribeRequest.cs

@@ -4,7 +4,7 @@ using BCG;
 
 using bts.websocket_core.json_rpc;
 
-using Bts.DataStream;
+using Bts.DataStreamCore;
 using Bts.DataStreamExpressions;
 
 using System.Diagnostics.CodeAnalysis;
@@ -60,7 +60,7 @@ public class SubscribeRequest {
       }
       Exception? exception = null;
       try {
-         stream = exp.getView(new Dictionary<string, StatementExpression>(0), streamFactory);
+         stream = exp.Evaluate(streamFactory) as ViewStream;
       } catch (Exception ex) {
          exception = ex;
       }

src/DataStreamSuite/Bts.DataStreamWebApp.Server/Program.cs

@@ -79,8 +79,8 @@ static void ConfigureServices(IServiceCollection services, IConfiguration config
       .AddSparkApi(config.GetSection("SparkApi"))
       .AddMasterWorker(config.GetSection("MasterWorker"))
       .AddViewStreamDatabase()
-      .AddPrimitiveDataStreamFactory(config.GetSection("PrimitiveStreams"))
-      .AddOhlcvStreams(config.GetSection("StreamDatabase"))
+      .AddPrimitiveStreams(config.GetSection("PrimitiveStreams"))
+      .AddMarketDataStreams()
       .AddNaturalLanguageStreams(config.GetSection("NaturalLanguageStream"))
       .AddCompositeDataStreamFactory()
       .AddPubSubClient(config.GetSection("PubSub"));

src/DataStreamSuite/Bts.DataStreamWebApp.Server/appsettings.json

@@ -116,7 +116,7 @@
           },
           {
             "natural_language": "Get bbo for 10 year T-Notes in the first minute after the market opened yesterday.",
-            "dsl": "Bbo(symbol: \"ES\", venue: \"CME\", begin: \"2024-10-10T17:00:00-06:00\", end: \"2024-10-10T17:01:00-06:00\")"
+            "dsl": "Bbo(symbol: \"ES\", venue: \"CME\", begin: \"2024-10-10T17:00:00-06:00\", end: \"2024-10-10T17:01:00-06:00\", exclusive_end: true)"
           },
           {
             "natural_language": "Retrieve NVIDIA L1 books from Oct 3",
@@ -147,10 +147,10 @@
           "interval (sample frequency of the data), ",
           "levels (the number of price levels to examine)",
           "begin and end (inclusive beginning and end of the time window to query, respectively), ",
+          "inclusive_end (a bool flag that, when true, makes the end time inclusive). ",
           "Arguments are specified by name, for example: OrderImbalance(symbol: \"ZN\", venue: \"CME\", interval: \"PT1M\", begin: \"2024-10-10\", end: \"2024-10-11\"). ",
           "Depending on the wording of the request, some arguments may be omitted. ",
           "The begin, end, and interval args should be expressed in iso 8601 format. ",
-          "If the prompt doesn't contain an explicit interval, please choose a reasonable value depending on the duration between begin and end.",
           "Interpret dates according to the trading schedule rather than calendar date whenever possible. ",
           "In some prompts, the time window will be expressed in relative terms (e.g., \"last Monday\", or \"yesterday\") ",
           "which must be translated to exact dates. To correctly interpret the dsl in the examples, assume the current time is 2024-10-11T12:00-05:00."

src/DataStreamSuite/Bts.Extensions.Hosting/Bts.Extensions.Hosting.csproj

@@ -1,13 +1,13 @@
 <Project Sdk="Microsoft.NET.Sdk">
 
   <PropertyGroup>
-    <TargetFramework>net7.0</TargetFramework>
+    <TargetFramework>net8.0</TargetFramework>
     <ImplicitUsings>enable</ImplicitUsings>
     <Nullable>enable</Nullable>
   </PropertyGroup>
 
   <ItemGroup>
-    <PackageReference Include="Microsoft.Extensions.Hosting" Version="7.0.1" />
+    <PackageReference Include="Microsoft.Extensions.Hosting" Version="9.0.7" />
   </ItemGroup>
 
 </Project>

src/DataStreamSuite/Bts.Extensions.Hosting/HostBuilderExtensions.cs

@@ -3,15 +3,12 @@
 using Microsoft.Extensions.Configuration;
 using Microsoft.Extensions.Hosting;
 
+using System.Collections.Generic;
 using System.Diagnostics;
 
 public static class HostBuilderExtensions {
    public static IHostBuilder CreateBtsBuilder(string[] args) {
-      // Obtain calling assembly name here, rather than in the lambda where it is actually needed,
-      // because want the name of the assembly that called this function, not the assembly that will eventually invoke
-      // the lambda...that will be the Microsoft.Extensions.Hosting assembly.
-      // Get the name of the assembly that called this function. That will act as the "appsettings"
-      // json file prefix
+      // The calling assembly's name serves as the "appsettings" json file prefix.
       var exeName = GetCallingAssemblyName();
       return Host.CreateDefaultBuilder(args)
          .ConfigureAppConfiguration((context, configBuilder) =>
@@ -56,4 +53,21 @@ public static class HostBuilderExtensions {
          ?.GetName()
          ?.Name
          ?? throw new InvalidOperationException("Failed to determine name of calling assembly (calling method is null)");
+
+   /// <summary>
+   /// Separates command line args from config args, using '--' as the delimiter. Args
+   /// before the delimiter are considered command line args, while those after it are
+   /// config args.
+   /// </summary>
+   public static (string[] Args, string[] ConfigArgs) SplitConfigArgs(string[] args) {
+      int splitIdx = Array.IndexOf(args, "--");
+      if (splitIdx == -1) {
+         return (args, Array.Empty<string>());
+      }
+      var cfgArgs = new string[args.Length - splitIdx - 1];
+      Array.Copy(args, splitIdx + 1, cfgArgs, 0, cfgArgs.Length);
+      var nonCfgArgs = new string[args.Length - cfgArgs.Length - 1];
+      Array.Copy(args, 0, nonCfgArgs, 0, nonCfgArgs.Length);
+      return (nonCfgArgs, cfgArgs);
+   }
 }

src/DataStreamSuite/Bts.Net/Bts.Net.csproj

@@ -1,7 +1,7 @@
 <Project Sdk="Microsoft.NET.Sdk">
 
   <PropertyGroup>
-    <TargetFramework>net7.0</TargetFramework>
+    <TargetFramework>net8.0</TargetFramework>
     <ImplicitUsings>enable</ImplicitUsings>
     <Nullable>enable</Nullable>
   </PropertyGroup>

src/DataStreamSuite/ColumnMapExpressions/ColumnMapExpressions.csproj

@@ -1,7 +1,7 @@
 <Project Sdk="Microsoft.NET.Sdk">
 
   <PropertyGroup>
-    <TargetFramework>net7.0</TargetFramework>
+    <TargetFramework>net8.0</TargetFramework>
     <ImplicitUsings>enable</ImplicitUsings>
     <Nullable>enable</Nullable>
   </PropertyGroup>

src/DataStreamSuite/DataStream/BlueOrb/AddColumnRow.cs

@@ -0,0 +1,31 @@
+namespace bts.BlueOrb;
+
+using BCG;
+
+public class AddColumnRow : IBlueRow {
+   readonly IBlueRow inputRow;
+   readonly IBlueTableSchema outputSchema;
+   readonly object? value;
+   readonly int valueIndex;
+
+   public AddColumnRow(IBlueRow inputRow, IBlueTableSchema outputSchema, object? value, int valueIndex) {
+      this.inputRow = inputRow;
+      this.outputSchema = outputSchema;
+      this.value = value;
+      this.valueIndex = valueIndex;
+   }
+
+   public object? this[int index] {
+      get {
+         if (index == valueIndex) {
+            return value;
+         }
+         if (index >= valueIndex) {
+            index--;
+         }
+         return inputRow[index];
+      }
+   }
+
+   IBlueTableSchema IBlueRow.schema => outputSchema;
+}

src/DataStreamSuite/DataStream/BlueOrb/AddColumnTable.cs

@@ -0,0 +1,62 @@
+namespace bts.BlueOrb;
+
+using BCG;
+
+using Bts.DataStreamCore;
+
+using System;
+using System.Collections.Generic;
+
+public class AddColumnTable : IBlueRowCollection {
+   readonly IBlueRowCollection inputRows;
+   readonly Func<IBlueRow, object?> getValue;
+   readonly IBlueTableSchema outputSchema;
+   readonly int valueIndex;
+
+   public AddColumnTable(IBlueRowCollection inputRows, string columnName, Type columnType, Func<IBlueRow, object?> getValue) {
+      this.inputRows = inputRows;
+      this.getValue = getValue;
+      var inputCols = inputRows.schema.columnInfo;
+      var outputCols = new ColumnInfo[inputCols.Length + 1];
+      Array.Copy(inputCols, outputCols, inputCols.Length);
+      outputCols[^1] = new(columnName, inputCols.Length, columnType);
+      outputSchema = new BlueTableSchema(outputCols, "AddColumnTable");
+      valueIndex = inputCols.Length;
+   }
+
+   public IBlueRow this[int id] {
+      get {
+         var r = inputRows[id];
+         return new AddColumnRow(r, outputSchema, getValue(r), valueIndex);
+      }
+   }
+
+   public IBlueTableSchema schema => outputSchema;
+
+   public int Count => inputRows.Count;
+
+   public bool containsID(int id) =>
+      inputRows.containsID(id);
+
+   public IEnumerator<IBlueRow> GetEnumerator() {
+      foreach (var r in inputRows) {
+         yield return new AddColumnRow(r, outputSchema, getValue(r), valueIndex);
+      }
+   }
+
+   public IBlueRow GetRow(int index) {
+      var r = inputRows.GetRow(index);
+      return new AddColumnRow(r, outputSchema, getValue(r), valueIndex);
+   }
+
+   public int IndexOf(IBlueRow item) {
+      int id = item.Id();
+      if (!inputRows.containsID(id)) {
+         return -1;
+      }
+      var r = inputRows[id];
+      return BlueRowEqualityComparer.Equals(new AddColumnRow(r, outputSchema, getValue(r), valueIndex), item)
+         ? inputRows.IndexOf(r)
+         : -1;
+   }
+}

src/DataStreamSuite/DataStream/BlueOrb/AddColumnsStream.cs

@@ -0,0 +1,11 @@
+namespace bts.BlueOrb {
+   using BCG;
+
+   [DataStreamUserRespresentation("AddColumnsStream")]
+   public class AddColumnsStream: ViewStream {
+      public AddColumnsStream(ViewStream parent_str, string add_col_expr) {
+         ViewStream rs = new BCG.AddColumnsStream(parent_str, add_col_expr);
+         mySetRealSource(rs);
+      }
+   }
+}

src/DataStreamSuite/DataStream/BlueOrb/ApplyStream.cs

@@ -0,0 +1,93 @@
+#nullable enable
+namespace bts.BlueOrb;
+
+using BCG;
+
+using Bts.DataStream;
+using Bts.DataStreamCore;
+
+using Spark.Core.Disposables;
+
+using System;
+
+public class ApplyStream : ViewStream {
+   readonly ViewStream source;
+   readonly Func<IBlueRow, object?> apply;
+   IBlueTableSchema? outputSchema;
+   BlueTableImmutable? lastFired;
+   IDisposable? sub;
+
+   public ApplyStream(ViewStream source, IFunctionEvaluator applyEvaluator) {
+      this.source = source;
+      apply = Apply;
+
+      object? Apply(IBlueRow r) {
+         var res = applyEvaluator.Evaluate(string.Empty, new StreamArgument[] { (null, 0, r) });
+         return res;
+      }
+   }
+
+   public ApplyStream(ViewStream source, Func<IBlueRow, object?> apply) {
+      this.source = source;
+      this.apply = apply;
+   }
+
+   protected override void myOnFirstSubscriber() {
+      sub = source.Subscribe(OnSourceChanged);
+
+      void OnSourceChanged(IDataPacket dp) {
+         Lazy<BlueTableImmutable> lastFired = new(() => this.lastFired ??= new(Log.notnull(outputSchema)));
+         Lazy<BlueTableImmutable.DataPacketBuilder> bd = new(() => lastFired.Value.ToDataPacketBuilder());
+
+         foreach (var r in dp.deletedRows) {
+            var res = apply(r);
+            if (res is IBlueRow blueRow) {
+               bd.Value.Remove(blueRow.Id());
+            } else if (res is IBlueRowCollection blueRows) {
+               foreach (var br in blueRows) {
+                  bd.Value.Remove(br.Id());
+               }
+            } else {
+               Log.assert(false, $"apply delegate returned obj of unexpected type '{res?.GetType()}'");
+            }
+         }
+         foreach (var r in dp.addedRows) {
+            var res = apply(r);
+            if (res is IBlueRow blueRow) {
+               outputSchema ??= blueRow.schema;
+               bd.Value.AddOrUpdate(blueRow);
+            } else if (res is IBlueRowCollection blueRows) {
+               outputSchema ??= blueRows.schema;
+               foreach (var br in blueRows) {
+                  bd.Value.AddOrUpdate(br);
+               }
+            } else {
+               if (outputSchema is null) {
+                  // We get the outputSchema from the result of the first apply, if it's null, we
+                  // cannot determine the output data type.
+                  // TODO: maybe support specifying return type in DSL?
+                  Log.notnull(res);
+               }
+               var resType = res?.GetType();
+               if (outputSchema is null) {
+                  Log.notnull(resType);
+                  Log.assert(ColumnInfo.isAllowedType(resType));
+                  outputSchema = new BlueTableSchema(new ColumnInfo[] {
+                     new("id", 0, typeof(int)),
+                     new("result", 1, resType),
+                  }, dp.newRows.schema.tableName + "_Apply");
+               }
+               bd.Value.AddOrUpdate(new BlueRow(outputSchema, new object?[] { r.Id(), res }));
+               // Log.assert(false, $"apply delegate returned obj of unexpected type '{res?.GetType()}'");
+            }
+         }
+         if (bd.IsValueCreated) {
+            myFireEvent(bd.Value.ToDataPacket(out this.lastFired));
+         }
+      }
+   }
+
+   protected override void myOnLastSubscriber() {
+      Disposable.dispose(ref sub);
+   }
+}

src/DataStreamSuite/DataStream/BlueOrb/ApplyTable.cs

@@ -0,0 +1,62 @@
+namespace bts.BlueOrb;
+
+using BCG;
+
+using Bts.DataStreamCore;
+
+using System;
+using System.Collections.Generic;
+
+public class ApplyTable : IBlueRowCollection {
+   readonly IBlueRowCollection source;
+   readonly IBlueTableSchema outputSchema;
+   readonly List<IBlueRow> outputRows;
+   readonly Dictionary<int, int> idToIndex = new();
+   readonly Func<IBlueRow, IBlueRow> applyFn;
+
+   public ApplyTable(IBlueRowCollection source, IBlueTableSchema? outputSchema, Func<IBlueRow, IBlueRow> applyFn) {
+      this.source = source;
+      this.applyFn = applyFn;
+      outputRows = new(source.Count);
+      foreach (var r in source) {
+         outputRows.Add(applyFn(r));
+      }
+      if (outputRows.Count > 0) {
+         outputSchema ??= outputRows[0].schema;
+      }
+      this.outputSchema = outputSchema ?? source.schema;
+   }
+
+   public IBlueRow this[int id] => outputRows[idToIndex[id]];
+
+   IBlueRow IList<IBlueRow>.this[int index] {
+      get => GetRow(index);
+      set => throw new NotSupportedException();
+   }
+
+   public IBlueTableSchema schema => outputSchema;
+
+   public int Count => source.Count;
+
+   public bool containsID(int id) =>
+      source.containsID(id);
+
+   public IEnumerator<IBlueRow> GetEnumerator() =>
+      outputRows.GetEnumerator();
+
+   public IBlueRow GetRow(int index) =>
+      outputRows[index];
+
+   public int IndexOf(IBlueRow item) {
+      int id = item.Id();
+      if (!source.containsID(id)) {
+         return -1;
+      }
+      var inputRow = source[id];
+      int idx = source.IndexOf(inputRow);
+      if (BlueRowEqualityComparer.Equals(outputRows[idx], item)) {
+         return idx;
+      }
+      return -1;
+   }
+}

src/DataStreamSuite/DataStream/BlueOrb/AsyncViewStream.cs

@@ -0,0 +1,61 @@
+namespace Bts.DataStream.BlueOrb;
+
+using BCG;
+
+using Bts.DataStream;
+using Bts.DataStreamCore;
+using Bts.PubSub;
+
+using Spark.Core.Disposables;
+
+using System;
+using System.Threading;
+using System.Threading.Tasks;
+
+public static class AsyncViewStream {
+   public static ViewStream Create(IWorkScheduler workScheduler, Func<Task<ViewStream?>> factory) {
+      return ViewStreamFactory.Create(Subscribe);
+
+      IDisposable Subscribe(Action<IDataPacket> fire) {
+         var cts = new CancellationTokenSource();
+         var tcs = new TaskCompletionSource<IDisposable>();
+         var syncRoot = new object();
+         IDisposable? streamSub = null;
+         workScheduler.Post(Init);
+         return Disposable.Create(Dispose);
+
+         async void Init() {
+            lock (syncRoot) {
+               if (cts.IsCancellationRequested) {
+                  return;
+               }
+            }
+            var stream = await factory();
+            if (stream is null) {
+               return;
+            }
+            var tempStreamSub = stream.Subscribe(fire);
+            lock (syncRoot) {
+               if (cts.IsCancellationRequested) {
+                  tempStreamSub.Dispose();
+               } else {
+                  streamSub = tempStreamSub;
+               }
+            }
+         }
+
+         void Dispose() {
+            IDisposable? tempStreamSub;
+            lock (syncRoot) {
+               tempStreamSub = streamSub;
+               if (tempStreamSub is null) {
+                  cts.Cancel();
+                  return;
+               }
+               streamSub = null;
+            }
+            tempStreamSub?.Dispose();
+         }
+      }
+   }
+}

src/DataStreamSuite/DataStream/BlueOrb/BlueAddIndexColumnTable.cs

@@ -0,0 +1,115 @@
+namespace bts.BlueOrb;
+
+using BCG;
+
+using bts.collections;
+using bts.utils;
+
+using Bts.DataStream;
+using Bts.DataStreamCore;
+
+using Spark.Core.Collections;
+
+using System;
+using System.Collections.Generic;
+using System.Linq;
+
+public class BlueAddIndexColumnTable : IBlueRowCollection {
+   readonly IBlueTableSchema schema;
+   readonly IList<object?>[] columnValues;
+   readonly int count;
+
+   BlueAddIndexColumnTable(IBlueTableSchema schema, IList<object?>[] columnValues, int? count = null) {
+      this.schema = schema;
+      this.columnValues = columnValues;
+      this.count = count ?? columnValues[0].Count;
+   }
+
+   IBlueTableSchema IBlueRowCollection.schema => schema;
+
+   public int Count => count;
+
+   public IBlueRow this[int id] =>
+      id > 0 && id <= count
+         ? GetRow(id - 1)
+         : throw new KeyNotFoundException();
+
+   public static BlueAddIndexColumnTable Create(string tableName, ColumnInfo[] columnInfos, IList<object?>[] columnValues) {
+      if (columnInfos.Any(c => c.colName == "id")) {
+         throw new ArgumentException("'id' column already exists", nameof(columnInfos));
+      }
+      if (columnInfos.Length != columnValues.Length) {
+         throw new ArgumentException("columnValues length differs from columnInfos length", nameof(columnValues));
+      }
+      int count = columnValues.FirstOrDefault()?.Count ?? 0;
+      var origCols = columnInfos;
+      var origColValues = columnValues;
+      columnInfos = new ColumnInfo[origCols.Length + 1];
+      columnInfos[0] = new ColumnInfo("id", 0, typeof(int));
+      columnValues = new IList<object?>[origCols.Length + 1];
+      columnValues[0] = new StepList<int>(1, 1, count).WrapSelector(id => (object?)id, null);
+      for (int i = 1; i < columnInfos.Length; i++) {
+         columnInfos[i] = new ColumnInfo(origCols[i - 1].colName, i, origCols[i - 1].colType);
+         columnValues[i] = origColValues[i - 1];
+      }
+      var schema = new BlueTableSchema(columnInfos, tableName);
+      return new(schema, columnValues, count);
+   }
+
+   public static BlueAddIndexColumnTable Create(IBlueTableSchema schema, IList<object?>[] columnValues) {
+      int count = columnValues.FirstOrDefault()?.Count ?? 0;
+      int? idColIdx = schema.TryResolveIndices("id");
+      var idColValues = new StepList<int>(1, 1, count).WrapSelector(id => (object?)id, null);
+      var columnInfos = schema.columnInfo;
+      if (idColIdx.HasValue) {
+         // Replace existing id column. columnValues may or may not contain the old ids (if its
+         // length is one less column than that of columnInfos, it doesn't contain them).
+         switch (columnValues.Length - schema.columnInfo.Length) {
+            case 0:
+               columnValues[idColIdx.Value] = idColValues;
+               break;
+            case -1:
+               columnValues = ArrayUtils.InsertExact(columnValues, idColIdx.Value, true, idColValues);
+               break;
+            default:
+               throw new ArgumentException("ColumnInfos length disagrees with columnValues length");
+         }
+      } else {
+         if (columnValues.Length != columnInfos.Length) {
+            throw new ArgumentException("ColumnInfos length disagrees with columnValues length");
+         }
+         var origColInfos = columnInfos;
+         columnInfos = new ColumnInfo[columnInfos.Length + 1];
+         columnInfos[0] = new("id", 0, typeof(int));
+         for (int i = 1; i < columnInfos.Length; i++) {
+            var c = origColInfos[i - 1];
+            columnInfos[i] = new(c.colName, c.colNo + 1, c.colType);
+         }
+         columnValues = ArrayUtils.InsertExact(columnValues, 0, true, idColValues);
+      }
+      return new(new BlueTableSchema(schema.tableName, columnInfos), columnValues);
+   }
+
+   public bool containsID(int id) =>
+      id > 0 && id <= count;
+
+   public IList<object?> GetColumn(int index) =>
+      columnValues[index];
+
+   public IBlueRow GetRow(int index) =>
+      new BlueRowColumnWrapper(schema, index, columnValues);
+
+   public int IndexOf(IBlueRow item) {
+      int id = item.Id();
+      return id > 0 && id <= count && BlueRowEqualityComparer.Equals(GetRow(id - 1), item)
+         ? id - 1
+         : -1;
+   }
+
+   public IEnumerator<IBlueRow> GetEnumerator() {
+      int count = Count;
+      for (int i = 0; i < count; i++) {
+         yield return GetRow(i);
+      }
+   }
+}

src/DataStreamSuite/DataStream/BlueOrb/BlueColumnRowWrapper.cs

@@ -0,0 +1,21 @@
+namespace bts.BlueOrb;
+
+using BCG;
+
+using System.Collections.Generic;
+
+public class BlueRowColumnWrapper : IBlueRow {
+   readonly int index;
+   readonly IList<object?>[] columns;
+
+   public BlueRowColumnWrapper(IBlueTableSchema schema, int index, IList<object?>[] columns) {
+      this.schema = schema;
+      this.index = index;
+      this.columns = columns;
+   }
+
+   public object? this[int index] =>
+      columns[index][this.index];
+
+   public IBlueTableSchema schema { get; }
+}

src/DataStreamSuite/DataStream/BlueOrb/BlueIndexedColumnTable.cs

@@ -0,0 +1,52 @@
+namespace bts.BlueOrb;
+
+using BCG;
+
+using Bts.DataStreamCore;
+
+using mbts.utils;
+
+using System.Collections.Generic;
+using System.Linq;
+
+public class BlueIndexedColumnTable : IBlueRowCollection {
+   readonly IBlueTableSchema schema;
+   readonly IList<object?>[] columnValues;
+   readonly Dictionary<int, int> rowIdToIndex;
+
+   public BlueIndexedColumnTable(IBlueTableSchema schema, IList<object?>[] columnValues) {
+      this.schema = schema;
+      this.columnValues = columnValues;
+      int idColumnIndex = schema.ResolveIndices("id");
+      rowIdToIndex = columnValues[idColumnIndex]
+         .Select((idObj, idx) => new KeyValuePair<int, int>(ExceptionAssert.Argument.Is<int>(idObj), idx))
+         .ToDictionary();
+   }
+
+   public IBlueRow this[int id] =>
+      rowIdToIndex.TryGetValue(id, out var idx)
+         ? GetRow(idx)
+         : throw new KeyNotFoundException();
+
+   IBlueTableSchema IBlueRowCollection.schema => schema;
+
+   public int Count => rowIdToIndex.Count;
+
+   public bool containsID(int id) =>
+      rowIdToIndex.ContainsKey(id);
+
+   public IEnumerator<IBlueRow> GetEnumerator() {
+      int count = Count;
+      for (int i = 0; i < count; i++) {
+         yield return GetRow(i);
+      }
+   }
+
+   public IBlueRow GetRow(int index) =>
+      new BlueRowColumnWrapper(schema, index, columnValues);
+
+   public int IndexOf(IBlueRow item) =>
+      rowIdToIndex.TryGetValue(item.Id(), out var idx) && BlueRowEqualityComparer.Equals(GetRow(idx), item)
+         ? idx
+         : -1;
+}

src/DataStreamSuite/DataStream/BlueOrb/CSVReaderStream.cs

@@ -0,0 +1,18 @@
+namespace bts.BlueOrb {
+   using System;
+
+   using BCG;
+
+   [DataStreamUserRespresentation("CSVReaderStream")]
+   public class CSVReaderStream : ViewStream {
+      public CSVReaderStream(string file_name)
+         : this(file_name, false) { }
+      public CSVReaderStream(string file_name, bool guess_schema) {
+         ViewStream rs = new BCG.CSVReaderStream(file_name);
+         if (guess_schema) {
+            rs = new SchemaGuesserStream(rs);
+         }
+         mySetRealSource(rs);
+      }
+   }
+}

src/DataStreamSuite/DataStream/BlueOrb/CSVSaverStream.cs

@@ -0,0 +1,13 @@
+namespace bts.BlueOrb {
+   using BCG;
+
+   using System;
+
+   [DataStreamUserRespresentation("CSVSaverStream")]
+   public class CSVSaverStream: ViewStream {
+      public CSVSaverStream(ViewStream parent_str, string file_name) {
+         BCG.CSVSaverStream rs = new BCG.CSVSaverStream(parent_str, file_name, true, Int32.MaxValue, false);
+         mySetRealSource(rs);
+      }
+   }
+}

src/DataStreamSuite/DataStream/BlueOrb/CastStream.cs

@@ -0,0 +1,43 @@
+namespace bts.BlueOrb;
+
+using BCG;
+
+using Bts.DataStream;
+
+using Spark.Core.Disposables;
+
+using System;
+
+public class CastStream : ViewStream, IDisposable {
+   readonly ViewStream inputStream;
+   readonly (string, Type)[] casts;
+   IDisposable? sub;
+
+   public CastStream(ViewStream inputStream, params (string, Type)[] casts) {
+      this.inputStream = inputStream;
+      this.casts = casts;
+   }
+
+   ~CastStream() =>
+      Dispose();
+
+   public void Dispose() {
+      GC.SuppressFinalize(this);
+      Disposable.dispose(ref sub);
+   }
+
+   protected override void myOnFirstSubscriber() {
+      inputStream.Subscribe(OnInputChanged);
+
+      void OnInputChanged(IDataPacket dp) {
+         var dpOut = new DataPacket(
+            new CastTable(dp.newRows, casts),
+            new CastTable(dp.addedRows, casts),
+            new CastTable(dp.deletedRows, casts));
+         myFireEvent(dpOut);
+      }
+   }
+
+   protected override void myOnLastSubscriber() =>
+      Disposable.dispose(ref sub);
+}

src/DataStreamSuite/DataStream/BlueOrb/CastTable.cs

@@ -0,0 +1,123 @@
+namespace bts.BlueOrb;
+
+using BCG;
+
+using bts.collections;
+
+using Bts.DataStream;
+using Bts.DataStreamCore;
+
+using Spark.Core.Collections;
+
+using System;
+using System.Collections.Generic;
+
+public class CastTable : IBlueRowCollection {
+   readonly IBlueRowCollection rows;
+   readonly IBlueTableSchema outputSchema;
+   readonly (int Index, Func<IBlueRow, object?> GetValue)[] casts;
+   readonly IComparer<(int Index, Func<IBlueRow, object?> GetValue)> castComparer =
+      TupleComparer.Comparer<int, Func<IBlueRow, object?>>(item1Ordinal: 0);
+
+   public CastTable(IBlueRowCollection rows, params (string, Type)[] casts) {
+      this.rows = rows;
+      var inputSchema = rows.schema;
+      var inputCols = inputSchema.columnInfo;
+      var castIndices = inputSchema.ResolveIndices(Array.ConvertAll(casts, c => c.Item1));
+      this.casts = new (int Index, Func<IBlueRow, object?> GetValue)[casts.Length];
+      var outputCols = inputCols.ToArray();
+      for (int i = 0; i < casts.Length; i++) {
+         var colIdx = castIndices[i];
+         this.casts[i] = (colIdx, GetCaster(casts[i].Item2));
+         var inputCol = inputCols[colIdx];
+         Log.assert(inputCol.colNo == colIdx);
+         outputCols[colIdx] = new(inputCol.colName, inputCol.colNo, casts[i].Item2);
+
+         Func<IBlueRow, object?> GetCaster(Type to) {
+            if (to == typeof(short)) {
+               return r => r.GetInt16(colIdx);
+            }
+            if (to == typeof(int)) {
+               return r => r.GetInt32(colIdx);
+            }
+            if (to == typeof(long)) {
+               return r => r.GetInt64(colIdx);
+            }
+            if (to == typeof(ushort)) {
+               return r => r.GetUInt16(colIdx);
+            }
+            if (to == typeof(uint)) {
+               return r => r.GetUInt32(colIdx);
+            }
+            if (to == typeof(ulong)) {
+               return r => r.GetUInt64(colIdx);
+            }
+            if (to == typeof(double)) {
+               return r => r.GetFloat64(colIdx);
+            }
+            if (to == typeof(float)) {
+               return r => r.GetFloat32(colIdx);
+            }
+            if (to == typeof(DateTime)) {
+               return r => r.GetDateTime(colIdx);
+            }
+            if (to == typeof(TimeSpan)) {
+               return r => r.GetTimeSpan(colIdx);
+            }
+            throw new NotImplementedException($"CastTable cast not implemented for (out) type {to}");
+         }
+      }
+      outputSchema = new BlueTableSchema(inputSchema.tableName + "_Cast", outputCols);
+   }
+
+   public IBlueRow this[int id] =>
+      new Row(rows[id], outputSchema, casts, castComparer);
+
+   public IBlueTableSchema schema => outputSchema;
+
+   public int Count => rows.Count;
+
+   public bool containsID(int id) =>
+      rows.containsID(id);
+
+   public IEnumerator<IBlueRow> GetEnumerator() {
+      foreach (var r in rows) {
+         yield return new Row(r, outputSchema, casts, castComparer);
+      }
+   }
+
+   public IBlueRow GetRow(int index) =>
+      new Row(rows.GetRow(index), outputSchema, casts, castComparer);
+
+   public int IndexOf(IBlueRow item) {
+      var inputRow = rows[item.Id()];
+      return BlueRowEqualityComparer.Equals(item, new Row(inputRow, outputSchema, casts, castComparer))
+         ? rows.IndexOf(inputRow)
+         : -1;
+   }
+
+   class Row : IBlueRow {
+      readonly IBlueRow source;
+      readonly (int Index, Func<IBlueRow, object?> GetValue)[] casts;
+      readonly IComparer<(int Index, Func<IBlueRow, object?> GetValue)> castComparer;
+      readonly IBlueTableSchema outputSchema;
+
+      public Row(IBlueRow source, IBlueTableSchema outputSchema, (int Index, Func<IBlueRow, object?> GetValue)[] casts, IComparer<(int Index, Func<IBlueRow, object?> GetValue)> castComparer) {
+         this.source = source;
+         this.casts = casts;
+         this.castComparer = castComparer;
+         this.outputSchema = outputSchema;
+      }
+
+      public object? this[int index] {
+         get {
+            var castIdx = casts.BinarySearch((index, null!), castComparer);
+            return castIdx > -1
+               ? casts[castIdx].GetValue(source)
+               : source[index];
+         }
+      }
+
+      IBlueTableSchema IBlueRow.schema => outputSchema;
+   }
+}

src/DataStreamSuite/DataStream/BlueOrb/ColumnInjectionStream.cs

@@ -0,0 +1,22 @@
+namespace bts.BlueOrb;
+
+using BCG;
+
+using Bts.DataStream;
+
+using System.Reactive.Linq;
+
+public class ColumnInjectionStream : ViewStream {
+   public ColumnInjectionStream(ViewStream inputStream, IInjectedColumn[] outputColumns) {
+      IBlueTableSchema? outputSchema = null;
+      mySetRealSource(inputStream.AsObservable().Select(Transform).AsViewStream());
+
+      IDataPacket Transform(IDataPacket dp) {
+         outputSchema ??= ColumnInjectionTable.GetOutputSchema(dp.Schema, outputColumns);
+         var newRows = new ColumnInjectionTable(dp.newRows, outputSchema, outputColumns);
+         var addedRows = new ColumnInjectionTable(dp.addedRows, outputSchema, outputColumns);
+         var deletedRows = new ColumnInjectionTable(dp.deletedRows, outputSchema, outputColumns);
+         return new DataPacket(newRows, addedRows, deletedRows);
+      }
+   }
+}

src/DataStreamSuite/DataStream/BlueOrb/ColumnInjectionTable.cs

@@ -0,0 +1,156 @@
+namespace bts.BlueOrb;
+
+using BCG;
+
+using Bts.DataStreamCore;
+
+using System;
+using System.Collections.Generic;
+
+public interface IInjectedColumn {
+   string Name { get; }
+   Type Type { get; }
+   int Ordinal { get; }
+   ColumnInfo ColumnInfo => new(Name, Ordinal, Type);
+   object? GetValue(IBlueRow row);
+}
+
+public interface IInjectedColumn<T> : IInjectedColumn {
+   Type IInjectedColumn.Type => typeof(T);
+
+   new T? GetValue(IBlueRow row);
+
+   object? IInjectedColumn.GetValue(IBlueRow row) =>
+      GetValue(row);
+}
+
+public class InjectedColumn : IInjectedColumn {
+   readonly Func<IBlueRow, object?> getValue;
+
+   public InjectedColumn(string name, int index, Type type, Func<IBlueRow, object?> getValue) {
+      Name = name;
+      Ordinal = index;
+      Type = type;
+      this.getValue = getValue;
+   }
+
+   public string Name { get; }
+   public int Ordinal { get; }
+   public Type Type { get; }
+
+   public object? GetValue(IBlueRow row) =>
+      getValue(row);
+}
+
+public class InjectedColumn<T> : IInjectedColumn<T> {
+   readonly Func<IBlueRow, T?> getValue;
+
+   public InjectedColumn(string name, int index, Func<IBlueRow, T?> getValue) {
+      Name = name;
+      Ordinal = index;
+      this.getValue = getValue;
+   }
+
+   public string Name { get; }
+   public int Ordinal { get; }
+
+   public T? GetValue(IBlueRow row) =>
+      getValue(row);
+}
+
+public class ColumnInjectionTable : IBlueRowCollection {
+   readonly IBlueTableSchema outputSchema;
+   readonly IInjectedColumn[] outputColumns;
+   readonly IBlueRowCollection source;
+
+   public ColumnInjectionTable(IBlueRowCollection source, IBlueTableSchema outputSchema, IInjectedColumn[] outputColumns) {
+      this.source = source;
+      this.outputSchema = outputSchema;
+      this.outputColumns = outputColumns;
+   }
+
+   public static ColumnInjectionTable Create(IBlueRowCollection source, IEnumerable<IInjectedColumn> injectedColumns) {
+      var inputCols = source.schema.columnInfo;
+      var outputColumns = new List<IInjectedColumn>(inputCols.Length + 1);
+      var outputColInfos = new List<ColumnInfo>(inputCols.Length + 1);
+      int injectionIdx = 0;
+      using var en = injectedColumns.GetEnumerator();
+      bool hasMoreInjections = en.MoveNext();
+      for (int i = 0; i < inputCols.Length;) {
+         if (hasMoreInjections && en.Current.Ordinal <= i) {
+            outputColInfos.Add(new(en.Current.Name, outputColumns.Count, en.Current.Type));
+            outputColumns.Add(en.Current);
+            injectionIdx++;
+            continue;
+         }
+         var ci = inputCols[i];
+         outputColInfos.Add(new(ci.colName, outputColumns.Count, ci.colType));
+         outputColumns.Add(new InjectedColumn(ci.colName, outputColumns.Count, ci.colType, r => r[ci.colNo]));
+      }
+      var outputSchema = new BlueTableSchema(outputColInfos.ToArray(), $"ColumnInjection({source.schema.tableName})");
+      return new(source, outputSchema, outputColumns.ToArray());
+   }
+
+   public static IBlueTableSchema GetOutputSchema(IBlueTableSchema inputSchema, IEnumerable<IInjectedColumn> injectedColumns) {
+      var inputCols = inputSchema.columnInfo;
+      var outputColumns = new List<IInjectedColumn>(inputCols.Length + 1);
+      var outputColInfos = new List<ColumnInfo>(inputCols.Length + 1);
+      using var en = injectedColumns.GetEnumerator();
+      bool hasMoreInjections = en.MoveNext();
+      for (int i = 0; i < inputCols.Length;) {
+         if (hasMoreInjections && en.Current.Ordinal <= i) {
+            outputColInfos.Add(new(en.Current.Name, outputColumns.Count, en.Current.Type));
+            outputColumns.Add(en.Current);
+            hasMoreInjections = en.MoveNext();
+            continue;
+         }
+         var ci = inputCols[i];
+         outputColInfos.Add(new(ci.colName, outputColumns.Count, ci.colType));
+         outputColumns.Add(new InjectedColumn(ci.colName, outputColumns.Count, ci.colType, r => r[ci.colNo]));
+         i++;
+      }
+      return new BlueTableSchema(outputColInfos.ToArray(), $"ColumnInjection({inputSchema.tableName})");
+   }
+
+   public IBlueRow this[int id] =>
+      new ColumnInjectionRow(source[id], outputSchema, outputColumns);
+
+   public IBlueTableSchema schema => throw new System.NotImplementedException();
+
+   public int Count => source.Count;
+
+   public bool containsID(int id) =>
+      source.containsID(id);
+
+   public IEnumerator<IBlueRow> GetEnumerator() {
+      foreach (var r in source) {
+         yield return new ColumnInjectionRow(r, outputSchema, outputColumns);
+      }
+   }
+
+   public IBlueRow GetRow(int index) =>
+      new ColumnInjectionRow(source.GetRow(index), outputSchema, outputColumns);
+
+   public int IndexOf(IBlueRow item) {
+      int id = item.Id();
+      return source.containsID(id) && BlueRowEqualityComparer.Equals(item, this[id])
+         ? source.IndexOf(source[id])
+         : -1;
+   }
+
+   class ColumnInjectionRow : IBlueRow {
+      readonly IBlueRow row;
+      readonly IInjectedColumn[] columns;
+
+      public ColumnInjectionRow(IBlueRow row, IBlueTableSchema schema, IInjectedColumn[] columns) {
+         this.row = row;
+         this.schema = schema;
+         this.columns = columns;
+      }
+
+      public IBlueTableSchema schema { get; }
+
+      public object? this[int index] =>
+         columns[index].GetValue(row);
+   }
+}

src/DataStreamSuite/DataStream/BlueOrb/ColumnMaskRow.cs

@@ -0,0 +1,19 @@
+namespace bts.BlueOrb;
+
+using BCG;
+
+public class ColumnMaskRow : IBlueRow {
+   readonly IBlueRow inputRow;
+   readonly (string? Name, int Index)[] columnMap;
+
+   public ColumnMaskRow(IBlueRow inputRow, (string? Name, int Index)[] columnMap, IBlueTableSchema schema) {
+      this.inputRow = inputRow;
+      this.columnMap = columnMap;
+      this.schema = schema;
+   }
+
+   public IBlueTableSchema schema { get; }
+
+   public object? this[int index] =>
+      inputRow[columnMap[index].Index];
+}

src/DataStreamSuite/DataStream/BlueOrb/ColumnMaskStream.cs

@@ -0,0 +1,68 @@
+namespace bts.BlueOrb;
+
+using BCG;
+
+using Bts.DataStream;
+
+using Spark.Core.Disposables;
+
+using System;
+using System.Diagnostics.CodeAnalysis;
+
+public class ColumnMaskStream : ViewStream {
+   readonly ViewStream source;
+   readonly string? columnMaskExpression;
+
+   IDisposable? sub;
+   IBlueTableSchema? outputSchema;
+   // Each item in this array represents an output column as a tuple of
+   // new name, and corresponding index in the input schema.
+   (string? OutputName, int InputIndex)[]? columnMappings;
+   (string From, string To)[]? columnNameMappings;
+   readonly bool includeUnmappedColumns;
+
+   public ColumnMaskStream(ViewStream source, string columnMaskExpression) {
+      this.source = source;
+      this.columnMaskExpression = columnMaskExpression;
+   }
+
+   public ColumnMaskStream(ViewStream source, bool includeUnmappedColumns, params (string From, string To)[] columnNameMappings) {
+      this.source = source;
+      this.columnNameMappings = columnNameMappings;
+      this.includeUnmappedColumns = includeUnmappedColumns;
+   }
+
+   ~ColumnMaskStream() =>
+      Disposable.dispose(ref sub);
+
+   protected override void myOnFirstSubscriber() {
+      Log.isnull(sub);
+      sub = source.Subscribe(OnSourceChanged);
+
+      void OnSourceChanged(IDataPacket dp) {
+         ensureOutputSchema(dp.Schema);
+         var addedRows = new ColumnMaskTable(dp.addedRows, columnMappings, outputSchema);
+         var deletedRows = new ColumnMaskTable(dp.deletedRows, columnMappings, outputSchema);
+         var allRows = new ColumnMaskTable(dp.newRows, columnMappings, outputSchema);
+         myFireEvent(new DataPacket(allRows, addedRows, deletedRows));
+      }
+   }
+
+   protected override void myOnLastSubscriber() =>
+      Disposable.dispose(ref sub);
+
+   [MemberNotNull(nameof(outputSchema), nameof(columnMappings))]
+   void ensureOutputSchema(IBlueTableSchema inputSchema) {
+      if (outputSchema is not null) {
+         Log.notnull(columnMappings);
+         return;
+      }
+      if (columnMaskExpression is not null) {
+         outputSchema = ColumnMaskTable.GetOutputSchema(inputSchema, columnMaskExpression, out columnMappings);
+      } else if (columnNameMappings is not null) {
+         outputSchema = ColumnMaskTable.GetOutputSchema(inputSchema, includeUnmappedColumns, columnNameMappings, out columnMappings);
+      } else {
+         throw new InvalidOperationException("Cannot build output schema");
+      }
+   }
+}

src/DataStreamSuite/DataStream/BlueOrb/ColumnMaskTable.cs

@@ -0,0 +1,120 @@
+namespace bts.BlueOrb;
+
+using BCG;
+
+using bts.collections;
+using bts.utils;
+
+using Bts.DataStreamCore;
+
+using System;
+using System.Collections.Generic;
+using System.Linq;
+
+public class ColumnMaskTable : IBlueRowCollection {
+   readonly IBlueRowCollection inputRows;
+   readonly (string? Name, int Index)[] columnMap;
+
+   public ColumnMaskTable(IBlueRowCollection inputRows, (string? Name, int Index)[] columnMap, IBlueTableSchema schema) {
+      this.inputRows = inputRows;
+      this.columnMap = columnMap;
+      this.schema = schema;
+   }
+
+   public IBlueTableSchema schema { get; }
+
+   public int Count => inputRows.Count;
+
+   public IBlueRow this[int id] =>
+      new ColumnMaskRow(inputRows[id], columnMap, schema);
+
+   public static ColumnMaskTable Create(IBlueRowCollection table, string columnMaskExpression) {
+      var outputSchema = GetOutputSchema(table.schema, columnMaskExpression, out var columnMappings);
+      return new ColumnMaskTable(table, columnMappings, outputSchema);
+   }
+
+   public static IBlueTableSchema GetOutputSchema(IBlueTableSchema inputSchema, string columnMaskExpression, out (string? Name, int Index)[] columnMappings) {
+      bool includeOthers;
+      if (columnMaskExpression.StartsWith('!')) {
+         columnMaskExpression = columnMaskExpression[1..].TrimStart();
+         includeOthers = false;
+      } else {
+         includeOthers = true;
+      }
+      var map = columnMaskExpression.Split(' ').Select(MapColumn).ToOrderedDictionary(tup => tup.Index, tup => tup.Name);
+      var inputCols = inputSchema.columnInfo;
+      if (includeOthers) {
+         var implicitCols = inputCols.Where(c => !map.ContainsKey(c.colNo)).ToList();
+         map.AddRange(implicitCols.Select(c => new KeyValuePair<int, string?>(c.colNo, null)));
+      } else if (!map.Any(c => inputCols[c.Key].colName == "id") && inputCols.TryGetFirst(c => c.colName == "id", out var idColInfo)) {
+         // Add 'id' col if missing
+         map.Insert(0, new(idColInfo.colNo, "id"));
+      }
+      var outputCols = new ColumnInfo[map.Count];
+      for (int i = 0; i < outputCols.Length; i++) {
+         var inputColInfo = inputCols[map.Keys[i]];
+         outputCols[i] = new(map.Values[i] ?? inputColInfo.colName, i, inputColInfo.colType);
+      }
+      columnMappings = map.WrapSelector(kv => (kv.Value, kv.Key), null).ToArray();
+      return new BlueTableSchema(outputCols, inputSchema.tableName + "_ColumnMask");
+
+      (string? Name, int Index) MapColumn(string columnStr) {
+         var parts = columnStr.Split(':');
+         return parts.Length switch {
+            1 => ((string?)null, inputSchema[parts[0]].colNo),
+            2 => (parts[1], inputSchema[parts[0]].colNo),
+            _ => throw new InvalidOperationException("Unable to get output schema due to invalid column expression"),
+         };
+      }
+   }
+
+   public static IBlueTableSchema GetOutputSchema(IBlueTableSchema inputSchema, bool includeUnmapped, (string From, string To)[] columnNameMappings, out (string? OutputName, int InputIndex)[] columnMappings) {
+      var inputCols = inputSchema.columnInfo;
+      ColumnInfo[]? outputCols;
+      if (includeUnmapped) {
+         columnMappings = Array.ConvertAll(inputCols, c => ((string?)c.colName, c.colNo));
+         outputCols = ArrayUtils.Clone(inputCols);
+         foreach (var (from, to) in columnNameMappings) {
+            var idx = inputCols.ResolveIndices(from);
+            var c = outputCols[idx];
+            Log.assert(c.colNo == idx);
+            outputCols[idx] = new(to, c.colNo, c.colType);
+            columnMappings[idx] = (to, idx);
+         }
+      } else {
+         outputCols = new ColumnInfo[columnNameMappings.Length];
+         columnMappings = Array.ConvertAll(columnNameMappings, m => ((string?)m.To, inputCols.ResolveIndices(m.From)));
+         Array.Copy(inputCols, outputCols, inputCols.Length);
+         for (int i = 0; i < columnNameMappings.Length; i++) {
+            var (from, to) = columnNameMappings[i];
+            var idx = inputCols.ResolveIndices(from);
+            outputCols[idx] = new(to, i, inputCols[columnMappings[i].InputIndex].colType);
+         }
+      }
+      return new BlueTableSchema(outputCols, inputSchema.tableName + "_ColumnMask");
+   }
+
+   public bool containsID(int id) =>
+      inputRows.containsID(id);
+
+   public IEnumerator<IBlueRow> GetEnumerator() {
+      foreach (var r in inputRows) {
+         yield return new ColumnMaskRow(r, columnMap, schema);
+      }
+   }
+
+   public IBlueRow GetRow(int index) =>
+      new ColumnMaskRow(inputRows.GetRow(index), columnMap, schema);
+
+   public int IndexOf(IBlueRow item) {
+      int id = item.Id();
+      if (inputRows.containsID(id)) {
+         var inputRow = inputRows[id];
+         var outputRow = new ColumnMaskRow(inputRow, columnMap, schema);
+         if (BlueRowEqualityComparer.Equals(item, outputRow)) {
+            return inputRows.IndexOf(inputRow);
+         }
+      }
+      return -1;
+   }
+}

src/DataStreamSuite/DataStream/BlueOrb/ColumnSelectorRow.cs

@@ -0,0 +1,61 @@
+// namespace bts.BlueOrb;
+
+// using BCG;
+
+// using System;
+// using System.Linq;
+
+// /// <summary>
+// /// Represents a row transformation table (all columns except 'id' can be changed).
+// /// </summary>
+// public class ColumnSelectorRow : IBlueRow {
+//    readonly IBlueRow source;
+//    readonly IBlueTableSchema outputSchema;
+//    readonly Func<object?, object?>[] columnSelectors;
+
+//    public ColumnSelectorRow(IBlueRow source, params (int Index, Type Type, Func<object?, object?> GetValue)[] columnSelectors) {
+//       this.source = source;
+//       this.columnSelectors = Array.ConvertAll(columnSelectors, c => (c.Index, c.GetValue));
+//       outputSchema = getSchema(source.schema, columnSelectors);
+//    }
+
+//    public ColumnSelectorRow(IBlueRow source, IBlueTableSchema outputSchema, params (int Index, Func<object?, object?> GetValue)[] columnSelectors) {
+//       this.source = source;
+//       this.outputSchema = outputSchema;
+//       this.columnSelectors = columnSelectors;
+//    }
+
+//    public object? this[int index] {
+//       get {
+//          for (int i = 0; i < columnSelectors.Length; i++) {
+//             if (columnSelectors[i].Index == index) {
+//                return columnSelectors[i].GetValue(source[index]);
+//             }
+//          }
+//          return source[index];
+//       }
+//    }
+
+//    public IBlueTableSchema schema => outputSchema;
+
+//    public static Func<object?, object?>[] BuildColumnSelectors(IBlueTableSchema schema, params (int Index, Func<object?, object?> GetValue)[] columnSelectors) {
+//       var cols = schema.columnInfo;
+//       var rv = Array.ConvertAll(schema.columnInfo, c => new Func<object?, object?>(_ => _));
+//       for (int i = 0; i < columnSelectors.Length; i++) {
+//          var cs = columnSelectors[i];
+//          rv[cs.Index] = cs.GetValue;
+//       }
+//       return rv;
+//    }
+
+//    static IBlueTableSchema getSchema(IBlueTableSchema inputSchema, (int Index, Type Type, Func<object?, object?> GetValue)[] columnSelectors) {
+//       var inputCols = inputSchema.columnInfo;
+//       var outputCols = inputCols.ToArray();
+//       for (int i = 0; i < columnSelectors.Length; i++) {
+//          var (idx, typ, _) = columnSelectors[i];
+//          var col = inputCols[idx];
+//          outputCols[idx] = new(col.colName, col.colNo, typ);
+//       }
+//       return new BlueTableSchema(outputCols, "ColumnSelector");
+//    }
+// }

src/DataStreamSuite/DataStream/BlueOrb/ColumnSelectorRowFactory.cs

@@ -0,0 +1,107 @@
+namespace bts.BlueOrb;
+
+using BCG;
+
+using bts.collections;
+
+using Bts.DataStreamCore;
+
+using System;
+using System.Collections.Generic;
+using System.Linq;
+
+public class ColumnSelectorRowFactory {
+   readonly IBlueTableSchema outputSchema;
+   readonly Func<object?, object?>[] columnSelectors;
+
+   public ColumnSelectorRowFactory(IBlueTableSchema outputSchema, Func<object?, object?>[] columnSelectors) {
+      this.outputSchema = outputSchema;
+      this.columnSelectors = columnSelectors;
+   }
+
+   public IBlueTableSchema OutputSchema => outputSchema;
+
+   public static ColumnSelectorRowFactory Create(IBlueTableSchema inputSchema, params (string Name, Type Type, Func<object?, object?> GetValue)[] columnOverrides) =>
+      Create(inputSchema, columnOverrides.AsEnumerable());
+
+   public static ColumnSelectorRowFactory Create(IBlueTableSchema inputSchema, params (int Index, Type Type, Func<object?, object?> GetValue)[] columnOverrides) =>
+      Create(inputSchema, columnOverrides.AsEnumerable());
+
+   public static ColumnSelectorRowFactory Create(IBlueTableSchema inputSchema, IEnumerable<(string Name, Type Type, Func<object?, object?> GetValue)> columnOverrides) =>
+      Create(inputSchema, columnOverrides.Select(c => (inputSchema.ResolveIndices(c.Name), c.Type, c.GetValue)));
+
+   public static ColumnSelectorRowFactory Create(IBlueTableSchema inputSchema, IEnumerable<(int Index, Type Type, Func<object?, object?> GetValue)> columnOverrides) {
+      var bd = new Builder(inputSchema);
+      foreach (var c in columnOverrides) {
+         bd.AddOrUpdate(c);
+      }
+      return bd.ToImmutable();
+   }
+
+   public IBlueRow Create(IBlueRow row) =>
+      new Row(row, outputSchema, columnSelectors);
+
+   public Builder ToBuilder() =>
+      new(outputSchema);
+
+   class Row : IBlueRow {
+      readonly IBlueRow source;
+      readonly IBlueTableSchema schema;
+      readonly Func<object?, object?>[] columnSelectors;
+
+      public Row(IBlueRow source, IBlueTableSchema schema, Func<object?, object?>[] columnSelectors) {
+         this.source = source;
+         this.schema = schema;
+         this.columnSelectors = columnSelectors;
+      }
+
+      public object? this[int index] =>
+         columnSelectors[index](source[index]);
+
+      IBlueTableSchema IBlueRow.schema => schema;
+   }
+
+   public class Builder {
+      readonly IBlueTableSchema inputSchema;
+      readonly List<(int Index, Type Type, Func<object?, object?> GetValue)> columnOverrides = new();
+
+      public Builder(IBlueTableSchema inputSchema) {
+         this.inputSchema = inputSchema;
+      }
+
+      public IBlueTableSchema ToSchema() =>
+         ToSchema(inputSchema.tableName + "_ColumnSelector");
+
+      public IBlueTableSchema ToSchema(string tableName) {
+         var inputCols = inputSchema.columnInfo;
+         var outputCols = inputCols.ToArray();
+         foreach (var c in columnOverrides) {
+            var inputCol = inputCols[c.Index];
+            outputCols[c.Index] = new(inputCol.colName, c.Index, c.Type);
+         }
+         return new BlueTableSchema(outputCols, tableName);
+      }
+
+      public ColumnSelectorRowFactory ToImmutable() {
+         var outputSchema = ToSchema();
+         var columnSelectors = Array.ConvertAll(outputSchema.columnInfo, c => new Func<object?, object?>(_ => _));
+         foreach (var cs in columnOverrides) {
+            columnSelectors[cs.Index] = cs.GetValue;
+         }
+         return new(outputSchema, columnSelectors);
+      }
+
+      public void AddOrUpdate((int Index, Type Type, Func<object?, object?> GetValue) item) =>
+         columnOverrides.Add(item);
+
+      public void AddOrUpdate(int index, Type type, Func<object?, object?> getValue) =>
+         columnOverrides.Add((index, type, getValue));
+
+      public void AddOrUpdate<T>(int index, Func<object?, T> getValue) {
+         columnOverrides.Add((index, typeof(T), GetValueWrapper));
+
+         object? GetValueWrapper(object? inputVal) =>
+            getValue(inputVal);
+      }
+   }
+}

src/DataStreamSuite/DataStream/BlueOrb/CountStream.cs

@@ -0,0 +1,46 @@
+using System;
+
+namespace BCG {
+   [DataStreamUserRespresentation("CountStream")]
+   public class CountStream : ViewStream {
+      private DataPacket myLastFiredDataPacket;
+      private IBlueTableSchema mySchema = new BlueTableSchema("count",
+         new ColumnInfo("id", 0, typeof(int)),
+         new ColumnInfo("count", 1, typeof(int)));
+      private int myId = 0;
+      private ViewStream myVS;
+
+      public CountStream(ViewStream vs) {
+         myVS = vs;
+      }
+
+      protected override void myOnFirstSubscriber() {
+         myVS.onChange += myHandleChanges;
+      }
+      protected override void myOnLastSubscriber() {
+         myVS.onChange -= myHandleChanges;
+      }
+
+
+      private void myHandleChanges(object newValue) {
+         myFireEvent(
+            myLastFiredDataPacket =
+               myLastFiredDataPacket == null ?
+                  new DataPacket(myGetRows(newValue, ++myId), myGetRows(newValue, myId),
+                  new BlueTable(mySchema))
+                  :
+                  new DataPacket(myGetRows(newValue, ++myId), myGetRows(newValue, myId),
+                  myLastFiredDataPacket.newRows)
+            );
+      }
+
+      private IBlueRowCollection myGetRows(object dp, int id) {
+         BlueTable retVal = new BlueTable(mySchema);
+         retVal.addRow
+            (new BlueRow(mySchema, id, (dp as DataPacket).newRows.Count));
+         return retVal;
+      }
+
+
+   }
+}

src/DataStreamSuite/DataStream/BlueOrb/CovarianceTable.cs

@@ -0,0 +1,117 @@
+namespace bts.BlueOrb;
+
+using BCG;
+
+using Bts.DataStreamCore;
+
+using MathNet.Numerics.Statistics;
+
+using System;
+using System.Collections.Generic;
+using System.Linq;
+
+public class CovarianceTable : IBlueRowCollection {
+   readonly IBlueTableSchema schema;
+   readonly Lazy<IBlueRow> row;
+
+   public CovarianceTable(IBlueRowCollection rows1, IBlueRowCollection rows2) {
+      var inputSchema = rows1.schema;
+      var inputCols = inputSchema.columnInfo;
+      var outputCols = Array.ConvertAll(rows1.schema.columnInfo, ColumnFactory);
+      schema = new BlueTableSchema(outputCols, inputSchema.tableName + "_" + rows2.schema.tableName + "_Covariance");
+      row = new(RowFactory);
+
+      static ColumnInfo ColumnFactory(ColumnInfo inputCol) {
+         var t = inputCol.colType;
+         if (inputCol.colName != "id") {
+            if (t == typeof(short) || t == typeof(int) || t == typeof(long)
+               || t == typeof(ushort) || t == typeof(uint) || t == typeof(ulong)
+               || t == typeof(double) || t == typeof(float)) {
+               t = typeof(double);
+            } else if (t == typeof(DateTime)) {
+               t = typeof(TimeSpan);
+            }
+         }
+         return new(inputCol.colName, inputCol.colNo, t);
+      }
+
+      IBlueRow RowFactory() {
+         var rowVals = new object[inputCols.Length];
+         for (int i = 0; i < inputCols.Length; i++) {
+            var col = inputCols[i];
+            var colType = col.colType;
+            if (col.colName == "id") {
+               rowVals[i] = 1;
+            } else if (colType == typeof(short)) {
+               rowVals[i] = GetValue(v => v as short?);
+            } else if (colType == typeof(int)) {
+               rowVals[i] = GetValue(v => v as int?);
+            } else if (colType == typeof(long)) {
+               rowVals[i] = GetValue(v => v as long?);
+            } else if (colType == typeof(ushort)) {
+               rowVals[i] = GetValue(v => v as ushort?);
+            } else if (colType == typeof(uint)) {
+               rowVals[i] = GetValue(v => v as uint?);
+            } else if (colType == typeof(ulong)) {
+               rowVals[i] = GetValue(v => v as ulong?);
+            } else if (colType == typeof(double)) {
+               rowVals[i] = GetValue(v => v as double?);
+            } else if (colType == typeof(float)) {
+               rowVals[i] = GetValue(v => v as float?);
+            } else if (colType == typeof(DateTime)) {
+               rowVals[i] = TimeSpan.FromSeconds(GetValue(v => (v as DateTime?)?.Ticks / 10_000_000));
+            }
+
+            double GetValue(Func<object?, double?> convert) {
+               var vals1 = rows1.GetColumn(i)
+                  .Select(convert)
+                  .SelectWhereValues();
+               var vals2 = rows2.GetColumn(i)
+                  .Select(convert)
+                  .SelectWhereValues();
+               return vals1.Covariance(vals2);
+            }
+         }
+         return new Row(schema, rowVals);
+      }
+   }
+
+   public IBlueRow this[int id] =>
+      id == 1 ? row.Value : throw new KeyNotFoundException();
+
+   IBlueTableSchema IBlueRowCollection.schema => schema;
+
+   public int Count => 1;
+
+   IBlueRow IList<IBlueRow>.this[int index] {
+      get => GetRow(index);
+      set => throw new NotSupportedException();
+   }
+
+   public bool containsID(int id) =>
+      id == 1;
+
+   public IEnumerator<IBlueRow> GetEnumerator() {
+      yield return row.Value;
+   }
+
+   public IBlueRow GetRow(int index) =>
+      index == 0 ? row.Value : throw new IndexOutOfRangeException();
+
+   public int IndexOf(IBlueRow item) =>
+      BlueRowEqualityComparer.Equals(item, row.Value) ? 0 : -1;
+
+   class Row : IBlueRow {
+      readonly IBlueTableSchema schema;
+      readonly object?[] values;
+
+      public Row(IBlueTableSchema schema, object?[] values) {
+         this.schema = schema;
+         this.values = values;
+      }
+
+      public object? this[int index] => values[index];
+
+      IBlueTableSchema IBlueRow.schema => schema;
+   }
+}

src/DataStreamSuite/DataStream/BlueOrb/CummulativeSumStream.cs

@@ -0,0 +1,93 @@
+namespace bts.BlueOrb;
+
+using BCG;
+
+using bts.collections;
+
+using Bts.DataStream;
+using Bts.DataStreamCore;
+
+using Spark.Core.Disposables;
+
+using System;
+using System.Diagnostics.CodeAnalysis;
+using System.Linq;
+
+public class CummulativeSumStream : ViewStream, IDisposable {
+   readonly ViewStream inputStream;
+   readonly CummulativeSumTable.StringBehavior stringBehavior;
+   IBlueTableSchema? schema;
+   IDisposable? sub;
+   SingleRowTable? lastFired;
+   object?[]? lastFiredValues;
+   Func<object?, object?, object?>[]? columnOperators;
+   int idColumnIdx = -1;
+
+   public CummulativeSumStream(ViewStream inputStream, CummulativeSumTable.StringBehavior stringBehavior = CummulativeSumTable.StringBehavior.LastValue) {
+      this.inputStream = inputStream;
+      this.stringBehavior = stringBehavior;
+   }
+
+   ~CummulativeSumStream() =>
+      Disposable.dispose(ref sub);
+
+   public void Dispose() {
+      GC.SuppressFinalize(this);
+      Disposable.dispose(ref sub);
+   }
+
+   protected override void myOnFirstSubscriber() {
+      Disposable.dispose(ref sub);
+      sub = inputStream.Subscribe(OnInputStreamChanged);
+
+      void OnInputStreamChanged(IDataPacket dp) {
+         ensureInit(dp);
+         var values = lastFiredValues?.ToArray() ?? new object?[schema.columnInfo.Length];
+         foreach (var r in dp.addedRows) {
+            for (int i = 0; i < values.Length; i++) {
+               values[i] = columnOperators[i](values[i], r[i]);
+            }
+         }
+         var table = new SingleRowTable(new BlueRow(schema, values));
+         DataPacket dpOut = new(table, table, lastFired as IBlueRowCollection ?? new BlueTable(schema));
+         lastFired = table;
+         lastFiredValues = values;
+         myFireEvent(dpOut);
+      }
+   }
+
+   protected override void myOnLastSubscriber() =>
+      Disposable.dispose(ref sub);
+
+   [MemberNotNull(nameof(schema), nameof(columnOperators))]
+   void ensureInit(IDataPacket dp) {
+      if (schema is not null) {
+         Log.notnull(columnOperators);
+         return;
+      }
+      schema = dp.Schema;
+      idColumnIdx = schema.ResolveIndices("id");
+      var colInfos = schema.columnInfo;
+      columnOperators = new Func<object?, object?, object?>[schema.columnInfo.Length];
+      for (int i = 0; i < colInfos.Length; i++) {
+         if (i == idColumnIdx) {
+            columnOperators[i] = (x, y) => 1;
+            continue;
+         }
+         columnOperators[i] = schema.columnInfo[i].type_code() switch {
+            TypeCode.Int32 => (x, y) => (x as int? ?? 0) + (BlueDataUtility.GetInt32(y) ?? 0),
+            TypeCode.Int64 => (x, y) => (x as long? ?? 0) + (BlueDataUtility.GetInt64(y) ?? 0),
+            TypeCode.UInt32 => (x, y) => (x as uint? ?? 0) + (BlueDataUtility.GetUInt32(y) ?? 0),
+            TypeCode.UInt64 => (x, y) => (x as ulong? ?? 0) + (BlueDataUtility.GetUInt64(y) ?? 0),
+            TypeCode.Single => (x, y) => (x as float? ?? 0) + (BlueDataUtility.GetFloat32(y) ?? 0),
+            TypeCode.Double => (x, y) => (x as double? ?? 0) + (BlueDataUtility.GetFloat64(y) ?? 0),
+            TypeCode.String => stringBehavior switch {
+               CummulativeSumTable.StringBehavior.Concatenate => (x, y) => (x as string ?? string.Empty) + (y as string ?? string.Empty),
+               _ => (accum, next) => next,
+            },
+            TypeCode.DateTime => (x, y) => y,
+            _ => throw new InvalidOperationException($"Failed to add row values for col '{schema.columnInfo[i]}'"),
+         };
+      }
+   }
+}

src/DataStreamSuite/DataStream/BlueOrb/CummulativeSumTable.cs

@@ -0,0 +1,119 @@
+namespace bts.BlueOrb;
+
+using BCG;
+
+using bts.collections;
+
+using Bts.DataStreamCore;
+
+using System;
+using System.Collections.Generic;
+using System.Linq;
+
+public class CummulativeSumTable : IBlueRowCollection {
+   readonly IBlueRowCollection input;
+   readonly IList<object?>[] columns;
+   readonly StringBehavior stringBehavior;
+
+   public CummulativeSumTable(IBlueRowCollection input, StringBehavior stringBehavior = StringBehavior.LastValue) {
+      this.input = input;
+      var colInfos = input.schema.columnInfo.OrderBy(c => c.colNo).ToArray();
+      if (colInfos.Length > 0) {
+         Log.assert(colInfos[0].colNo == 0 && colInfos[^1].colNo == colInfos.Length - 1);
+      }
+      columns = colInfos.WrapSelector(c => GetColumn(c.colNo), null).ToArray();
+   }
+
+   /// <summary>
+   /// By default, we take the last value for strings intstead of adding (concatenating) them.
+   /// Otherwise it would be too easy to create massive concatenated strings by mistake.
+   /// </summary>
+   public enum StringBehavior {
+      LastValue,
+      Concatenate
+   }
+
+   public IBlueRow this[int id] =>
+      id == 1 ? GetRow(0) : throw new KeyNotFoundException();
+
+   public IBlueTableSchema schema => input.schema;
+
+   public int Count => 1;
+
+   public bool containsID(int id) =>
+      id == 1;
+
+   public IList<object?> GetColumn(int colIndex) {
+      var count = input.Count;
+      var colInfo = input.schema.columnInfo[colIndex];
+      if (colInfo.colName == "id") {
+         return new object?[] { 1 };
+      }
+      object? res = null;
+      var colVals = input.GetColumn(colIndex);
+      if (colInfo.colType == typeof(TimeSpan)) {
+         // TimeSpan is a special case because it doesn't have a TypeCode.
+         res = TimeSpanImpl(colVals);
+      } else {
+         var typeCode = colInfo.type_code();
+         const int MAX_STRING_LENGTH = 1024;
+         res = typeCode switch {
+            TypeCode.Int32 => colVals.Cast<int>().Sum(),
+            TypeCode.Int64 => colVals.Cast<long>().Sum(),
+            TypeCode.UInt32 => colVals.Cast<uint>().Sum(),
+            TypeCode.UInt64 => colVals.Cast<ulong>().Aggregate(0UL, (s, v) => s + v),
+            TypeCode.Single => colVals.Cast<float>().Sum(),
+            TypeCode.Double => colVals.Cast<double>().Sum(),
+            TypeCode.DateTime => colVals.Cast<DateTime>().Aggregate(DateTime.MinValue, (s, v) => new DateTime(s.Ticks + v.Ticks)),
+            TypeCode.String => stringBehavior switch {
+               StringBehavior.Concatenate => colVals.Cast<string>().Aggregate(string.Empty, (s, v) => {
+                  if (s.Length + v.Length > MAX_STRING_LENGTH) {
+                     throw new InvalidOperationException("Cannot sum strings because the result is too long");
+                  }
+                  return s + v;
+               }),
+               _ => count == 0 ? null : colVals[^1] as string,
+            },
+            _ => null,
+         };
+      }
+      return new object?[] { res };
+
+      static object? TimeSpanImpl(IList<object?> colVals) {
+         var ticks = colVals.Cast<TimeSpan>().Select(ts => ts.Ticks).Sum();
+         return TimeSpan.FromTicks(ticks);
+      }
+   }
+
+   public IEnumerator<IBlueRow> GetEnumerator() {
+      yield return GetRow(0);
+   }
+
+   public IBlueRow GetRow(int index) =>
+      new BlueRowColumnWrapper(input.schema, index, columns);
+
+   public int IndexOf(IBlueRow item) {
+      int id = item.Id();
+      var inputRow = input[id];
+      int idx = input.IndexOf(inputRow);
+      if (idx != -1) {
+         var row = GetRow(idx);
+         if (BlueRowEqualityComparer.Equals(item, row)) {
+            return idx;
+         }
+      }
+      return -1;
+   }
+}
+
+public class CummulativeSumTableEvaluator : IFunctionEvaluator {
+   public object? Evaluate(string functionId, IEnumerable<StreamArgument> args) {
+      if (args.TryGetFirst(out var a)) {
+         var table = a.Value as IBlueRowCollection;
+         if (table is not null) {
+            return new CummulativeSumTable(table);
+         }
+      }
+      return null;
+   }
+}

src/DataStreamSuite/DataStream/BlueOrb/DateTimeExpanderStream.cs

@@ -1,19 +1,19 @@
-namespace bts.SilverPine {
+namespace bts.BlueOrb {
    using bts;
 
    using BCG;
 
    using System;
-
+   [DataStreamUserRespresentation("DateTimeExpanderStream")]
    public class DateTimeExpanstionStream : ViewStream {
       public DateTimeExpanstionStream(ViewStream parent_stream, string inp_col) {
          Func<IBlueTableSchema, IBlueTableSchema> get_schema = (s) => {
             IBlueTableSchema rv = s.add_column(System.TypeCode.DateTime, $"{inp_col}_date");
-            rv = rv.add_column(System.TypeCode.String, $"{inp_col}_day");
+            rv = rv.add_column(System.TypeCode.String, $"{inp_col}_day_of_week");
             rv = rv.add_column(System.TypeCode.Int32, $"{inp_col}_day_of_month");
             rv = rv.add_column(System.TypeCode.Int32, $"{inp_col}_month");
             rv = rv.add_column(System.TypeCode.Int32, $"{inp_col}_hour");
-            rv = rv.add_column(System.TypeCode.Int32, $"{inp_col}_min");
+            rv = rv.add_column(System.TypeCode.Int32, $"{inp_col}_minute");
             rv = rv.add_column(System.TypeCode.Int32, $"{inp_col}_seconds_of_day");
             return rv;
          };
@@ -24,15 +24,15 @@ namespace bts.SilverPine {
             IBlueTableSchema s = get_schema(r.schema);
             MutableBlueRow rv = new MutableBlueRow(s, r);
             rv.setValue($"{inp_col}_date", dt.Date);
-            rv.setValue($"{inp_col}_day", dt.DayOfWeek.ToString());
+            rv.setValue($"{inp_col}_day_of_week", dt.DayOfWeek.ToString());
             rv.setValue($"{inp_col}_day_of_month", dt.Day);
             rv.setValue($"{inp_col}_month", dt.Date.Month);
             rv.setValue($"{inp_col}_hour", dt.Hour);
-            rv.setValue($"{inp_col}_min", dt.Minute);
+            rv.setValue($"{inp_col}_minute", dt.Minute);
             rv.setValue($"{inp_col}_seconds_of_day", (dt - dt.Date).TotalSeconds);
             return rv;
          });
          mySetRealSource(rs);
       }
    }
-}
+}

src/DataStreamSuite/DataStream/BlueOrb/DiffStream.cs

@@ -0,0 +1,163 @@
+namespace bts.BlueOrb;
+
+using BCG;
+
+using Bts.DataStream;
+using Bts.DataStreamCore;
+
+using Spark.Core.Collections;
+using Spark.Core.Disposables;
+
+using System;
+using System.Collections.Generic;
+using System.Linq.Expressions;
+
+public class DiffStream : ViewStream, IDisposable {
+   readonly ViewStream source;
+   readonly int period;
+   IBlueTableSchema? outputSchema;
+   IDisposable? sub;
+   BlueTableImmutable? lastFired;
+
+   public DiffStream(ViewStream source, int period) {
+      this.source = source;
+      this.period = period;
+   }
+
+   ~DiffStream() =>
+      Dispose();
+
+   public void Dispose() =>
+      Disposable.dispose(ref sub);
+
+   protected override void myOnFirstSubscriber() {
+      Log.isnull(sub);
+      sub = source.Subscribe(OnSourceChange);
+
+      void OnSourceChange(IDataPacket dp) {
+         if (outputSchema is null) {
+            var inputSchema = dp.Schema;
+            var outputCols = new ColumnInfo[inputSchema.columnInfo.Length];
+            for (int i = 0; i < inputSchema.columnInfo.Length; i++) {
+               outputCols[i] = GetOutputColumn(inputSchema.columnInfo[i]);
+            }
+            outputSchema = new BlueTableSchema($"{inputSchema.tableName}_Diff{period}", outputCols);
+         }
+         lastFired ??= new(outputSchema);
+         var newTable = new DiffTable(dp.newRows, period);
+         var allRows = new List<object?[]>();
+         foreach (IBlueRow r in newTable) {
+            var rvals = new object?[r.schema.columnInfo.Length];
+            int x = 0;
+            foreach (var col in r.schema.columnInfo) {
+               var val = r[col.colNo];
+               rvals[x++] = val;
+            }
+            allRows.Add(rvals);
+         }
+         var bd = lastFired.ToDataPacketBuilder();
+         var removedIndices = new List<int>();
+         var addedIndices = new List<int>();
+         var ops = new List<(int Index, bool IsAdd)>();
+         foreach (var r in dp.deletedRows) {
+            int id = r.Id();
+            var rOut = lastFired[id];
+            var idx = lastFired.IndexOf(rOut);
+            Log.assert(idx >= 0 && idx < lastFired.Count);
+            removedIndices.Add(idx);
+            ops.Add((idx, false));
+         }
+         foreach (var r in dp.addedRows) {
+            var idx = dp.newRows.IndexOf(r);
+            Log.assert(idx >= 0 && idx < dp.newRows.Count);
+            addedIndices.Add(idx);
+            ops.Add((idx, true));
+         }
+         ops.Sort(Comparer<(int Index, bool IsAdd)>.Create((a, b) => a.Index.CompareTo(b.Index)));
+         int rmCount = 0;
+         var removedRows = new Dictionary<int, IBlueRow>();
+         for (int i = 0; i < ops.Count; i++) {
+            var op = ops[i];
+            if (op.IsAdd) {
+               continue;
+            }
+            rmCount++;
+            int j = op.Index/* - rmCount*/;
+            var stop = Math.Min(j + period, lastFired.Count/* - rmCount*/);
+            // Remove the row that was removed in the incoming DataPacket.
+            bd.Remove(lastFired.GetRow(j).Id());
+            for (j++; j < stop; j++) {
+               // Remove rows that changed as a result. We will add the updated versions 
+               // of these rows in the next pass.
+               var r = lastFired.GetRow(j);
+               bd.Remove(r.Id());
+            }
+         }
+         int addCount = ops.Count - rmCount;
+         for (int i = ops.Count - 1; i >= 0; i--) {
+            var op = ops[i];
+            if (op.IsAdd) {
+               addCount--;
+               int j = op.Index;
+               var stop = Math.Min(j + period, newTable.Count);
+               for (; j < stop; j++) {
+                  var r = newTable.GetRow(j);
+                  bd.AddOrUpdate(r);
+               }
+            } else {
+               rmCount--;
+               int j = op.Index - rmCount + addCount;
+               var stop = Math.Min(j + period, newTable.Count);
+               for (; j < stop; j++) {
+                  // Add rows that changed due to the removed row (the row with which they compute
+                  // the diff has changed).
+                  var r = newTable.GetRow(j);
+                  bd.AddOrUpdate(r);
+               }
+            }
+         }
+         var dpOut = bd.ToDataPacket(out lastFired, false);
+         myFireEvent(dpOut);
+      }
+
+      ColumnInfo GetOutputColumn(ColumnInfo inputColumn) {
+         return inputColumn.type_code() switch {
+            TypeCode.Boolean => inputColumn.Clone(),
+            TypeCode.Byte or TypeCode.SByte or TypeCode.Int16 => WithType(TypeCode.Int32),
+            TypeCode.UInt32 or TypeCode.UInt64 => WithType(TypeCode.Int64),
+            _ => inputColumn.Clone(),
+         };
+
+         ColumnInfo WithType(TypeCode typeCode) =>
+            new(inputColumn.colName, inputColumn.colNo, typeCode);
+      }
+   }
+
+   protected override void myOnLastSubscriber() =>
+      Disposable.dispose(ref sub);
+
+   static class ExpressionBuilder {
+      public static Func<IBlueRow, IBlueRow, int?> Int32DiffExpression(int colIndex) {
+         var colIndexParam = Expression.Constant(colIndex, typeof(int));
+         var rowParam = Expression.Parameter(typeof(IBlueRow), "row");
+         var getValMethod = typeof(ExpressionBuilder).GetMethod(nameof(getInt32Value))
+            ?? throw new InvalidOperationException("Failed to find int32 value getter via reflection");
+         var getVal = Expression.Lambda<Func<IBlueRow, int?>>(Expression.Call(null, getValMethod, rowParam, colIndexParam), rowParam).Compile();
+         return ComputeDiff;
+
+         int? ComputeDiff(IBlueRow leftRow, IBlueRow rightRow) {
+            var leftVal = getVal(leftRow);
+            var rightVal = getVal(rightRow);
+            if (leftVal.HasValue && rightVal.HasValue) {
+               return leftVal.Value - rightVal.Value;
+            }
+            return null;
+         }
+      }
+
+      static int? getInt32Value(IBlueRow row, int colIndex) {
+         var obj = row[colIndex];
+         return obj as int?;
+      }
+   }
+}

src/DataStreamSuite/DataStream/BlueOrb/DiffTable.cs

@@ -0,0 +1,101 @@
+namespace bts.BlueOrb;
+
+using BCG;
+
+using bts.collections;
+
+using Bts.DataStreamCore;
+
+using Spark.Core.Collections;
+
+using System;
+using System.Collections.Generic;
+using System.Linq;
+
+public class DiffTable : IBlueRowCollection {
+   readonly IBlueRowCollection input;
+   readonly int period;
+   readonly IList<object?>[] columns;
+
+   public DiffTable(IBlueRowCollection input, int period) {
+      this.input = input;
+      this.period = period;
+      var colInfos = input.schema.columnInfo.OrderBy(c => c.colNo).ToArray();
+      if (colInfos.Length > 0) {
+         Log.assert(colInfos[0].colNo == 0 && colInfos[^1].colNo == colInfos.Length - 1);
+      }
+      columns = colInfos.WrapSelector(c => GetColumn(c.colNo), null).ToArray();
+   }
+
+   public IBlueRow this[int id] =>
+      GetRow(input.IndexOf(input[id]));
+
+   public IBlueTableSchema schema => input.schema;
+
+   public int Count => input.Count;
+
+   public bool containsID(int id) =>
+      input.containsID(id);
+
+   public IList<object?> GetColumn(int colIndex) {
+      var count = input.Count;
+      var colInfo = input.schema.columnInfo[colIndex];
+      if (colInfo.colName == "id") {
+         return new StepList<int>(0, 1, count).as_mutable().WrapSelector(GetId, null);
+      }
+      var typeCode = colInfo.type_code();
+      return new StepList<int>(0, 1, count).as_mutable().WrapSelector(GetValue, null);
+
+      object? GetId(int rowIndex) {
+         var row = input.GetRow(rowIndex);
+         return row.Id();
+      }
+
+      object? GetValue(int rowIndex) {
+         var rightIndex = rowIndex - period;
+         if (rowIndex < 0 || rowIndex >= count || rightIndex < 0 || rightIndex >= count) {
+            return null;
+         }
+         var leftRow = input.GetRow(rowIndex);
+         var rightRow = input.GetRow(rightIndex);
+         return GetDiff(leftRow[colIndex], rightRow[colIndex], typeCode);
+      }
+   }
+
+   public IEnumerator<IBlueRow> GetEnumerator() {
+      var count = input.Count;
+      for (int i = 0; i < count; i++) {
+         yield return GetRow(i);
+      }
+   }
+
+   public IBlueRow GetRow(int index) =>
+      new BlueRowColumnWrapper(input.schema, index, columns);
+
+   public int IndexOf(IBlueRow item) {
+      int id = item.Id();
+      var inputRow = input[id];
+      int idx = input.IndexOf(inputRow);
+      if (idx != -1) {
+         var row = GetRow(idx);
+         if (BlueRowEqualityComparer.Equals(item, row)) {
+            return idx;
+         }
+      }
+      return -1;
+   }
+
+   static object? GetDiff(object? leftVal, object? rightVal, TypeCode colType) {
+      if (leftVal is null || rightVal is null) {
+         return null;
+      }
+      return colType switch {
+         TypeCode.Int32 => (int)leftVal - (int)rightVal,
+         TypeCode.Int64 => (long)leftVal - (long)rightVal,
+         TypeCode.Double => (double)leftVal - (double)rightVal,
+         TypeCode.String => ((string)leftVal).Replace((string)rightVal, string.Empty),
+         TypeCode.DateTime => new DateTime(((DateTime)leftVal).Ticks - ((DateTime)rightVal).Ticks),
+         _ => null,
+      };
+   }
+}

src/DataStreamSuite/DataStream/BlueOrb/FeatherReaderStream.cs

@@ -0,0 +1,506 @@
+namespace bts.BlueOrb;
+
+using Apache.Arrow;
+using Apache.Arrow.Ipc;
+using Apache.Arrow.Types;
+
+using BCG;
+
+using bts.collections;
+
+using Bts.DataStreamCore;
+
+using Spark.Core.Disposables;
+using Spark.Core.Threading;
+
+using System;
+using System.Collections;
+using System.Collections.Generic;
+using System.Collections.Immutable;
+using System.IO;
+using System.Linq;
+using System.Threading.Tasks;
+
+[DataStreamUserRespresentation("featherReader")]
+public class FeatherReaderStream : ViewStream {
+   readonly string file;
+   FileStream fileStream;
+   // Let the table leak in case some dangling references linger around after
+   // myOnLastSubscriber is called. This should be fine since it disposes itself
+   // in its finalizer.
+   FeatherTable? table;
+
+   public FeatherReaderStream(string file) =>
+      this.file = file;
+
+   protected override void myOnFirstSubscriber() {
+      if (!Path.Exists(file)) {
+         throw new InvalidOperationException("Feather file does not exist");
+      }
+      fileStream = File.OpenRead(file);
+      table = new(new ArrowStreamReader(fileStream));
+      myFireEvent(new DataPacket(table, table, new BlueTableImmutable(table.schema)));
+   }
+
+   interface IFeatherColumn {
+      int Count { get; }
+
+      object? GetValue(int i);
+   }
+
+   interface IFeatherColumn<T> : IFeatherColumn {
+      new T? GetValue(int i);
+
+      object? IFeatherColumn.GetValue(int i) =>
+         GetValue(i);
+   }
+
+   class FeatherColumn : IFeatherColumn {
+      readonly IArrowArray array;
+
+      public FeatherColumn(IArrowArray array) {
+         this.array = array;
+      }
+
+      public int Count => array.Length;
+
+      public static IFeatherColumn Create(IArrowArray array) =>
+         array switch {
+            Int32Array a => new FeatherInt32Column(a),
+            Int64Array a => new FeatherInt64Column(a),
+            DoubleArray a => new FeatherFloat64Column(a),
+            StringArray a => new FeatherStringColumn(a),
+            BooleanArray a => new FeatherBooleanColumn(a),
+            Date64Array a => new FeatherDateTimeColumn(a),
+            Time64Array a => new FeatherTimeStampColumn(a),
+            _ => throw new NotSupportedException($"Unsupported type: {array.GetType()}")
+         };
+
+      public object? GetValue(int i) =>
+         array switch {
+            Int32Array a => a.GetValue(i),
+            Int64Array a => a.GetValue(i),
+            DoubleArray a => a.GetValue(i),
+            StringArray a => a.GetString(i),
+            BooleanArray a => a.GetValue(i),
+            Date64Array a => a.GetValue(i),
+            Time64Array a => a.GetValue(i),
+            _ => throw new NotSupportedException($"Unsupported type: {array.GetType()}")
+         };
+   }
+
+   class FeatherInt32Column : IFeatherColumn<int?> {
+      readonly Int32Array array;
+
+      public FeatherInt32Column(Int32Array array) =>
+         this.array = array;
+
+      public int Count => array.Length;
+
+      public int? GetValue(int i) => array.GetValue(i);
+   }
+
+   class FeatherInt64Column : IFeatherColumn<long?> {
+      readonly Int64Array array;
+
+      public FeatherInt64Column(Int64Array array) =>
+         this.array = array;
+
+      public int Count => array.Length;
+
+      public long? GetValue(int i) => array.GetValue(i);
+   }
+
+   class FeatherFloat64Column : IFeatherColumn<double?> {
+      readonly DoubleArray array;
+
+      public FeatherFloat64Column(DoubleArray array) =>
+         this.array = array;
+
+      public int Count => array.Length;
+
+      public double? GetValue(int i) => array.GetValue(i);
+   }
+
+   class FeatherFloat32Column : IFeatherColumn<float?> {
+      readonly FloatArray array;
+
+      public FeatherFloat32Column(FloatArray array) =>
+         this.array = array;
+
+      public int Count => array.Length;
+
+      public float? GetValue(int i) => array.GetValue(i);
+   }
+
+   class FeatherStringColumn : IFeatherColumn<string?> {
+      readonly StringArray array;
+
+      public FeatherStringColumn(StringArray array) =>
+         this.array = array;
+
+      public int Count => array.Length;
+
+      public string? GetValue(int i) => array.GetString(i);
+   }
+
+   class FeatherBooleanColumn : IFeatherColumn<bool?> {
+      readonly BooleanArray array;
+
+      public FeatherBooleanColumn(BooleanArray array) =>
+         this.array = array;
+
+      public int Count => array.Length;
+
+      public bool? GetValue(int i) => array.GetValue(i);
+   }
+
+   class FeatherDateTimeColumn : IFeatherColumn<DateTime?> {
+      readonly Date64Array array;
+
+      public FeatherDateTimeColumn(Date64Array array) =>
+         this.array = array;
+
+      public int Count => array.Length;
+
+      public DateTime? GetValue(int i) {
+         var nanos = array.GetValue(i);
+         if (!nanos.HasValue) {
+            return null;
+         }
+         return new(nanos.Value / 100);
+      }
+   }
+
+   class FeatherTimeStampColumn : IFeatherColumn<DateTime?> {
+      readonly Time64Array array;
+
+      public FeatherTimeStampColumn(Time64Array array) =>
+         this.array = array;
+
+      public int Count => array.Length;
+
+      public DateTime? GetValue(int i) {
+         var nanos = array.GetValue(i);
+         if (!nanos.HasValue) {
+            return null;
+         }
+         return new(nanos.Value / 100);
+      }
+   }
+
+   class FeatherBlueRow : IBlueRow {
+      readonly int id;
+      readonly int rowIndex;
+      readonly IFeatherColumn[] data;
+
+      public FeatherBlueRow(int id, int rowIndex, IFeatherColumn[] data, IBlueTableSchema schema) {
+         this.id = id;
+         this.rowIndex = rowIndex;
+         this.data = data;
+         Schema = schema;
+      }
+
+      public object? this[int index] =>
+         index == 0
+            ? id
+            : data[index - 1].GetValue(rowIndex);
+
+      public IBlueTableSchema Schema { get; }
+      IBlueTableSchema IBlueRow.schema => Schema;
+   }
+
+   class FeatherColumnDataList : IReadOnlyList<object?> {
+      // Represents 1 column split into batches (hence it being an array)
+      readonly IReadOnlyList<IFeatherColumn> columns;
+      int? count;
+
+      public FeatherColumnDataList(IReadOnlyList<IFeatherColumn> columns) {
+         this.columns = columns;
+      }
+
+      public object? this[int index] {
+         get {
+            for (int i = 0; i < columns.Count; i++) {
+               var col = columns[i];
+               if (index < col.Count) {
+                  return col.GetValue(index);
+               }
+               index -= col.Count;
+            }
+            throw new IndexOutOfRangeException();
+         }
+      }
+
+      public int Count =>
+         count ??= columns.Sum(c => c.Count);
+
+      public IEnumerator<object?> GetEnumerator() {
+         for (int i = 0; i < columns.Count; i++) {
+            var col = columns[i];
+            for (int j = 0; j < col.Count; j++) {
+               yield return col.GetValue(i);
+            }
+         }
+      }
+
+      IEnumerator IEnumerable.GetEnumerator() =>
+         GetEnumerator();
+   }
+
+   class FeatherTable : IImmutableRowCollection, IDisposable {
+      ImmutableList<(RecordBatch Records, IFeatherColumn[] Columns)> recordBatches = ImmutableList<(RecordBatch Records, IFeatherColumn[] Columns)>.Empty;
+      readonly IReadOnlyDictionary<string, Field> fields;
+      readonly AtomicGate gate = new();
+      int count;
+      ArrowStreamReader? reader;
+
+      public FeatherTable(ArrowStreamReader reader) {
+         this.reader = reader;
+         if (reader.Schema is null) {
+            var batch = reader.ReadNextRecordBatch();
+            Log.notnull(batch);
+            Log.notnull(reader.Schema);
+            var fcols = new IFeatherColumn[batch.ColumnCount];
+            for (int i = 0; i < fcols.Length; i++) {
+               fcols[i] = FeatherColumn.Create(batch.Column(i));
+            }
+            recordBatches = recordBatches.Add((batch, fcols));
+            count = batch.Length;
+         }
+         fields = reader.Schema.Fields;
+         var columns = new ColumnInfo[fields.Count + 1];
+         columns[0] = new("id", 0, typeof(int));
+         for (int i = 0; i < fields.Count; i++) {
+            var f = reader.Schema.GetFieldByIndex(i);
+            Log.assert(f.Name != "id");
+            columns[i + 1] = new(f.Name, i + 1, getFieldType(f));
+         }
+         schema = new BlueTableSchema("FeatherReader", columns);
+      }
+
+      ~FeatherTable() =>
+         Disposable.DisposeThreadSafe(ref reader);
+
+      public IBlueTableSchema schema { get; }
+
+      public int Count {
+         get {
+            if (reader is not null) {
+               readBatches().Wait();
+               Log.isnull(reader);
+            }
+            return count;
+         }
+      }
+
+      public IBlueRow this[int id] => GetRow(id - 1);
+
+      public bool containsID(int id) {
+         while (id > count && reader is not null) {
+            readBatches(1).Wait();
+         }
+         return id <= count;
+      }
+
+      public int IndexOf(IBlueRow item) {
+         int id = item.Id();
+         while (id > count && reader is not null) {
+            readBatches(1).Wait();
+         }
+         return id <= count && BlueRowEqualityComparer.Equals(GetRow(id - 1), item)
+            ? id - 1
+            : -1;
+      }
+
+      public IBlueRow GetRow(int index) {
+         if (index < 0) {
+            throw new IndexOutOfRangeException();
+         }
+         if (count <= index) {
+            readRows(index + 1).Wait();
+         }
+         int id = index + 1;
+         foreach (var (records, columns) in recordBatches) {
+            if (index < records.Length) {
+               return new FeatherBlueRow(id, index, columns, schema);
+            }
+            index -= records.Length;
+         }
+         throw new IndexOutOfRangeException();
+      }
+
+      public IList<object?> GetColumn(int index) =>
+         new FeatherColumnDataList(new MyColList(this, index)).as_mutable();
+
+      public IEnumerator<IBlueRow> GetEnumerator() {
+         int i = 0;
+         foreach (var (records, columns) in recordBatches) {
+            yield return new FeatherBlueRow(i + 1, i, columns, schema);
+            i++;
+         }
+         while (reader is not null) {
+            var task = readBatches(1);
+            task.Wait();
+            if (!task.Result) {
+               break;
+            }
+            yield return new FeatherBlueRow(i + 1, i, recordBatches[i].Columns, schema);
+            i++;
+         }
+      }
+
+      public void Dispose() =>
+         Disposable.DisposeThreadSafe(ref reader);
+
+      static Type getFieldType(Field f) =>
+         f.DataType.TypeId switch {
+            ArrowTypeId.Null => throw new NotImplementedException(),
+            ArrowTypeId.Boolean => typeof(bool),
+            ArrowTypeId.UInt8 => typeof(byte),
+            ArrowTypeId.Int8 => typeof(sbyte),
+            ArrowTypeId.UInt16 => typeof(ushort),
+            ArrowTypeId.Int16 => typeof(short),
+            ArrowTypeId.UInt32 => typeof(uint),
+            ArrowTypeId.Int32 => typeof(int),
+            ArrowTypeId.UInt64 => typeof(ulong),
+            ArrowTypeId.Int64 => typeof(long),
+            ArrowTypeId.HalfFloat => throw new NotImplementedException(),
+            ArrowTypeId.Float => typeof(float),
+            ArrowTypeId.Double => typeof(double),
+            ArrowTypeId.String => typeof(string),
+            ArrowTypeId.Binary => throw new NotImplementedException(),
+            ArrowTypeId.FixedSizedBinary => throw new NotImplementedException(),
+            ArrowTypeId.Date32 => throw new NotImplementedException(),
+            ArrowTypeId.Date64 => throw new NotImplementedException(),
+            ArrowTypeId.Timestamp => throw new NotImplementedException(),
+            ArrowTypeId.Time32 => throw new NotImplementedException(),
+            ArrowTypeId.Time64 => throw new NotImplementedException(),
+            ArrowTypeId.Interval => throw new NotImplementedException(),
+            ArrowTypeId.Decimal128 => throw new NotImplementedException(),
+            ArrowTypeId.Decimal256 => throw new NotImplementedException(),
+            ArrowTypeId.List => throw new NotImplementedException(),
+            ArrowTypeId.Struct => throw new NotImplementedException(),
+            ArrowTypeId.Union => throw new NotImplementedException(),
+            ArrowTypeId.Dictionary => throw new NotImplementedException(),
+            ArrowTypeId.Map => throw new NotImplementedException(),
+            _ => throw new ArgumentException("Unsupported field type", nameof(f)),
+         };
+
+      async Task<bool> readRows(int minCount) {
+         if (reader is null) {
+            return count >= minCount;
+         }
+         int n = 0;
+         using (gate.BeginTransaction()) {
+            if (reader is not null) {
+               int origCount = count;
+               while ((count + n) < minCount) {
+                  Log.assert(count == origCount);
+                  var batch = await reader.ReadNextRecordBatchAsync();
+                  if (batch is null) {
+                     Disposable.dispose(ref reader);
+                     break;
+                  }
+                  var columns = new IFeatherColumn[batch.ColumnCount];
+                  for (int i = 0; i < columns.Length; i++) {
+                     columns[i] = FeatherColumn.Create(batch.Column(i));
+                  }
+                  recordBatches.Add((batch, columns));
+                  n += batch.Length;
+               }
+               count += n;
+            }
+         }
+         return count >= minCount;
+      }
+
+      async Task<bool> readBatches(int? count = null) {
+         if (reader is null) {
+            return !count.HasValue || count.Value == 0;
+         }
+         if (count.HasValue && count.Value <= recordBatches.Count) {
+            return true;
+         }
+         int n = 0;
+         using (gate.BeginTransaction()) {
+            if (reader is not null) {
+               int origCount = this.count;
+               var recordBatchesBd = recordBatches.ToBuilder();
+               while (!count.HasValue || count-- > recordBatchesBd.Count) {
+                  Log.assert(count == origCount);
+                  var batch = await reader.ReadNextRecordBatchAsync();
+                  if (batch is null) {
+                     Disposable.dispose(ref reader);
+                     break;
+                  }
+                  var columns = new IFeatherColumn[batch.ColumnCount];
+                  for (int i = 0; i < columns.Length; i++) {
+                     columns[i] = FeatherColumn.Create(batch.Column(i));
+                  }
+                  recordBatchesBd.Add((batch, columns));
+                  n += batch.Length;
+               }
+               recordBatches = recordBatchesBd.ToImmutable();
+               this.count += n;
+            }
+         }
+         return !count.HasValue || count.Value <= recordBatches.Count;
+      }
+
+      class MyColList : IReadOnlyList<IFeatherColumn> {
+         readonly FeatherTable owner;
+         readonly int columnIndex;
+
+         public MyColList(FeatherTable owner, int columnIndex) {
+            this.owner = owner;
+            this.columnIndex = columnIndex;
+         }
+
+         public IFeatherColumn this[int index] {
+            get {
+               if (index < 0) {
+                  throw new IndexOutOfRangeException();
+               }
+               if (index >= owner.recordBatches.Count) {
+                  var task = owner.readBatches(index + 1);
+                  task.Wait();
+                  if (!task.Result) {
+                     throw new IndexOutOfRangeException();
+                  }
+               }
+               return owner.recordBatches[index].Columns[columnIndex];
+            }
+         }
+
+         public int Count {
+            get {
+               if (owner.reader is not null) {
+                  owner.readBatches().Wait();
+               }
+               return owner.recordBatches.Count;
+            }
+         }
+
+         public IEnumerator<IFeatherColumn> GetEnumerator() {
+            int i = 0;
+            foreach (var batch in owner.recordBatches) {
+               yield return batch.Columns[columnIndex];
+               i++;
+            }
+            while (owner.reader is not null) {
+               var task = owner.readBatches(1);
+               task.Wait();
+               if (!task.Result) {
+                  break;
+               }
+               yield return owner.recordBatches[i].Columns[columnIndex];
+               i++;
+            }
+         }
+
+         IEnumerator IEnumerable.GetEnumerator() =>
+            GetEnumerator();
+      }
+   }
+}

src/DataStreamSuite/DataStream/BlueOrb/FilterStream.cs

@@ -0,0 +1,159 @@
+namespace bts.BlueOrb {
+   using BCG;
+
+   using Bts.DataStreamCore;
+
+   using mbts.utils;
+
+   using System;
+   using System.Text.RegularExpressions;
+
+   [DataStreamUserRespresentation("FilterStream")]
+   public class FilterStream : ViewStream {
+      public class RegExpFilter : IStreamFilter {
+         private Regex[] myRegExps;
+         private string[] myCols;
+
+         public RegExpFilter(string col, string regExp)
+            : this(new string[] { col }, new string[] { regExp }) { }
+
+         public RegExpFilter(string[] cols, string[] regExps) {
+            myCols = cols;
+            myRegExps = new Regex[regExps.Length];
+            for (int i = 0; i < regExps.Length; i++)
+               myRegExps[i] = new Regex(regExps[i]);
+         }
+
+         #region IStreamFilter Members
+         public bool filter(IBlueRow dr) {
+            bool rv = true;
+            for (int i = 0; i < myCols.Length; i++) {
+               rv &= myRegExps[i].IsMatch(dr[myCols[i]] is DBNull ? "" : dr[myCols[i]]?.ToString() ?? string.Empty);
+               if (!rv)
+                  return rv;
+            }
+            return rv;
+         }
+         #endregion
+      }
+      BlueTableImmutable? myLastFiredRows;
+      IBlueRowCollection? myLastReceivedRows;
+      private IStreamFilter myFilter;
+      private ViewStream myParentStream;
+      private string myFilterString;
+      private bool myUseNewFilter = false;// Configuration.get<DataStreamConfig>().UseNewFilter;
+
+      #region constructors
+      public delegate bool OnFilter(IBlueRow r);
+      private class OnFilterWrapper : IStreamFilter {
+         public OnFilterWrapper(OnFilter f) { this.f = f; }
+
+         public bool filter(IBlueRow dr) { return f(dr); }
+
+         private OnFilter f;
+      }
+      public FilterStream(ViewStream vs, OnFilter f)
+         : this(vs, new OnFilterWrapper(f)) { }
+
+
+      public FilterStream(ViewStream vs, IStreamFilter filter) {
+         myParentStream = vs;
+         myFilter = filter;
+         myFilterString = filter.ToString() ?? string.Empty;
+         if (filter is IDynamicStreamFilter dynFilter) {
+            dynFilter.onFilterChange += new FilterChangeDelegate(myHandleFilterChange);
+         }
+      }
+
+      public FilterStream(ViewStream vs, string filterExpression)
+         : this(vs, new BooleanExpressionFilter(filterExpression)) {
+         myFilterString = filterExpression;
+      }
+
+      public FilterStream(ViewStream vs, string col, string regExp)
+         : this(vs, new RegExpFilter(col, regExp)) {
+         myFilterString = col + " , " + regExp;
+      }
+
+      public FilterStream(ViewStream vs)
+         : this(vs, new BinaryFilter(true)) {
+         myFilterString = true.ToString();
+      }
+      #endregion
+
+      #region overridden methods
+      protected override void myOnFirstSubscriber() =>
+         myParentStream.onChange += myHandleCallbacks;
+
+      protected override void myOnLastSubscriber() {
+         myLastFiredRows = null;
+         myParentStream.onChange -= myHandleCallbacks;
+      }
+
+      protected override void myFireEvent(object o) {
+         var dp = ExceptionAssert.Argument.Is<IDataPacket>(o);
+         if (myLastValue == null || dp.addedRows.Count + dp.deletedRows.Count > 0)
+            base.myFireEvent(o);
+      }
+      #endregion
+
+      private void myHandleCallbacks(object newValue) {
+         var arrivedPacket = ExceptionAssert.Argument.Is<IDataPacket>(newValue);
+         myLastReceivedRows = arrivedPacket.newRows;
+         var bd = (myLastFiredRows ?? new BlueTableImmutable(arrivedPacket.Schema)).ToDataPacketBuilder();
+         foreach (IBlueRow r in arrivedPacket.deletedRows) {
+            // Don't bother evaluating the filter, just try to remove the row. The latter is
+            // cheaper than the former (it's a noop if this row wasn't in the filtered table).
+            bd.Remove(r.Id());
+         }
+         foreach (IBlueRow r in arrivedPacket.newRows) {
+            if (myFilter.filter(r)) {
+               bd.AddOrUpdate(r);
+            }
+         }
+         var dpOut = bd.ToDataPacket(out myLastFiredRows);
+         myFireEvent(dpOut);
+      }
+
+      public IStreamFilter filter {
+         set {
+            {
+               myFilter = value;
+               if (numberOfSubscribers > 0) {
+                  if (myLastReceivedRows != null) {
+                     var bd = (myLastFiredRows ?? new BlueTableImmutable(myLastReceivedRows.schema)).ToDataPacketBuilder();
+                     foreach (IBlueRow r in myLastReceivedRows) {
+                        int id = r.Id();
+                        bool includeNow = myFilter.filter(r);
+                        bool includeBefore = myLastFiredRows?.containsID(id) ?? false;
+                        if (includeNow == includeBefore) {
+                           continue;
+                        }
+                        if (includeNow) {
+                           bd.AddOrUpdate(r);
+                        } else {
+                           bd.Remove(id);
+                        }
+                     }
+                     var dpOut = bd.ToDataPacket(out myLastFiredRows);
+                     myFireEvent(dpOut);
+                  }
+               }
+            }
+         }
+         get {            
+               return myFilter;
+         }
+      }
+
+      //public string filterString { set { filter = new BooleanExpressionFilter(value); } }
+
+      public override string ToString() {
+         return GetType().ToString() + " " + myFilterString;
+      }
+
+      private void myHandleFilterChange() {
+         filter = myFilter;
+      }
+   }
+}

src/DataStreamSuite/DataStream/BlueOrb/FilterTable.cs

@@ -0,0 +1,72 @@
+namespace bts.BlueOrb;
+
+using BCG;
+
+using Bts.DataStreamCore;
+
+using System;
+using System.Collections.Generic;
+
+public class FilterTable : IBlueRowCollection {
+   readonly IBlueRowCollection source;
+   readonly HashSet<int> excludedIndices = new();
+   readonly Dictionary<int, (int Index, int NumSkips, bool Included, IBlueRow Row)> rowMap = new();
+   readonly List<int> filteredIndexToId = new();
+
+   public FilterTable(IBlueRowCollection source, Func<IBlueRow, bool> filter) {
+      this.source = source;
+      int idx = 0;
+      int numSkips = 0;
+      foreach (var r in source) {
+         int id = r.Id();
+         bool included = filter(r);
+         rowMap.Add(id, (idx, numSkips, included, r));
+         if (!included) {
+            excludedIndices.Add(idx);
+            numSkips++;
+         } else {
+            filteredIndexToId.Add(id);
+         }
+         idx++;
+      }
+   }
+
+   public IBlueRow this[int id] {
+      get {
+         var r = rowMap[id];
+         return r.Included ? r.Row : throw new KeyNotFoundException();
+      }
+   }
+
+   public IBlueTableSchema schema => source.schema;
+
+   public int Count => filteredIndexToId.Count;
+
+   public bool containsID(int id) =>
+      rowMap.TryGetValue(id, out var r) && r.Included;
+
+   public IEnumerator<IBlueRow> GetEnumerator() {
+      int idx = 0;
+      foreach (var r in source) {
+         if (!excludedIndices.Contains(idx)) {
+            yield return r;
+         }
+         idx++;
+      }
+   }
+
+   public IBlueRow GetRow(int index) {
+      if (index < 0 || index >= source.Count) {
+         throw new IndexOutOfRangeException();
+      }
+      var id = filteredIndexToId[index];
+      return rowMap[id].Row;
+   }
+
+   public int IndexOf(IBlueRow item) =>
+      rowMap.TryGetValue(item.Id(), out var r)
+         && r.Included
+         && BlueRowEqualityComparer.Equals(r.Row, item)
+            ? r.Index - r.NumSkips
+            : -1;
+}

src/DataStreamSuite/DataStream/BlueOrb/GroupByApplyStream.cs

@@ -0,0 +1,222 @@
+namespace Bts.SharedStreams.StreamFactories;
+
+using BCG;
+
+using bts;
+using bts.api.core;
+using bts.BlueOrb;
+
+using Bts.DataStream;
+using Bts.DataStreamCore;
+
+using System;
+using System.Collections.Generic;
+using System.Collections.Immutable;
+using System.Linq;
+using System.Reactive.Linq;
+using System.Threading;
+
+public class GroupByApplyStream : ViewStream {
+   readonly ViewStream inputStream;
+   readonly IFunctionEvaluator groupBy;
+   readonly IFunctionEvaluator apply;
+
+   public GroupByApplyStream(ViewStream inputStream, IFunctionEvaluator groupBy, IFunctionEvaluator apply) {
+      this.inputStream = inputStream;
+      this.groupBy = groupBy;
+      this.apply = apply;
+   }
+
+   public static IObservable<TableUpdate<IBlueRow, int>> CreateGroupByApplyTable(ViewStream inputStream, IFunctionEvaluator groupBy, IFunctionEvaluator apply) =>
+      CreateApplyTable(CreateGroupByTable(inputStream, groupBy), apply);
+
+   public static IObservable<TableUpdate<IBlueRow, int>> CreateApplyTable<TGroupKey>(IObservable<TableUpdate<TableUpdateGrouping<IBlueRow, int, TGroupKey>, TGroupKey>> inputStream, Func<TGroupKey, IBlueRowCollection, IBlueRowCollection?> apply) where TGroupKey : IEquatable<TGroupKey> {
+      var groupTables = ImmutableDictionary<TGroupKey, GroupTableBuilder>.Empty;
+      var inputToOutputIds = ImmutableDictionary<int, int>.Empty;
+      var outputToInputIds = ImmutableDictionary<int, int>.Empty;
+      int idCounter = 0;
+
+      return Observable.Create<TableUpdate<IBlueRow, int>>(Subscribe);
+
+      int GetNextId() =>
+         Interlocked.Increment(ref idCounter);
+
+      IDisposable Subscribe(IObserver<TableUpdate<IBlueRow, int>> observer) {
+         int hashCounter = 0;
+         var hashIndex = new Dictionary<TGroupKey, int>();
+         var tableOut = TableUpdate<IBlueRow, int>.mk_empty(r => r.Id());
+         return inputStream.Subscribe(OnNext, OnError, OnCompleted);
+
+         void OnNext(TableUpdate<TableUpdateGrouping<IBlueRow, int, TGroupKey>, TGroupKey> upd) {
+            var bd = tableOut.to_builder();
+            foreach (var group in upd.removed) {
+               var hash = hashIndex[group.group_key];
+               bd.remove(hash);
+            }
+            var groupTablesBd = groupTables.ToBuilder();
+            foreach (var group in upd.added.Concat(upd.changed)) {
+               Log.assert(group.table.state.Count > 0);
+               var hash = hashIndex.GetOrAdd(group.group_key, _ => hashCounter++);
+               var groupRowCollection = new TableUpdateGroupingRowCollection<TGroupKey>(group, group.table.state.Values.First().schema);
+               var applyArgs = new (string?, int?, object?)[] { ("rows", 0, groupRowCollection) };
+               var resRows = apply(group.group_key, groupRowCollection);
+               if (resRows is null) {
+                  bd.remove(hash);
+                  continue;
+               }
+               UpdateGroupTable(group.group_key, resRows);
+            }
+            groupTables = groupTablesBd.ToImmutable();
+            tableOut = bd.to_table();
+            observer.OnNext(tableOut);
+
+            void UpdateGroupTable(TGroupKey groupKey, IBlueRowCollection groupRows) {
+               var indexedRows = groupTablesBd.GetOrAdd(groupKey, _ => new(GetNextId)).Reset(groupRows);
+               foreach (var r in indexedRows) {
+                  bd.add_or_update(r);
+               }
+            }
+         }
+      }
+
+      void OnError(Exception ex) {
+         Log.DataFlow.error($$"""[stream_err.json]{ "typ": "{{nameof(GroupByApplyStream)}}", "inpTyp": {{inputStream.GetType()}}, "err": "{{ex}}" }""");
+      }
+
+      void OnCompleted() {
+         Log.DataFlow.warn($$"""[stream_fin.json]{ "typ": "{{nameof(GroupByApplyStream)}}", "inpTyp": "{{inputStream.GetType()}}" }""");
+      }
+   }
+
+   public static IObservable<TableUpdate<IBlueRow, int>> CreateApplyTable<TGroupKey>(IObservable<TableUpdate<TableUpdateGrouping<IBlueRow, int, TGroupKey>, TGroupKey>> inputStream, IFunctionEvaluator apply) where TGroupKey : IEquatable<TGroupKey> {
+      var groupTables = ImmutableDictionary<TGroupKey, GroupTableBuilder>.Empty;
+      var inputToOutputIds = ImmutableDictionary<int, int>.Empty;
+      var outputToInputIds = ImmutableDictionary<int, int>.Empty;
+      int idCounter = 0;
+
+      return Observable.Create<TableUpdate<IBlueRow, int>>(Subscribe);
+
+      int GetNextId() =>
+         Interlocked.Increment(ref idCounter);
+
+      IDisposable Subscribe(IObserver<TableUpdate<IBlueRow, int>> observer) {
+         int hashCounter = 0;
+         var hashIndex = new Dictionary<TGroupKey, int>();
+         var tableOut = TableUpdate<IBlueRow, int>.mk_empty(r => r.Id());
+         return inputStream.Subscribe(OnNext, OnError, OnCompleted);
+
+         void OnNext(TableUpdate<TableUpdateGrouping<IBlueRow, int, TGroupKey>, TGroupKey> upd) {
+            var bd = tableOut.to_builder();
+            foreach (var group in upd.removed) {
+               var hash = hashIndex[group.group_key];
+               bd.remove(hash);
+            }
+            var groupTablesBd = groupTables.ToBuilder();
+            foreach (var group in upd.added.Concat(upd.changed)) {
+               Log.assert(group.table.state.Count > 0);
+               var hash = hashIndex.GetOrAdd(group.group_key, _ => hashCounter++);
+               var groupRowCollection = new TableUpdateGroupingRowCollection<TGroupKey>(group, group.table.state.Values.First().schema);
+               var applyArgs = new StreamArgument[] { ("rows", 0, groupRowCollection) };
+               var res = apply.Evaluate(string.Empty, applyArgs);
+               if (res is IBlueRow resSingleRow) {
+                  UpdateGroupTable(group.group_key, new SingleRowTable(resSingleRow));
+               } else if (res is IBlueRowCollection resRows) {
+                  UpdateGroupTable(group.group_key, resRows);
+               } else {
+                  throw new InvalidOperationException("Invalid result returned by 'Apply' fn");
+               }
+            }
+            groupTables = groupTablesBd.ToImmutable();
+            tableOut = bd.to_table();
+            observer.OnNext(tableOut);
+
+            void UpdateGroupTable(TGroupKey groupKey, IBlueRowCollection groupRows) {
+               var indexedRows = groupTablesBd.GetOrAdd(groupKey, _ => new(GetNextId)).Reset(groupRows);
+               foreach (var r in indexedRows) {
+                  bd.add_or_update(r);
+               }
+            }
+         }
+      }
+
+      void OnError(Exception ex) {
+         Log.DataFlow.error($$"""[stream_err.json]{ "typ": "{{nameof(GroupByApplyStream)}}", "inpTyp": {{inputStream.GetType()}}, "err": "{{ex}}" }""");
+      }
+
+      void OnCompleted() {
+         Log.DataFlow.warn($$"""[stream_fin.json]{ "typ": "{{nameof(GroupByApplyStream)}}", "inpTyp": "{{inputStream.GetType()}}" }""");
+      }
+   }
+
+   public static IObservable<TableUpdate<TableUpdateGrouping<IBlueRow, int, EquatableKeyWrapper>, EquatableKeyWrapper>> CreateGroupByTable(ViewStream inputStream, IFunctionEvaluator groupBy) {
+      return Observable.Create<TableUpdate<TableUpdateGrouping<IBlueRow, int, EquatableKeyWrapper>, EquatableKeyWrapper>>(Subscribe);
+
+      IDisposable Subscribe(IObserver<TableUpdate<TableUpdateGrouping<IBlueRow, int, EquatableKeyWrapper>, EquatableKeyWrapper>> observer) {
+         TableUpdate<TableUpdateGrouping<IBlueRow, int, EquatableKeyWrapper>, EquatableKeyWrapper> tableOut = TableUpdate<TableUpdateGrouping<IBlueRow, int, EquatableKeyWrapper>, EquatableKeyWrapper>.mk_empty(r => r.group_key);
+         return inputStream.AsObservable().AsTables().GroupTable(r => r.Id(), GroupKey).Subscribe(observer);
+
+         EquatableKeyWrapper GroupKey(IBlueRow r) {
+            var groupByArgs = new StreamArgument[] { ("row", 0, r) };
+            var groupKey = new EquatableKeyWrapper(groupBy.Evaluate(string.Empty, groupByArgs) ?? DBNull.Value);
+            return groupKey;
+         }
+      }
+   }
+
+   public static IObservable<TableUpdate<TableUpdateGrouping<IBlueRow, int, EquatableKeyArrayWrapper>, EquatableKeyArrayWrapper>> CreateGroupByTable(ViewStream inputStream, string[] columns, bool useColumnIndices = true) {
+      Func<IBlueRow, EquatableKeyArrayWrapper> groupBy;
+      if (!useColumnIndices) {
+         groupBy = GroupKeyByName;
+
+         EquatableKeyArrayWrapper GroupKeyByName(IBlueRow r) =>
+            new(Array.ConvertAll(columns, c => r[c]));
+      } else {
+         int[]? columnIndices = null;
+         groupBy = GroupKeyByIndex;
+
+         EquatableKeyArrayWrapper GroupKeyByIndex(IBlueRow r) {
+            columnIndices ??= r.schema.ResolveIndices(columns);
+            return new(Array.ConvertAll(columnIndices, c => r[c]));
+         }
+      }
+      return CreateGroupByTable(inputStream, groupBy);
+   }
+
+   public static IObservable<TableUpdate<TableUpdateGrouping<IBlueRow, int, TKey>, TKey>> CreateGroupByTable<TKey>(ViewStream inputStream, Func<IBlueRow, TKey> groupBy) where TKey : IEquatable<TKey> {
+      return Observable.Create<TableUpdate<TableUpdateGrouping<IBlueRow, int, TKey>, TKey>>(Subscribe);
+
+      IDisposable Subscribe(IObserver<TableUpdate<TableUpdateGrouping<IBlueRow, int, TKey>, TKey>> observer) =>
+         inputStream.AsObservable().AsTables().GroupTable(r => r.Id(), groupBy).Subscribe(observer);
+   }
+
+   class GroupTableBuilder {
+      readonly Func<int> getUniqueId;
+      ImmutableDictionary<int, int> inputToOutputIds = ImmutableDictionary<int, int>.Empty;
+      ImmutableDictionary<int, int> outputToInputIds = ImmutableDictionary<int, int>.Empty;
+      IndexTable? current;
+
+      public GroupTableBuilder(Func<int> getUniqueId) =>
+         this.getUniqueId = getUniqueId;
+
+      public IndexTable? Current => current;
+
+      public IndexTable Reset(IBlueRowCollection rows) {
+         var inputToOutputIdsBd = inputToOutputIds.ToBuilder();
+         var outputToInputIdsBd = outputToInputIds.ToBuilder();
+         foreach (var r in rows) {
+            int inputId = r.Id();
+            if (!inputToOutputIdsBd.TryGetValue(inputId, out var outputId)) {
+               outputId = getUniqueId();
+               inputToOutputIdsBd.Add(inputId, outputId);
+               outputToInputIdsBd.Add(outputId, inputId);
+            }
+            Log.assert(outputToInputIdsBd.TryGetValue(outputId, out var oinputId) && oinputId == inputId);
+            Log.assert(inputToOutputIdsBd.TryGetValue(inputId, out var ioutputId) && ioutputId == outputId);
+         }
+         inputToOutputIds = inputToOutputIdsBd.ToImmutable();
+         outputToInputIds = outputToInputIdsBd.ToImmutable();
+         current = new(rows, outputToInputIds, inputToOutputIds);
+         return current;
+      }
+   }
+}

src/DataStreamSuite/DataStream/BlueOrb/GroupByStream.cs

@@ -0,0 +1,127 @@
+using System;
+using System.Collections.Generic;
+using System.Diagnostics;
+using System.Linq;
+using System.Text.Json;
+
+using bts;
+//using bts.SilverPine;
+using BCG;
+using mbts.utils;
+
+namespace bts.BlueOrb {
+   [DataStreamUserRespresentation("GroupByStream")]
+   public class GroupByStream : ViewStream {
+
+      public interface INonIncGroupByEvaluator {
+         // grouped_rows
+         IBlueRowCollection? evaluate(IDictionary<int, List<IBlueRow>> grouped_rows);
+         string[]? get_group_by_cols();
+      }
+
+      private class DelNonIncGroupByEvaluator : INonIncGroupByEvaluator {
+         Func<string[]?> _groupby_cols_func;
+         Func<List<IBlueRow>, List<IBlueRow>> _evaluate_func;
+         public DelNonIncGroupByEvaluator(Func<string[]?> groupby_cols_func, Func<List<IBlueRow>, List<IBlueRow>> evaluate_func) {
+            _groupby_cols_func = groupby_cols_func;
+            _evaluate_func = evaluate_func;
+         }
+         public IBlueRowCollection? evaluate(IDictionary<int, List<IBlueRow>> grouped_rows) {
+            BlueTable? rv = null;
+            foreach (var kv in grouped_rows) {
+               List<IBlueRow> g_rows = kv.Value;
+               List<IBlueRow> out_rows = _evaluate_func.Invoke(g_rows);
+               if (out_rows is null || out_rows.Count == 0) {
+                  continue;
+               }
+               rv ??= new BlueTable(out_rows.ElementAt(0).schema);
+               rv.addRows(out_rows);
+            }
+            return rv;
+         }
+         public string[]? get_group_by_cols() {
+            return _groupby_cols_func();
+         }
+      }
+      readonly ViewStream? _main_stream;
+      readonly INonIncGroupByEvaluator? _groupby_func;
+      readonly string[]? _groupby_cols;
+      readonly Stopwatch _timer = new();
+
+      BlueRowHasher? _row_hasher;
+      int _n_total_fires = 0;
+      int _n_expensive_fires = 0;
+
+      public GroupByStream(ViewStream main_stream, INonIncGroupByEvaluator groupby_func) {
+         _main_stream = main_stream;
+         _groupby_func = groupby_func;
+         _groupby_cols = _groupby_func.get_group_by_cols();
+         _row_hasher = new BlueRowHasher(_groupby_cols == null ? 0 : _groupby_cols.Length);
+      }
+
+      public GroupByStream(ViewStream main_stream, Func<string[]?> groupby_cols_func, Func<List<IBlueRow>, List<IBlueRow>> evaluate_func)
+         : this(main_stream, new DelNonIncGroupByEvaluator(groupby_cols_func, evaluate_func)) { }
+
+      public GroupByStream(ViewStream parent_str, string groupby_cols, string standard_group_by_text)
+      : this(parent_str, groupby_cols, standard_group_by_text, String.Empty) { }
+
+      // this is the main constructor that does everything
+      public GroupByStream(ViewStream parent_str, string groupby_cols, string standard_group_by_ops, string optional_sort_col) {
+         MultiRowOpHelper std_group_by = MultiRowOpHelper.parse_ops(standard_group_by_ops);
+         Func<string[]?> groupby_cols_func = () => {
+            if (String.IsNullOrEmpty(groupby_cols))
+               return null;
+            return groupby_cols.Split(',').Select(x => x.Trim()).ToArray();
+         };
+         Func<List<IBlueRow>, List<IBlueRow>> evaluate_func = (group_rows) => {
+            if (!String.IsNullOrEmpty(optional_sort_col))
+               group_rows.Sort((r1, r2) => BlueDataUtility.compare_rows(r2, r1, optional_sort_col));
+            // this si where the group by aggregation happens
+            IBlueRow rv = MultiRowOpHelper.group_by_compute(group_rows, std_group_by);
+            if (rv == null)
+               return new List<IBlueRow>();
+            return new IBlueRow[] { rv }.ToList();
+         };
+         ViewStream rs = new GroupByStream(parent_str, groupby_cols_func, evaluate_func);
+         mySetRealSource(rs);
+      }
+      public GroupByStream(ViewStream parent_str, string standard_group_by_text) :
+      this(parent_str, String.Empty, standard_group_by_text) {
+      }
+
+
+      protected override void myOnFirstSubscriber() { Log.notnull(_main_stream).onChange += _main_stream_onchange; }
+
+      protected override void myOnLastSubscriber() { Log.notnull(_main_stream).onChange -= _main_stream_onchange; }
+
+      private void _main_stream_onchange(object nv) {
+         var dp = ExceptionAssert.Argument.Is<IDataPacket>(nv);
+         ///grouping together
+         _timer.Restart();
+         IDictionary<int, List<IBlueRow>> grouped_rows = (_groupby_cols == null || _groupby_cols.Length == 0) ?
+         new Dictionary<int, List<IBlueRow>>() { { 0, dp.newRows.to_list() } } :
+         BlueDataUtility.group_together(dp.newRows, _groupby_cols, ref _row_hasher);
+         _timer.Stop();
+         var groupElapsedMs = _timer.ElapsedMilliseconds;
+         if (groupElapsedMs >= 50) {
+            Log.DataFlow.info($"Time to group {dp.newRows.Count} rows into {grouped_rows.Count} groups: {_timer.ElapsedMilliseconds}ms");
+         }
+         // Log.log(bts.Log.Level.Info, bts.log.Subsystem.gen, String.Format("Time to group {0} rows into {1} groups = {2}", dp.newRows.Count, grouped_rows.Count, _timer.lastIntervalTime));
+         /// reducing
+         _timer.Restart();
+         var total_state = Log.notnull(_groupby_func).evaluate(grouped_rows);
+         if (total_state == null)
+            // mayeb returna n empty table ?
+            return;
+         _timer.Stop();
+         var applyElapsedMs = _timer.ElapsedMilliseconds;
+         _n_total_fires++;
+         if (applyElapsedMs >= 50) {
+            _n_expensive_fires++;
+            Log.DataFlow.info(
+               $"Time to evaluate {grouped_rows.Count} groups {_timer.ElapsedMilliseconds}ms, ratio {_n_expensive_fires}/{_n_total_fires}");
+         }
+         myFireEvent(new DataPacket(total_state, total_state, myLastValue == null ? new BlueTable(total_state.schema) : (myLastValue as DataPacket).newRows));
+      }
+   }
+}

src/DataStreamSuite/DataStream/BlueOrb/GroupByTable.cs

@@ -0,0 +1,83 @@
+namespace bts.BlueOrb;
+
+using BCG;
+
+using Bts.DataStreamCore;
+
+using System;
+using System.Collections.Generic;
+using System.Linq;
+
+public class GroupByTable<TKey> : IBlueRowCollection {
+   readonly IBlueTableSchema outputSchema;
+   readonly IBlueRowCollection inputRows;
+   readonly List<IBlueRow> rows;
+   readonly Dictionary<int, int> idToIndex;
+
+   public GroupByTable(IBlueRowCollection inputRows, Func<IBlueRow, TKey> getKey, IEqualityComparer<TKey> keyEqualityComparer, IComparer<TKey> keyComparer, string? groupKeyCol = "group") {
+      this.inputRows = inputRows;
+      var cols = inputRows.schema.columnInfo.ToList();
+      rows = new(inputRows.Count);
+      int groupKeyColIdx = -1;
+      if (!string.IsNullOrEmpty(groupKeyCol)) {
+         groupKeyColIdx = cols.Count;
+         cols.Add(new(groupKeyCol, groupKeyColIdx, typeof(TKey)));
+      }
+      for (int i = 0; i < cols.Count; i++) {
+         Log.assert(cols[i].colNo == i);
+      }
+      outputSchema = new BlueTableSchema(cols.ToArray(), "GroupBy");
+      rows.AddRange(inputRows
+         .GroupBy(getKey, keyEqualityComparer)
+         .OrderBy(g => g.Key, keyComparer)
+         .SelectMany(rg => rg.Select(r => new Row(rg.Key, r, outputSchema, groupKeyColIdx))));
+      idToIndex = rows.Select((r, i) => (r, i)).ToDictionary(t => t.r.Id(), t => t.i);
+   }
+
+   public IBlueRow this[int id] =>
+      rows[idToIndex[id]];
+
+   public IBlueTableSchema schema => outputSchema;
+
+   public int Count => rows.Count;
+
+   public bool containsID(int id) =>
+      idToIndex.ContainsKey(id);
+
+   public IEnumerator<IBlueRow> GetEnumerator() =>
+      rows.GetEnumerator();
+
+   public IBlueRow GetRow(int index) =>
+      rows[index];
+
+   public int IndexOf(IBlueRow item) =>
+      idToIndex.TryGetValue(item.Id(), out var idx) && BlueRowEqualityComparer.Equals(item, rows[idx])
+         ? idx
+         : -1;
+
+   public class Row : IBlueRow {
+      readonly IBlueRow inputRow;
+      readonly IBlueTableSchema outputSchema;
+      readonly int keyColIdx;
+
+      public Row(TKey key, IBlueRow inputRow, IBlueTableSchema outputSchema, int keyColIdx = -1) {
+         Key = key;
+         this.inputRow = inputRow;
+         this.outputSchema = outputSchema;
+         this.keyColIdx = keyColIdx;
+      }
+
+      public object? this[int index] {
+         get {
+            if (index == keyColIdx) {
+               return Key;
+            }
+            return inputRow[index];
+         }
+      }
+
+      public TKey Key { get; }
+
+      public IBlueTableSchema schema => outputSchema;
+   }
+}

src/DataStreamSuite/DataStream/BlueOrb/ImmutableOrderedTable.cs

@@ -0,0 +1,229 @@
+namespace bts.BlueOrb;
+
+using BCG;
+
+using Bts.DataStreamCore;
+
+using Spark.Core.Collections;
+
+using System;
+using System.Collections.Generic;
+using System.Collections.Immutable;
+using System.Linq;
+
+public class ImmutableOrderedTable : IBlueRowCollection {
+   readonly IBlueTableSchema schema;
+   readonly ImmutableSortedSet<KeyValuePair<long, IBlueRow>> rows;
+   readonly IReadOnlyDictionary<long, IBlueRow> ordinalToRow;
+   readonly ImmutableDictionary<int, long> idToOrdinal;
+   long newRowOrdinal = 0;
+
+   public ImmutableOrderedTable(IBlueTableSchema schema)
+      : this(
+         schema,
+         ImmutableSortedSet.Create(KeyValuePairComparer.KeyComparer<long, IBlueRow>()),
+         ImmutableDictionary<int, long>.Empty, 0) { }
+
+   ImmutableOrderedTable(IBlueTableSchema schema, ImmutableSortedSet<KeyValuePair<long, IBlueRow>> rows, ImmutableDictionary<int, long> idToOrdinal, long newRowOrdinal) {
+      this.schema = schema;
+      this.rows = rows;
+      this.idToOrdinal = idToOrdinal;
+      this.newRowOrdinal = newRowOrdinal;
+      ordinalToRow = ImmutableSortedSetDictionary.Create(rows);
+   }
+
+   public static OrderedTable FromRowCollection(IBlueRowCollection rowCollection) {
+      var rowList = new List<IBlueRow>(rowCollection.Count);
+      var idToIndex = new Dictionary<int, int>();
+      foreach (var r in rowCollection) {
+         idToIndex[r.Id()] = rowList.Count;
+         rowList.Add(r);
+      }
+      return new(rowCollection.schema, rowList, idToIndex);
+   }
+
+   public IBlueTableSchema Schema => schema;
+
+   public int Count => rows.Count;
+
+   public IBlueRow this[int id] =>
+      idToOrdinal.TryGetValue(id, out var ord) ? ordinalToRow[ord] : throw new KeyNotFoundException();
+
+   IBlueTableSchema IBlueRowCollection.schema => schema;
+
+   public bool containsID(int id) =>
+      idToOrdinal.ContainsKey(id);
+
+   public IEnumerator<IBlueRow> GetEnumerator() {
+      foreach (var (_, row) in rows) {
+         yield return row;
+      }
+   }
+
+   public IBlueRow GetRow(int index) =>
+      rows[index].Value;
+
+   public int IndexOf(IBlueRow item) =>
+      idToOrdinal.TryGetValue(item.Id(), out var ord) ? rows.IndexOf(new(ord, null!)) : -1;
+
+   public Builder ToBuilder() =>
+      new(schema, rows, idToOrdinal, newRowOrdinal);
+
+   public class Builder : IBlueRowCollection {
+      readonly IBlueTableSchema schema;
+
+      ImmutableSortedSet<KeyValuePair<long, IBlueRow>> originalRows;
+      ImmutableSortedSet<KeyValuePair<long, IBlueRow>>.Builder rows;
+      ImmutableDictionary<int, long> originalIdToOrdinal;
+      ImmutableDictionary<int, long>.Builder idToOrdinal;
+      IReadOnlyDictionary<long, IBlueRow> originalOrdinalToRow;
+      IReadOnlyDictionary<long, IBlueRow> ordinalToRow;
+
+      readonly HashSet<int> removes = new();
+      readonly HashSet<int> adds = new();
+      long newRowOrdinal;
+
+      public Builder(IBlueTableSchema schema, ImmutableSortedSet<KeyValuePair<long, IBlueRow>> rows, ImmutableDictionary<int, long> idToOrdinal, long newRowOrdinal) {
+         this.schema = schema;
+         originalRows = rows;
+         this.rows = rows.ToBuilder();
+         originalIdToOrdinal = idToOrdinal;
+         this.idToOrdinal = idToOrdinal.ToBuilder();
+         originalOrdinalToRow = ImmutableSortedSetDictionary.Create(originalRows);
+         ordinalToRow = ImmutableSortedSetDictionary.CreateBuilder(this.rows);
+         this.newRowOrdinal = newRowOrdinal;
+      }
+
+      public IBlueRow this[int id] {
+         get {
+            if (idToOrdinal.TryGetValue(id, out var ord) && !removes.Contains(id)) {
+               return ordinalToRow[ord];
+            }
+            throw new KeyNotFoundException();
+         }
+      }
+
+      public int Count => rows.Count + adds.Count - removes.Count;
+
+      IBlueTableSchema IBlueRowCollection.schema => schema;
+
+      public void Add(IBlueRow row) =>
+         addOrUpdate(row, false);
+
+      public void AddOrUpdate(IBlueRow row) =>
+         addOrUpdate(row, true);
+
+      public bool containsID(int id) =>
+         (idToOrdinal.ContainsKey(id) && !removes.Contains(id)) || adds.Contains(id);
+
+      public IEnumerator<IBlueRow> GetEnumerator() {
+         foreach (var row in rows) {
+            if (!removes.Contains(row.Value.Id())) {
+               yield return row.Value;
+            }
+         }
+      }
+
+      public IBlueRow GetRow(int index) =>
+         rows[index].Value;
+
+      public int IndexOf(IBlueRow item) {
+         if (idToOrdinal.TryGetValue(item.Id(), out var ord)) {
+            return rows.IndexOf(new(ord, item));
+         }
+         return -1;
+      }
+
+      public bool Remove(int id) {
+         if (!idToOrdinal.TryGetValue(id, out var ord)) {
+            return false;
+         }
+         idToOrdinal.Remove(id);
+         Log.assert(rows.Remove(new(ord, null!)));
+         Log.assert(ordinalToRow.Count == rows.Count);
+         adds.Remove(id);
+         if (originalIdToOrdinal.ContainsKey(id)) {
+            removes.Add(id);
+         }
+         return true;
+      }
+
+      bool ICollection<IBlueRow>.Remove(IBlueRow item) {
+         int id = item.Id();
+         if (!idToOrdinal.TryGetValue(id, out var ord) || !BlueRowEqualityComparer.Equals(item, ordinalToRow[ord])) {
+            return false;
+         }
+         rows.Remove(new(ord, null!));
+         adds.Remove(id);
+         if (originalIdToOrdinal.ContainsKey(id)) {
+            removes.Add(id);
+         }
+         return true;
+      }
+
+      public ImmutableOrderedTable ToImmutable(out IDataPacket dp) {
+         var newRows = new ImmutableOrderedTable(schema, rows.ToImmutable(), idToOrdinal.ToImmutable(), newRowOrdinal);
+         dp = new DataPacket(
+            newRows,
+            new BlueTableListWrapper(schema, adds.Select(id => ordinalToRow[idToOrdinal[id]]).ToList()),
+            new BlueTableListWrapper(schema, removes.Select(id => originalOrdinalToRow[originalIdToOrdinal[id]]).ToList()));
+         return newRows;
+      }
+
+      void addOrUpdate(IBlueRow row, bool canUpdate) {
+         if (newRowOrdinal == long.MaxValue) {
+            resetOrdinal();
+         }
+         int id = row.Id();
+         if (idToOrdinal.TryGetValue(id, out var ord)) {
+            if (!canUpdate) {
+               throw new ArgumentException("Row id already present", nameof(row));
+            }
+            rows.Remove(new(ord, row));
+         }
+         ord = newRowOrdinal++;
+         idToOrdinal[id] = ord;
+         if (originalIdToOrdinal.ContainsKey(id)) {
+            removes.Add(id);
+         }
+         rows.Add(new(ord, row));
+         adds.Add(id);
+      }
+
+      void resetOrdinal() {
+         Log.DataFlow.warn($$"""[immutable_ordered_table_reset_ordinal.json]{ "tableName": "{{schema.tableName}}", "origNumRows": {{originalRows.Count}}, "numRows": {{rows.Count}}, "ts": {{DateTime.Now}} }""");
+         var originalRowsBd = originalRows.Clear().ToBuilder();
+         var originalIdToOrdinalBd = originalIdToOrdinal.Clear().ToBuilder();
+         var newRowsBd = ImmutableSortedSet.CreateBuilder(rows.KeyComparer);
+         var newIdToOrdinalBd = ImmutableDictionary.CreateBuilder<int, long>(idToOrdinal.KeyComparer);
+         long ord = 0;
+         long? prevRowOrd = null;
+         foreach (var kv in originalRows) {
+            if (prevRowOrd.HasValue) {
+               Log.assert(prevRowOrd.Value < kv.Key);
+            }
+            prevRowOrd = kv.Key;
+            originalRowsBd.Add(new(ord, kv.Value));
+            originalIdToOrdinalBd.Add(new(kv.Value.Id(), ord));
+            ord++;
+         }
+         prevRowOrd = null;
+         foreach (var kv in rows) {
+            if (prevRowOrd.HasValue) {
+               Log.assert(prevRowOrd.Value < kv.Key);
+            }
+            prevRowOrd = kv.Key;
+            newRowsBd.Add(new(ord, kv.Value));
+            newIdToOrdinalBd.Add(new(kv.Value.Id(), ord));
+            ord++;
+         }
+         rows = newRowsBd;
+         originalRows = originalRowsBd.ToImmutable();
+         originalIdToOrdinal = originalIdToOrdinalBd.ToImmutable();
+         idToOrdinal = newIdToOrdinalBd;
+         originalOrdinalToRow = ImmutableSortedSetDictionary.Create(originalRows);
+         ordinalToRow = ImmutableSortedSetDictionary.CreateBuilder(rows);
+         newRowOrdinal = ord;
+      }
+   }
+}

src/DataStreamSuite/DataStream/BlueOrb/IndexRow.cs

@@ -0,0 +1,23 @@
+namespace bts.BlueOrb;
+
+using BCG;
+
+public class IndexRow : IBlueRow {
+   readonly IBlueRow inputRow;
+   readonly int id;
+   readonly int idIndex;
+
+   public IndexRow(IBlueRow inputRow, int id, int idIndex, IBlueTableSchema schema) {
+      this.inputRow = inputRow;
+      this.id = id;
+      this.idIndex = idIndex;
+      this.schema = schema;
+   }
+
+   public IBlueRow InputRow => inputRow;
+
+   public IBlueTableSchema schema { get; }
+
+   public object? this[int index] =>
+      index == idIndex ? id : inputRow[index];
+}

src/DataStreamSuite/DataStream/BlueOrb/IndexStream.cs

@@ -0,0 +1,53 @@
+namespace bts.BlueOrb;
+
+using BCG;
+
+using Bts.DataStream;
+using Bts.DataStreamCore;
+
+using Spark.Core.Disposables;
+
+using System;
+using System.Collections.Immutable;
+
+public class IndexStream : ViewStream {
+   readonly ViewStream inputStream;
+
+   ImmutableDictionary<int, int> inputToOutputIds = ImmutableDictionary<int, int>.Empty;
+   ImmutableDictionary<int, int> outputToInputIds = ImmutableDictionary<int, int>.Empty;
+   int idCounter = 0;
+   IDisposable? sub;
+
+   public IndexStream(ViewStream inputStream) =>
+      this.inputStream = inputStream;
+
+   ~IndexStream() =>
+      Disposable.dispose(ref sub);
+
+   protected override void myOnFirstSubscriber() {
+      base.myOnFirstSubscriber();
+      sub = inputStream.Subscribe(OnInputChanged);
+
+      void OnInputChanged(IDataPacket dp) {
+         var inputToOutputIdsBd = inputToOutputIds.ToBuilder();
+         var outputToInputIdsBd = outputToInputIds.ToBuilder();
+         foreach (var r in dp.addedRows) {
+            int inputId = r.Id();
+            if (!inputToOutputIds.TryGetValue(inputId, out int outputId)) {
+               outputId = ++idCounter;
+               inputToOutputIdsBd[inputId] = outputId;
+               outputToInputIdsBd[outputId] = inputId;
+            }
+         }
+         inputToOutputIds = inputToOutputIdsBd.ToImmutable();
+         outputToInputIds = outputToInputIdsBd.ToImmutable();
+         myFireEvent(new DataPacket(
+            new IndexTable(dp.newRows, outputToInputIds, inputToOutputIds),
+            new IndexTable(dp.addedRows, outputToInputIds, inputToOutputIds),
+            new IndexTable(dp.deletedRows, outputToInputIds, inputToOutputIds)));
+      }
+   }
+
+   protected override void myOnLastSubscriber() =>
+      Disposable.dispose(ref sub);
+}

src/DataStreamSuite/DataStream/BlueOrb/IndexTable.cs

@@ -0,0 +1,93 @@
+namespace bts.BlueOrb;
+
+using BCG;
+
+using Bts.DataStreamCore;
+
+using Spark.Core.Collections;
+
+using System;
+using System.Collections.Generic;
+using System.Collections.Immutable;
+
+public class IndexTable : IBlueRowCollection {
+   readonly IBlueRowCollection inputRows;
+   readonly IDictionary<int, int> outputToInputIdMap;
+   readonly IDictionary<int, int> inputToOutputIdMap;
+   readonly int idColIndex;
+
+   public IndexTable(IBlueRowCollection inputRows, IDictionary<int, int> outputToInputIdMap, IDictionary<int, int> inputToOutputIdMap) {
+      this.inputRows = inputRows;
+      this.outputToInputIdMap = outputToInputIdMap;
+      this.inputToOutputIdMap = inputToOutputIdMap;
+      if (!inputRows.schema.columnInfo.TryGetFirst(c => c.colName == "id", out var idColInfo)) {
+         throw new ArgumentException("No 'id' column", nameof(inputRows));
+      }
+      idColIndex = idColInfo.colNo;
+   }
+
+   public IBlueTableSchema schema => inputRows.schema;
+
+   public int Count => inputRows.Count;
+
+   public IBlueRow this[int id] =>
+      outputToInputIdMap.TryGetValue(id, out var inputId)
+         ? new IndexRow(inputRows[inputId], id, idColIndex, schema)
+         : throw new KeyNotFoundException();
+
+   IBlueRow IList<IBlueRow>.this[int index] {
+      get => GetRow(index);
+      set => throw new NotSupportedException();
+   }
+
+   public static IndexTable Create(IBlueRowCollection inputRows) {
+      var outputToInputIdMap = new Dictionary<int, int>();
+      var inputToOutputIdMap = new Dictionary<int, int>();
+      foreach (var r in inputRows) {
+         int inputId = r.Id();
+         int outputId = outputToInputIdMap.Count + 1;
+         outputToInputIdMap[outputId] = inputId;
+         inputToOutputIdMap[inputId] = outputId;
+      }
+      return new(inputRows, outputToInputIdMap, inputToOutputIdMap);
+   }
+
+   public static BlueTableImmutable CreateImmutable(IBlueTableSchema schema, IEnumerable<IBlueRow> rows) {
+      int idIndex = schema.ResolveIndices("id");
+      var rowSetBd = ImmutableSortedSet.Create(KeyValuePairComparer.KeyComparer<int, IBlueRow>()).ToBuilder();
+      foreach (var r in rows) {
+         int idx = rowSetBd.Count + 1;
+         var inputRow = (r as IndexRow)?.InputRow ?? r;
+         rowSetBd.Add(new(idx, new IndexRow(inputRow, idx, idIndex, schema)));
+      }
+      return new(schema, rowSetBd.ToImmutable());
+   }
+
+   public bool containsID(int id) =>
+      inputRows.containsID(id);
+
+   public IEnumerator<IBlueRow> GetEnumerator() {
+      foreach (var r in inputRows) {
+         yield return new IndexRow(r, inputToOutputIdMap[r.Id()], idColIndex, schema);
+      }
+   }
+
+   public IBlueRow GetRow(int index) {
+      var r = inputRows.GetRow(index);
+      return new IndexRow(r, inputToOutputIdMap[r.Id()], idColIndex, schema);
+   }
+
+   public int IndexOf(IBlueRow item) {
+      int id = item.Id();
+      if (outputToInputIdMap.TryGetValue(id, out var inputId)) {
+         var inputRow = inputRows[inputId];
+         var r = new IndexRow(inputRow, id, idColIndex, schema);
+         if (BlueRowEqualityComparer.Equals(r, item)) {
+            var idx = inputRows.IndexOf(inputRow);
+            Log.assert(idx >= 0);
+            return idx;
+         }
+      }
+      return -1;
+   }
+}

src/DataStreamSuite/DataStream/BlueOrb/Join/SimpleJoinStream.cs

@@ -0,0 +1,196 @@
+namespace bts.BlueOrb {
+   using BCG;
+   using RowHash = System.Int32;
+   using System;
+   using System.Collections.Generic;
+   using System.Linq;
+   using System.Text;
+   using System.Threading.Tasks;
+   using System.Collections;
+   using System.Data.Entity.Core.Mapping;
+
+   [DataStreamUserRespresentation("JoinStream")]
+   public class SimpleJoinStream : ViewStream {
+      public class JoinBookkeepingData {
+         // initialized in cons
+         private readonly List<string> _join_col_names;
+         private readonly bool _is_left;
+         private readonly BlueRowHasher _hasher;
+         // initialized on first udpate         
+         private IBlueTableSchema _schema;
+         private int[] _join_cols;
+         private IDictionary<RowHash, List<IBlueRow>> _hash_row_map = new Dictionary<RowHash, List<IBlueRow>>();
+
+         public JoinBookkeepingData(ViewStream vs, List<string> join_cols, bool is_left, BlueRowHasher row_hasher) {
+            _join_col_names = join_cols.Where(c => !String.IsNullOrEmpty(c)).ToList();
+            _is_left = is_left;
+            _hasher = row_hasher;
+         }
+
+         public IBlueTableSchema schema => _schema;
+
+         public IDictionary<RowHash, List<IBlueRow>> rows => _hash_row_map;
+
+         public List<IBlueRow> get_matching_rows(int hash) {
+            List<IBlueRow> rv = null;
+            _hash_row_map.TryGetValue(hash, out rv);
+            return rv;
+         }
+
+         private RowHash _get_hash(IBlueRow r) {
+            if (_join_cols == null || _join_cols.Length == 0) {
+               return 0;
+            }
+            return _hasher.get_hash(r, _join_cols);
+         }
+
+         public void update_data(DataPacket dp) {
+            if (!ready) {
+               _schema = dp.newRows.schema;
+               _join_cols = _join_col_names.Count == 0 ? new int[] { } : _join_col_names.Select(n => dp.schema[n].colNo).ToArray();
+            }
+            _hash_row_map.Clear();
+            foreach (IBlueRow r in dp.newRows) {
+               int hash = _get_hash(r);
+               List<IBlueRow> hash_rows = null;
+               if (!_hash_row_map.TryGetValue(hash, out hash_rows)) {
+                  hash_rows = new List<IBlueRow>();
+                  _hash_row_map.Add(hash, hash_rows);
+               }
+               hash_rows.Add(r);
+            }
+         }
+
+         public bool ready {
+            get {
+               return _schema != null;
+            }
+         }
+
+      }
+
+      private ViewStream _ls, _rs;
+      private JoinBookkeepingData _ldata, _rdata;
+      private ViewStreamDatabase _vsd;
+      private IBlueTableSchema _schema = null;
+
+      public SimpleJoinStream(ViewStream ls, string ljc, ViewStream rs, string rjc) :
+         this(ViewStreamDatabase.instance, ls, ljc, rs, rjc) { }
+
+      public SimpleJoinStream(ViewStreamDatabase vsd, ViewStream ls, string ljc, ViewStream rs, string rjc) {
+         _ls = ls;
+         _rs = rs;
+         List<string> lj_col_list = ljc.Split(',').Where(x => !String.IsNullOrEmpty(x)).ToList();
+         List<string> rj_col_list = rjc.Split(',').Where(x => !String.IsNullOrEmpty(x)).ToList();
+         Log.assert(lj_col_list.Count == rj_col_list.Count);
+         int n_cols = lj_col_list.Count;
+         BlueRowHasher hasher = new BlueRowHasher(n_cols);
+         _ldata = new JoinBookkeepingData(ls, lj_col_list, true, hasher);
+         _rdata = new JoinBookkeepingData(rs, rj_col_list, false, hasher);
+      }
+
+      protected override void myOnFirstSubscriber() {
+         _ls.onChange += _ls_onchange;
+         _rs.onChange += _rs_onchange;
+      }
+
+      private void _ls_onchange(object dp_obj) {
+         DataPacket dp = dp_obj as DataPacket;
+         _ldata.update_data(dp);
+         _form_and_fire();
+      }
+
+      private void _rs_onchange(object dp_obj) {
+         DataPacket dp = dp_obj as DataPacket;
+         _rdata.update_data(dp);
+         _form_and_fire();
+      }
+
+      protected override void myOnLastSubscriber() {
+         _ls.onChange -= _ls_onchange;
+         _rs.onChange -= _rs_onchange;
+      }
+
+      private void _build_schema() {
+         if (_schema != null)
+            return;
+         List<ColumnInfo> cis = new List<ColumnInfo>();
+         cis.Add(new ColumnInfo("id", 0, TypeCode.Int32));
+         foreach (ColumnInfo ci in _ldata.schema.columnInfo) {
+            string col_name = _rdata.schema.contains_column(ci.colName) ? $"left_{ci.colName}" : ci.colName;
+            cis.Add(new ColumnInfo(col_name, cis.Count, ci.colType));
+         }
+         foreach (ColumnInfo ci in _rdata.schema.columnInfo) {
+            string col_name = _ldata.schema.contains_column(ci.colName) ? $"right_{ci.colName}" : ci.colName;
+            cis.Add(new ColumnInfo(col_name, cis.Count, ci.colType));
+         }
+         _schema = new BlueTableSchema($"{_ldata.schema.tableName}_join_{_rdata.schema.tableName}", cis.ToArray());
+      }
+
+      private void _form_and_fire() {
+         if (!_ldata.ready || !_rdata.ready)
+            return;
+         _build_schema();
+         List<object[]> res_rows = new List<object[]>();
+
+         Action<object[], IBlueRow, bool> copy_row = (rval, r, is_left) => {
+            int offset = 1;
+            JoinBookkeepingData sdata = _ldata;
+            if (is_left) {
+               offset = 1;
+               sdata = _ldata;
+            } else {
+               sdata = _rdata;
+               offset = 1 + _ldata.schema.columnInfo.Length;
+            }
+            for (int i = 0; i < sdata.schema.columnInfo.Length; i++) {
+               rval[offset + i] = r[i];
+            }
+         };
+         Func<int, IBlueRow, object[]> build_row_array = (id, l_row) => {
+            object[] rv = new object[_schema.columnNames.Length];
+            for (int i = 0; i < _schema.columnNames.Length; i++) {
+               rv[i] = DBNull.Value;
+            }
+            rv[0] = id;
+            copy_row(rv, l_row, true);
+            return rv;
+         };
+         // this is where the join happens
+         foreach (var kv in _ldata.rows) {
+            RowHash hash = kv.Key;
+            List<IBlueRow> l_rows = kv.Value;
+            List<IBlueRow> r_rows = _rdata.get_matching_rows(hash);
+
+            foreach (IBlueRow lr in l_rows) {
+               if (r_rows != null) {
+                  foreach (IBlueRow r in r_rows) {
+                     object[] rowvals = build_row_array(res_rows.Count, lr);
+                     copy_row(rowvals, r, false);
+                     res_rows.Add(rowvals);
+                  }
+               } else {
+                  object[] rowvals = build_row_array(res_rows.Count, lr);
+                  res_rows.Add(rowvals);
+               }
+            }
+         }
+         BlueTable rv = new BlueTable(_schema);
+         foreach (object[] rval in res_rows) {
+            rv.addRow(new BlueRow(_schema, rval));
+         }
+         var dp = BlueDataUtility.build_dp(rv, myLastValue as DataPacket);
+         myFireEvent(dp);
+      }
+   }
+
+   [DataStreamUserRespresentation("FullJoinStream")]
+   public class FullJoinStream : ViewStream {
+      public FullJoinStream(ViewStream l_str, string ljcs, ViewStream r_str, string rjcs) {
+         ViewStream js = new SimpleJoinStream(l_str, ljcs, r_str, rjcs);
+         ViewStream rs = new bts.BlueOrb.FilterStream(js, r => !DBNull.Value.Equals(r["right_id"]));
+         mySetRealSource(rs);
+      }
+   }
+
+}

src/DataStreamSuite/DataStream/BlueOrb/JsonSaverStream.cs

@@ -0,0 +1,28 @@
+//namespace bts.Blueorb {
+//   using BCG;
+
+//   public class JsonSaverStream : ViewStream {
+//      private ViewStream _stream;
+//      private string _file_name;
+//      public JsonSaverStream(ViewStream vs, string file_name) {
+//         _stream = vs;
+//         _file_name = file_name;
+//      }
+
+//      protected override void myOnFirstSubscriber() {
+//         _stream.onChange += _on_data;
+//      }
+
+//      private void _on_data(object dp_obj) {
+//         if (dp_obj == null) {
+//            return;
+//         }
+//         DataPacket dp = dp_obj as DataPacket;
+//         dp.newRows.write_to_json_file(dp.newRows, _file_name);
+//      }
+
+//      protected override void myOnLastSubscriber() {
+//         _stream.onChange -= _on_data;
+//      }
+//   }
+//}

src/DataStreamSuite/DataStream/BlueOrb/LiteralStream.cs

@@ -0,0 +1,91 @@
+using System;
+using System.Collections;
+using System.Collections.Generic;
+
+using Amazon.Runtime.Internal.Util;
+
+using BCG;
+using BCG.Utilities;
+
+using bts;
+
+namespace bts.BlueOrb{
+   public class LiteralStream : ViewStream {
+
+      public static List<Tuple<ColumnInfo, object>> parse_columns(string key_value_pairs) {
+         List<Tuple<ColumnInfo, object>> rv = new List<Tuple<ColumnInfo, object>>();
+         string[] pairs = key_value_pairs.Split(' ');
+         foreach (string p in pairs) {
+            string[] kvp = p.Split(':');
+            string col_name = kvp[0];
+            string val = kvp[1];
+            var tup = BlueDataUtility.guess_type(val);
+            var ci = new ColumnInfo(col_name, rv.Count, tup.Item1);
+            rv.Add(Tuple.Create(ci, tup.Item2));
+         }
+         return rv;
+      }
+
+      //literal("table1", "x:123, y:1.001, z:abc;x:123, y:1.001, z:abc")
+      // col_name: col_value ... ; col_name: col_value
+      public LiteralStream(string table_name, string rows) {
+         Func<string, int, IBlueRow> get_row = (json_row, id) => {
+            Dictionary<string, object> map = new Dictionary<string, object>();
+            string[] toks = json_row.Split(',', StringSplitOptions.TrimEntries);
+            //Logger.assert(col_vals.Length == 2);
+            foreach (string t in toks) {
+               var col_vals = t.Split(':', StringSplitOptions.TrimEntries);
+               string col = col_vals[0];
+               string val = col_vals[1];
+               map.Add(col, val);
+            }
+            map["id"] = id;
+            return new BlueRow(map);
+         };
+         string[] row_arr = rows.Split(';', StringSplitOptions.TrimEntries);
+         BlueTable table = null;
+         foreach (string json_string in row_arr) {
+            IBlueRow r = //BlueDataUtility.convert_json_to_row(json_string, table == null ? 0 : table.Count);
+            get_row(json_string, table == null ? 0 : table.Count);
+            table = table ?? new BlueTable(r.schema);
+            table.addRow(r);
+         }
+         ViewStream str = new BlueTableStream(table);
+         str = new SchemaGuesserStream(str);
+         mySetRealSource(str);
+      }
+
+      public LiteralStream(string tableName, string schema, string value) {
+         string[] columns = schema.Split(' ');
+
+
+         ColumnInfo[] ci = new ColumnInfo[columns.Length + 1];
+         ci[0] = new ColumnInfo("id", 0, typeof(int));
+         int n = 1;
+         foreach (string c in columns) {
+            string[] columnAndType = c.Split('/');
+            if (columnAndType.Length == 1)
+               ci[n] = new ColumnInfo(c, n, typeof(double));
+            else
+               ci[n] = new ColumnInfo(columnAndType[0], n, Type.GetType("System." + columnAndType[1], true));
+            n++;
+         }
+
+
+         BlueTable t = new BlueTable(new BlueTableSchema(tableName, ci));
+         string[] rows = value.Split(';');
+         n = 0;
+         foreach (string r in rows)
+            try {
+               if (r == string.Empty)
+                  continue;
+               t.addRow(
+                  BlueDataUtility.createRowFromCSVFileRow(r, t.schema, ++n));
+            } catch (Exception e) {
+               throw (new Exception("Error occured at line: " + n.ToString() + " " + e.Message));
+            }
+
+         mySetRealSource(new BlueRowCollectionStaticStream(t));
+      }
+   }
+}