fix use dep injection in bts.datastreamwebapp.server

Although it would build on net8, all other cs code targets net7.  Furthermore,
VS Code intellisense seems less reliable/crashes frequently when the sln file
targets both frameworks).

.config/dotnet-tools.json

@@ -9,7 +9,7 @@
       ]
     },
     "fable": {
-      "version": "4.24.0",
+      "version": "4.0.6",
       "commands": [
         "fable"
       ]
@@ -27,4 +27,4 @@
       ]
     }
   }
-}
+}

.devcontainer/Dockerfile

@@ -66,6 +66,7 @@ RUN \
 
 # Dotnet
 RUN apt-get update && apt-get install -y dotnet7
+RUN apt-get update && apt-get install -y dotnet8
 
 # Node
 ARG node_major=18
@@ -87,9 +88,6 @@ RUN dpkg -i /emumba/emumba-deribit-connector-3.3.1-Linux.LICENSE_DISABLED.deb \
 # Pcap
 RUN apt-get update && apt-get install -y libpcap0.8-dev
 
-# Databento (deps)
-RUN apt-get update && apt-get -y install libzstd-dev
-
 # Set up default executables (update-alternatives)
 RUN \
    update-alternatives --install /usr/bin/clang clang /usr/bin/clang-$clang_major 100; \

.devcontainer/devcontainer.json

@@ -1,10 +1,15 @@
 {
   "dockerFile": "Dockerfile",
+  // Creates local dir to mount to HOME in docker 
+  "initializeCommand": "mkdir -p ${localEnv:HOME}/.bts/devcontainer-mounts/${devcontainerId}",
   "hostRequirements": {
-    "memory": "8gb",
-    "storage": "10gb"
+    "memory": "4gb",
+    "storage": "6gb"
   },
+  "postCreateCommand": "/home/bts/spark/docker/sparkd/dev/devcontainer_post_create.sh",
   "containerUser": "bts",
+  "workspaceMount": "source=${localWorkspaceFolder},target=/home/bts/spark,type=bind",
+  "workspaceFolder": "/home/bts/spark",
   "updateRemoteUserUID": true,
   "shutdownAction": "none",
   // Note: If running multiple containers on the same host, each must use a
@@ -12,7 +17,15 @@
   // example, if 8000-8003 is taken, go with 9000-9003, then 10000-10003, etc.
   "runArgs": [
     "--shm-size",
-    "8gb",
+    "2gb",
+    "-p",
+    "5000:5000",
+    "-p",
+    "5001:5001",
+    "-p",
+    "5173:5173",
+    "-p",
+    "5174:5174",
     "-p",
     "8000:8000",
     "-p",
@@ -22,16 +35,23 @@
     "-p",
     "8003:8003",
     "-p",
-    "12345:12345"
+    "12345:12345",
+  ],
+  "mounts": [
+    "source=${localEnv:HOME}/.bts/devcontainer-mounts/${devcontainerId},target=/home/bts,type=bind"
   ],
   "customizations": {
     "vscode": {
+      "settings": {
+        "clangd.arguments": [
+          "--compile-commands-dir=${workspaceFolder}/build/clang/debug"
+        ]
+      },
       "extensions": [
         "eamodio.gitlens",
         "llvm-vs-code-extensions.vscode-clangd",
         "ms-vscode.cmake-tools",
-        "ms-vscode.cpptools",
-        "xaver.clang-format"
+        "ms-vscode.cpptools"
       ]
     }
   }

.editorconfig

@@ -207,17 +207,23 @@ dotnet_naming_symbols.parameters.applicable_accessibilities = public,internal,pr
 dotnet_naming_symbols.private_symbols.applicable_kinds = property,method,field,event,delegate
 dotnet_naming_symbols.private_symbols.applicable_accessibilities = protected,private
 
+# local symbols
+dotnet_naming_symbols.local_symbols.applicable_kinds = local
+
+# parameter symbols
+dotnet_naming_symbols.parameter_symbols.applicable_kinds = parameter
+
 # Naming styles
 
-dotnet_naming_style.snake_case.required_prefix =
-dotnet_naming_style.snake_case.required_suffix =
-dotnet_naming_style.snake_case.word_separator = _
-dotnet_naming_style.snake_case.capitalization = all_lower
+dotnet_naming_style.pascal_case.required_prefix =
+dotnet_naming_style.pascal_case.required_suffix =
+dotnet_naming_style.pascal_case.word_separator =
+dotnet_naming_style.pascal_case.capitalization = pascal_case
 
-dotnet_naming_style.private_snake_case.required_prefix = _
-dotnet_naming_style.private_snake_case.required_suffix =
-dotnet_naming_style.private_snake_case.word_separator = _
-dotnet_naming_style.private_snake_case.capitalization = all_lower
+dotnet_naming_style.camel_case.required_prefix =
+dotnet_naming_style.camel_case.required_suffix =
+dotnet_naming_style.camel_case.word_separator =
+dotnet_naming_style.camel_case.capitalization = camel_case
 
 dotnet_naming_style.user_types.required_prefix =
 dotnet_naming_style.user_types.required_suffix =
@@ -229,6 +235,10 @@ dotnet_naming_style.interface.required_suffix =
 dotnet_naming_style.interface.word_separator =
 dotnet_naming_style.interface.capitalization = pascal_case
 
+dotnet_naming_style.local.capitalization = camel_case
+
+dotnet_naming_style.parameter.capitalization = camel_case
+
 # rules
 dotnet_naming_rule.user_types.symbols = user_types
 dotnet_naming_rule.user_types.style = user_types
@@ -238,14 +248,18 @@ dotnet_naming_rule.interfaces.symbols = interface
 dotnet_naming_rule.interfaces.style = interface_style
 dotnet_naming_rule.interfaces.severity = suggestion
 
-dotnet_naming_rule.public_members_snake_case.symbols = public_symbols
-dotnet_naming_rule.public_members_snake_case.style = snake_case
-dotnet_naming_rule.public_members_snake_case.severity = suggestion
+dotnet_naming_rule.public_members.symbols = public_symbols
+dotnet_naming_rule.public_members.style = pascal_case
+dotnet_naming_rule.public_members.severity = suggestion
 
-dotnet_naming_rule.private_members_snake_case.symbols = private_symbols
-dotnet_naming_rule.private_members_snake_case.style = private_snake_case
-dotnet_naming_rule.private_members_snake_case.severity = suggestion
+dotnet_naming_rule.private_members.symbols = private_symbols
+dotnet_naming_rule.private_members.style = camel_case
+dotnet_naming_rule.private_members.severity = suggestion
 
-dotnet_naming_rule.parameters.symbols = parameters
-dotnet_naming_rule.parameters.style = snake_case
+dotnet_naming_rule.variables_are_camel_case.severity = suggestion
+dotnet_naming_rule.variables_are_camel_case.symbols = local_symbols
+dotnet_naming_rule.variables_are_camel_case.style = local
+
+dotnet_naming_rule.parameters.symbols = parameter_symbols
+dotnet_naming_rule.parameters.style = parameter
 dotnet_naming_rule.parameters.severity = suggestion

.gitmodules

@@ -1,6 +1,6 @@
 [submodule "externals"]
 	path = externals
-	url = git@bitbucket.org:bluetradesys/externals.git
+	url = https://bitbucket.org/bluetradesys/externals.git
 [submodule "vcpkg"]
 	path = vcpkg
 	url = https://github.com/microsoft/vcpkg.git

.vscode/launch.json

@@ -6,6 +6,35 @@
       "type": "coreclr",
       "request": "attach"
     },
+    {
+      "name": "Bts.DataStreamWebApp.Server",
+      "type": "coreclr",
+      "request": "launch",
+      "preLaunchTask": "Build DataStream WebApp (Debug)",
+      "program": "${workspaceFolder}/build/Debug/bin/net7.0/Bts.DataStreamWebApp.Server.dll",
+      "args": [
+        "--config",
+        "${workspaceFolder}/src/DataStreamSuite/Bts.DataStreamWebApp.Server/Configurations/DevServer.json",
+      ],
+      "env": {
+        "ASPNETCORE_ENVIRONMENT": "Development",
+        "ASPNETCORE_HOSTINGSTARTUPASSEMBLIES": "Microsoft.AspNetCore.SpaProxy",
+      }
+    },
+    {
+      "name": "Bts.DataStreamWebApp.Server (Skip Build)",
+      "type": "coreclr",
+      "request": "launch",
+      "program": "${workspaceFolder}/build/Debug/bin/net7.0/Bts.DataStreamWebApp.Server.dll",
+      "args": [
+        "--config",
+        "${workspaceFolder}/src/DataStreamSuite/Bts.DataStreamWebApp.Server/Configurations/DevServer.json",
+      ],
+      "env": {
+        "ASPNETCORE_ENVIRONMENT": "Development",
+        "ASPNETCORE_HOSTINGSTARTUPASSEMBLIES": "Microsoft.AspNetCore.SpaProxy",
+      }
+    },
     {
       "name": "DataStreamPublisher",
       "type": "coreclr",
@@ -22,59 +51,34 @@
         "--config",
         "${workspaceFolder}/src/DataStreamSuite/DataStreamPublisher/Configurations/${input:dataStreamPublisherConfig}.json"
       ],
-      "cwd": "build/Debug/bin/net7.0",
+      "cwd": "${workspaceFolder}/build/Debug/bin/net7.0",
       "stopAtEntry": false,
       "console": "integratedTerminal"
     },
     {
-      "name": "(gdb) Launch",
-      "type": "cppdbg",
+      "name": "DataStreamPublisher (Skip Build)",
+      "type": "coreclr",
       "request": "launch",
-      "program": "/home/bts/spark/build/sparkd-debug-clang/server/sparkd",
+      "program": "",
+      "linux": {
+        "program": "DataStreamPublisher",
+      },
+      "windows": {
+        "program": "DataStreamPublisher.exe",
+      },
       "args": [
-        "run"
+        "--config",
+        "${workspaceFolder}/src/DataStreamSuite/DataStreamPublisher/Configurations/${input:dataStreamPublisherConfig}.json"
       ],
+      "cwd": "${workspaceFolder}/build/Debug/bin/net7.0",
       "stopAtEntry": false,
-      "cwd": "/home/bts/spark/build/sparkd-debug-clang/server",
-      "environment": [],
-      "externalConsole": false,
-      "MIMode": "gdb",
-    },
-    {
-      "name": "spreader tests2",
-      "type": "cppdbg",
-      "request": "launch",
-      "program": "/home/bts/spark/build/sparkd-debug-clang/server/sparkd",
-      "args": [
-        "test",
-        "[spreader2]"
-      ],
-      "stopAtEntry": false,
-      "cwd": "/home/bts/spark/build/sparkd-debug-clang/server",
-      "environment": [],
-      "externalConsole": false,
-      "MIMode": "gdb",
-    },
-    {
-      "name": "spreader tests",
-      "type": "cppdbg",
-      "request": "launch",
-      "program": "/home/bts/spark/build/sparkd-debug-clang/server/sparkd",
-      "args": [
-        "test",
-        "[spreader]"
-      ],
-      "stopAtEntry": false,
-      "cwd": "/home/bts/spark/build/sparkd-debug-clang/server",
-      "environment": [],
-      "externalConsole": false,
-      "MIMode": "gdb",
+      "console": "integratedTerminal"
     },
     {
       "name": "DataStreamQueryAnalyzer",
       "type": "coreclr",
       "request": "launch",
-      //"preLaunchTask": "Build DataStreamQueryAnalyzer (Debug)",
+      "preLaunchTask": "Build DataStreamQueryAnalyzer (Debug)",
       "program": "",
       "windows": {
         "program": "DataStreamQueryAnalyzer.exe",
@@ -110,6 +114,25 @@
       "stopAtEntry": false,
       "console": "integratedTerminal"
     },
+    {
+      "name": "PubSubServer (Skip Build)",
+      "type": "coreclr",
+      "request": "launch",
+      "program": "",
+      "linux": {
+        "program": "PubSubServer",
+      },
+      "windows": {
+        "program": "PubSubServer.exe",
+      },
+      "args": [
+        "--config",
+        "${workspaceFolder}/src/DataStreamSuite/PubSubServer/Configurations/${input:pubSubServerConfig}.json"
+      ],
+      "cwd": "build/Debug/bin/net7.0",
+      "stopAtEntry": false,
+      "console": "integratedTerminal"
+    },
     {
       "name": "Spark",
       "type": "coreclr",
@@ -123,18 +146,6 @@
       "stopAtEntry": false,
       "console": "integratedTerminal"
     },
-    {
-      "name": "Spark (Skip Build)",
-      "type": "coreclr",
-      "request": "launch",
-      "program": "",
-      "windows": {
-        "program": "Spark.exe",
-        "cwd": "build/Debug/bin/net7.0-windows",
-      },
-      "stopAtEntry": false,
-      "console": "integratedTerminal"
-    },
     {
       "name": "Spark (Parameterized)",
       "type": "coreclr",
@@ -152,139 +163,22 @@
       "stopAtEntry": false,
       "console": "integratedTerminal"
     },
-    {
-      "name": "Spark Test",
-      "type": "coreclr",
-      "request": "launch",
-      "program": "",
-      "windows": {
-        "preLaunchTask": "Build Spark (Debug)",
-        "program": "Spark.Console.exe",
-        "cwd": "build/Debug/bin/net7.0",
-        "args": [
-          "test",
-          "${input:sparkTest}"
-        ]
-      },
-      "stopAtEntry": false,
-      "console": "integratedTerminal"
-    },
-    {
-      "name": "Spark Test (Skip Build)",
-      "type": "coreclr",
-      "request": "launch",
-      "program": "",
-      "windows": {
-        "program": "Spark.Console.exe",
-        "cwd": "build/Debug/bin/net7.0",
-        "args": [
-          "test",
-          "${input:sparkTest}"
-        ]
-      },
-      "stopAtEntry": false,
-      "console": "integratedTerminal"
-    },
     {
       "name": "Sparkd",
-      "type": "cppdbg",
+      "type": "coreclr",
       "request": "launch",
+      "preLaunchTask": "Build Sparkd (Debug | Clang)",
       "program": "",
       "linux": {
-        "preLaunchTask": "Build Sparkd (Debug)",
-        "program": "${workspaceFolder}/build/sparkd-debug-clang/server/sparkd",
-        "args": [
-          "run"
-        ],
-        "stopAtEntry": false,
-        "cwd": "${workspaceFolder}/build/sparkd-debug-clang/server",
-        "environment": [],
-        "externalConsole": false,
-        "MIMode": "gdb",
-        "setupCommands": [
-          {
-            "description": "Enable pretty-printing for gdb",
-            "text": "-enable-pretty-printing",
-            "ignoreFailures": true
-          }
-        ]
-      }
-    },
-    {
-      "name": "Sparkd (Skip Build)",
-      "type": "cppdbg",
-      "request": "launch",
-      "program": "",
-      "linux": {
-        "program": "${workspaceFolder}/build/sparkd-debug-clang/server/sparkd",
-        "args": [
-          "run"
-        ],
-        "stopAtEntry": false,
-        "cwd": "${workspaceFolder}/build/sparkd-debug-clang/server",
-        "environment": [],
-        "externalConsole": false,
-        "MIMode": "gdb",
-        "setupCommands": [
-          {
-            "description": "Enable pretty-printing for gdb",
-            "text": "-enable-pretty-printing",
-            "ignoreFailures": true
-          }
-        ]
-      }
-    },
-    {
-      "name": "Sparkd Test",
-      "type": "cppdbg",
-      "request": "launch",
-      "program": "",
-      "linux": {
-        "preLaunchTask": "Build Sparkd (Debug)",
-        "program": "${workspaceFolder}/build/sparkd-debug-clang/server/sparkd",
-        "args": [
-          "test",
-          "${input:sparkdTest}",
-        ],
-        "stopAtEntry": false,
-        "cwd": "${workspaceFolder}/build/sparkd-debug-clang/server",
-        "environment": [],
-        "externalConsole": false,
-        "MIMode": "gdb",
-        "setupCommands": [
-          {
-            "description": "Enable pretty-printing for gdb",
-            "text": "-enable-pretty-printing",
-            "ignoreFailures": true
-          }
-        ]
-      }
-    },
-    {
-      "name": "Sparkd Test (Skip Build)",
-      "type": "cppdbg",
-      "request": "launch",
-      "program": "",
-      "linux": {
-        "program": "${workspaceFolder}/build/sparkd-debug-clang/server/sparkd",
-        "args": [
-          "test",
-          "${input:sparkdTest}",
-        ],
-        "stopAtEntry": false,
-        "cwd": "${workspaceFolder}/build/sparkd-debug-clang/server",
-        "environment": [],
-        "externalConsole": false,
-        "MIMode": "gdb",
-        "setupCommands": [
-          {
-            "description": "Enable pretty-printing for gdb",
-            "text": "-enable-pretty-printing",
-            "ignoreFailures": true
-          }
-        ]
-      }
-    },
+        "program": "sparkd",
+      },
+      "args": [
+        "run"
+      ],
+      "cwd": "build/sparkd-debug-clang/server",
+      "stopAtEntry": false,
+      "console": "integratedTerminal"
+    }
   ],
   "inputs": [
     {
@@ -315,15 +209,5 @@
       "description": "Config Server URL",
       "default": "http://localhost:8000/config",
     },
-    {
-      "id": "sparkTest",
-      "type": "promptString",
-      "description": "Test to run",
-    },
-    {
-      "id": "sparkdTest",
-      "type": "promptString",
-      "description": "Test to run",
-    },
   ]
 }

.vscode/tasks.json

@@ -121,6 +121,97 @@
         "color": "terminal.ansiCyan"
       },
     },
+    {
+      "label": "Build DataStreamQueryAnalyzer",
+      "group": "build",
+      "type": "process",
+      "command": "",
+      "windows": {
+        "command": "tools/build_data_stream_query_analyzer.cmd",
+      },
+      "args": [
+        "--configuration",
+        "${input:buildConfiguration}",
+      ],
+      "icon": {
+        "id": "terminal",
+        "color": "terminal.ansiCyan"
+      },
+    },
+    {
+      "label": "Build DataStreamQueryAnalyzer (Debug)",
+      "group": "build",
+      "type": "process",
+      "command": "",
+      "windows": {
+        "command": "tools/build_data_stream_query_analyzer.cmd",
+      },
+      "args": [
+        "--configuration",
+        "Debug",
+      ],
+      "icon": {
+        "id": "terminal",
+        "color": "terminal.ansiCyan"
+      },
+    },
+    {
+      "label": "Build DataStreamQueryAnalyzer (Release)",
+      "group": "build",
+      "type": "process",
+      "command": "",
+      "windows": {
+        "command": "tools/build_data_stream_query_analyzer.cmd",
+      },
+      "args": [
+        "--configuration",
+        "Release",
+      ],
+      "icon": {
+        "id": "terminal",
+        "color": "terminal.ansiCyan"
+      },
+    },
+    {
+      "label": "Build DataStream WebApp",
+      "group": "build",
+      "type": "process",
+      "command": "",
+      "linux": {
+        "command": "tools/build_data_stream_web_app.sh",
+      },
+      "windows": {
+        "command": "tools/build_data_stream_web_app.cmd",
+      },
+      "args": [
+        "--configuration",
+        "${input:buildConfiguration}",
+      ],
+      "icon": {
+        "id": "terminal",
+        "color": "terminal.ansiCyan"
+      },
+    },
+    {
+      "label": "Build DataStream WebApp (Debug)",
+      "group": "build",
+      "type": "process",
+      "command": "",
+      "linux": {
+        "command": "tools/build_data_stream_web_app.sh",
+      },
+      "windows": {
+        "command": "tools/build_data_stream_web_app.cmd",
+      },
+      "args": [
+        "--configuration",
+        "Debug",
+      ],
+      "icon": {
+        "id": "terminal",
+        "color": "terminal.ansiCyan"
+      },
+    },
     {
       "label": "Build Spark",
       "group": "build",
@@ -303,6 +394,22 @@
         "color": "terminal.ansiCyan"
       },
     },
+    {
+      "label": "Publish DataStreamWebApp",
+      "group": "build",
+      "type": "process",
+      "command": "",
+      "linux": {
+        "command": "tools/publish_data_stream_web_app.sh",
+      },
+      "windows": {
+        "command": "tools/publish_data_stream_web_app.cmd",
+      },
+      "icon": {
+        "id": "terminal",
+        "color": "terminal.ansiCyan"
+      },
+    },
     {
       "label": "Edit DataStream Config",
       "type": "process",
@@ -353,7 +460,7 @@
     },
     {
       "label": "Run DataStreamQueryAnalyzer",
-      "dependsOn": "Build DataStream",
+      "dependsOn": "Build DataStreamQueryAnalyzer",
       "type": "process",
       "command": "",
       "windows": {
@@ -373,7 +480,7 @@
     },
     {
       "label": "Run DataStreamQueryAnalyzer (Debug)",
-      "dependsOn": "Build DataStream (Debug)",
+      "dependsOn": "Build DataStreamQueryAnalyzer (Debug)",
       "type": "process",
       "command": "",
       "windows": {
@@ -393,7 +500,7 @@
     },
     {
       "label": "Run DataStreamQueryAnalyzer (Release)",
-      "dependsOn": "Build DataStream (Release)",
+      "dependsOn": "Build DataStreamQueryAnalyzer (Release)",
       "type": "process",
       "command": "",
       "windows": {
@@ -641,21 +748,6 @@
         "color": "terminal.ansiGreen",
       }
     },
-    {
-      "label": "Run Spark (Skip Build)",
-      "type": "process",
-      "command": "",
-      "windows": {
-        "command": "Spark.exe",
-        "options": {
-          "cwd": "${workspaceFolder}/build/${input:buildConfiguration}/bin/net7.0-windows"
-        },
-      },
-      "icon": {
-        "id": "run",
-        "color": "terminal.ansiGreen",
-      }
-    },
     {
       "label": "Run Spark Config Query",
       "dependsOn": "Build Spark",
@@ -699,24 +791,6 @@
         "color": "terminal.ansiGreen",
       },
     },
-    {
-      "label": "Run Sparkd (Skip Build)",
-      "type": "process",
-      "command": "",
-      "linux": {
-        "command": "sparkd",
-      },
-      "args": [
-        "run"
-      ],
-      "options": {
-        "cwd": "${workspaceFolder}/build/${input:sparkdCmakePreset}/server"
-      },
-      "icon": {
-        "id": "run",
-        "color": "terminal.ansiGreen",
-      },
-    },
     {
       "label": "Run Sparkd Config Query",
       "dependsOn": "Build Sparkd",
@@ -738,65 +812,6 @@
         "color": "terminal.ansiGreen",
       },
     },
-    {
-      "label": "Test Spark (Debug)",
-      "dependsOn": "Build Spark (Debug)",
-      "type": "process",
-      "icon": {
-        "id": "coverage",
-        "color": "terminal.ansiYellow",
-      },
-      "command": "",
-      "windows": {
-        "command": "Spark.Console.exe",
-        "args": [
-          "test",
-          "${input:sparkTest}"
-        ],
-        "options": {
-          "cwd": "${workspaceFolder}/build/Debug/bin/net7.0",
-        }
-      },
-    },
-    {
-      "label": "Test Sparkd",
-      "dependsOn": "Build Sparkd",
-      "type": "process",
-      "command": "",
-      "linux": {
-        "command": "sparkd",
-      },
-      "args": [
-        "test",
-        "${input:sparkdTest}",
-      ],
-      "options": {
-        "cwd": "${workspaceFolder}/build/${input:sparkdCmakePreset}/server"
-      },
-      "icon": {
-        "id": "coverage",
-        "color": "terminal.ansiYellow",
-      },
-    },
-    {
-      "label": "Test Sparkd (Skip Build)",
-      "type": "process",
-      "command": "",
-      "linux": {
-        "command": "sparkd",
-      },
-      "args": [
-        "test",
-        "${input:sparkdTest}",
-      ],
-      "options": {
-        "cwd": "${workspaceFolder}/build/${input:sparkdCmakePreset}/server"
-      },
-      "icon": {
-        "id": "coverage",
-        "color": "terminal.ansiYellow",
-      },
-    },
     {
       "label": "View DataStreamQueryAnalyzer Logs (Last Run)",
       "type": "process",
@@ -915,16 +930,6 @@
       "id": "sparkdConfigQuery",
       "type": "promptString",
       "description": "Config Type(s)"
-    },
-    {
-      "id": "sparkTest",
-      "type": "promptString",
-      "description": "Test to run",
-    },
-    {
-      "id": "sparkdTest",
-      "type": "promptString",
-      "description": "Test to run",
-    },
+    }
   ]
 }

CMakeLists.txt

@@ -1,6 +1,5 @@
 cmake_minimum_required(VERSION 3.12)
 project(spark)
-include(GNUInstallDirs)
 set(CMAKE_CXX_STANDARD 20)
 
 if(WIN32)
@@ -50,6 +49,8 @@ else()
   # make sparkd look for libspark.so in its directory
   set(CMAKE_BUILD_WITH_INSTALL_RPATH TRUE)
   set(CMAKE_INSTALL_RPATH ".")
+  set(CMAKE_INSTALL_PREFIX "${CMAKE_SOURCE_DIR}/install")
+  set(SPARK_INSTALL_PREFIX "${CMAKE_INSTALL_PREFIX}/$<CONFIG>")
 
   if(BUILD_SPARKD)
     # Note: All macros that begin w/ 'CCAPI' are ccapi's macros (defined and/or
@@ -88,15 +89,9 @@ else()
     add_compile_definitions(BINANCE_US_PRICE_STATS_PATH="/api/v3/ticker/24hr")
 
     # ** Binance USDS Futures **
-    add_compile_definitions(CCAPI_BINANCE_USDS_FUTURES_URL_REST_BASE="https://fapi.binance.com")
-    add_compile_definitions(BINANCE_USDS_FUTURES_INSTRUMENTS_PATH="/fapi/v1/exchangeInfo")
-    add_compile_definitions(BINANCE_USDS_FUTURES_PRICE_STATS_PATH="/fapi/v1/ticker/24hr")
-    add_compile_definitions(BINANCE_USDS_FUTURES_USER_DATA_PATH="/api/v3/userDataStream")
-
-    # ** Binance Coin Futures **
-    add_compile_definitions(CCAPI_BINANCE_COIN_FUTURES_URL_REST_BASE="https://dapi.binance.com")
-    add_compile_definitions(BINANCE_COIN_FUTURES_INSTRUMENTS_PATH="/dapi/v1/exchangeInfo")
-    add_compile_definitions(BINANCE_COIN_FUTURES_PRICE_STATS_PATH="/dapi/v1/ticker/24hr")
+    add_compile_definitions(CCAPI_BINANCE_USDS_FUTURES_URL_REST_BASE="https://dapi.binance.com")
+    add_compile_definitions(BINANCE_USDS_FUTURES_INSTRUMENTS_PATH="/dapi/v1/exchangeInfo")
+    add_compile_definitions(BINANCE_USDS_FUTURES_PRICE_STATS_PATH="/dapi/v1/ticker/24hr")
 
     # ** Deribit **
     add_compile_definitions(DERIBIT_INSTRUMENT_PATH="/api/v2/public/get_instrument")
@@ -116,7 +111,6 @@ else()
 
   # subdirs
   if (BUILD_SPARKD)
-    add_subdirectory(src/lib_binance_execution)
     add_subdirectory(src/server/sparkd)
     add_subdirectory(src/data_files)
   endif()
@@ -138,8 +132,6 @@ else()
       COMMAND ln ARGS -sf ${CMAKE_BINARY_DIR}/src/server/sparkd/sparkd ${CMAKE_BINARY_DIR}/server
       COMMAND ln ARGS -sf ${CMAKE_BINARY_DIR}/data_files ${CMAKE_BINARY_DIR}/server
       COMMAND ln ARGS -sf ${CMAKE_BINARY_DIR}/src/lib/libspark.so ${CMAKE_BINARY_DIR}/server
-      COMMAND ln ARGS -sf ${EXTERNALS_ROOT}/lib/${CMAKE_BUILD_TYPE}/libonixs-fix-engine.so.4 ${CMAKE_BINARY_DIR}/server
-      
       COMMAND
         ln ARGS -sf
         ${EXTERNALS_ROOT}/lib/Release/libOnixS.IceImpactMulticastPriceFeedHandlerCpp.so.3.16.0.0
@@ -153,14 +145,11 @@ else()
 
     install(
       DIRECTORY "${CMAKE_BINARY_DIR}/server/plugins"
-      DESTINATION "${CMAKE_INSTALL_BINDIR}")
+      DESTINATION "${SPARK_INSTALL_PREFIX}/bin")
     install(
       FILES
       "${SPARK_LIB}/libACE-5.7.5.so"
       "${SPARK_LIB}/libOnixS.IceImpactMulticastPriceFeedHandlerCpp.so.3.16.0.0"
-      "${SPARK_LIB}/libonixs-fix-engine.so.4.10.1"
-      "${SPARK_LIB}/libonixs-fix-engine.so.4"
-      "${SPARK_LIB}/libonixs-fix-engine.so"
-      DESTINATION "${CMAKE_INSTALL_BINDIR}")
+      DESTINATION "${SPARK_INSTALL_PREFIX}/bin")
   endif()
 endif()

CMakePresets.json

@@ -26,14 +26,12 @@
         "CMAKE_CXX_FLAGS_DEBUG": "$env{SPARK_CMAKE_CXX_FLAGS} /MTd",
         "CMAKE_CXX_FLAGS_RELEASE": "$env{SPARK_CMAKE_CXX_FLAGS} /MT",
         "CMAKE_CXX_STANDARD": "20",
-        "CMAKE_INSTALL_PREFIX": "${sourceDir}/install/${presetName}",
         "CMAKE_SHARED_LINKER_FLAGS_DEBUG": "/DEBUG",
         "CMAKE_SHARED_LINKER_FLAGS_RELEASE": "/OPT:REF /OPT:ICF /LTCG",
         "CMAKE_TOOLCHAIN_FILE": "${sourceDir}/vcpkg/scripts/buildsystems/vcpkg.cmake",
         "EXTERNALS_ROOT": "${sourceDir}/externals/windows",
         "CONFIG_DEFAULTS_PATH": "config/profiles/default/*.conf",
-        "DOTNET_BINARY_DIR": "${sourceDir}/build/$<CONFIG>/bin/net7.0",
-        "DOTNET_WINDOWS_BINARY_DIR": "${sourceDir}/build/$<CONFIG>/bin/net7.0-windows",
+        "DOTNET_BINARY_DIR": "${sourceDir}/build/$<CONFIG>/bin/net7.0;${sourceDir}/build/$<CONFIG>/bin/net7.0-windows",
         "EXPORT_MD_CONSUMER": {
           "type": "BOOL",
           "value": "ON"
@@ -62,7 +60,7 @@
         },
         "CMAKE_CXX_STANDARD": "20",
         "CMAKE_INSTALL_RPATH": ".",
-        "CMAKE_INSTALL_PREFIX": "${sourceDir}/install/${presetName}",
+        "CMAKE_INSTALL_PREFIX": "${sourceDir}/install/${presetName}/sparkd",
         "CMAKE_EXE_LINKER_FLAGS": "-Wno-undef",
         "EXTERNALS_ROOT": "${sourceDir}/externals/linux",
         "BUILD_SDK": {
@@ -223,6 +221,7 @@
         "APP_DATA_DIR": "${dollar}{APPDATA}/bts/DataStream",
         "APP_DATA_DIR_OVERRIDE": "${dollar}{DATA_STREAM_DATA_DIR}",
         "CONFIG_DEFAULTS_PATH": "config/profiles/default/*.conf;config/profiles/data_stream/*.conf",
+        "DOTNET_BINARY_DIR": "${sourceDir}/build/$<CONFIG>/bin/net7.0;${sourceDir}/build/$<CONFIG>/bin/net7.0-windows;${sourceDir}/build/$<CONFIG>/bin/net8.0",
         "SHARED_MEMORY_PREFIX": "data_stream"
       }
     },
@@ -259,7 +258,7 @@
         "CMAKE_CXX_FLAGS_RELEASE": "-fPIC -Wall -pedantic -Wextra -Werror -Wno-missing-braces -Wno-ignored-qualifiers --std=c++20 -fdiagnostics-color -fno-omit-frame-pointer -DNDEBUG -O3",
         "CMAKE_INSTALL_PREFIX": "${sourceDir}/install/${presetName}",
         "CONFIG_DEFAULTS_PATH": "config/profiles/default/*.conf;config/profiles/data_stream/*.conf",
-        "DOTNET_BINARY_DIR": "${sourceDir}/build/$<CONFIG>/bin/net7.0",
+        "DOTNET_BINARY_DIR": "${sourceDir}/build/$<CONFIG>/bin/net7.0;${sourceDir}/build/$<CONFIG>/bin/net8.0",
         "EXPORT_MD_CONSUMER": {
           "type": "BOOL",
           "value": "ON"
@@ -303,7 +302,16 @@
         "lhs": "${hostSystemName}",
         "rhs": "Linux"
       },
-      "hidden": true
+      "hidden": true,
+      "targets": [
+        "data_files_dir",
+        "admin_web_app_bundle",
+        "install",
+        "libspark",
+        "rapidjson_project",
+        "server_layout",
+        "sparkd"
+      ]
     },
 
     {

Spark.Api.Server.sln

@@ -14,34 +14,6 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "spark_test", "src\client\sp
 EndProject
 Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Spark.Console", "src\client\Spark.Console\Spark.Console.csproj", "{89D666E6-B500-498F-B026-4D167F68ADFC}"
 EndProject
-Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "src", "src", "{267F81EF-A8AB-49FD-9F56-D7EC3C1F7C7C}"
-EndProject
-Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "DataStreamSuite", "DataStreamSuite", "{C09ED521-D2DF-40BC-A76F-EB3CB870B2F6}"
-EndProject
-Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "DataStreamCore", "src\DataStreamSuite\DataStreamCore\DataStreamCore.csproj", "{EF26A475-6101-4C9C-A717-D60BB92A2302}"
-EndProject
-Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "SharedStreams", "src\DataStreamSuite\SharedStreams\SharedStreams.csproj", "{911B08A3-515A-441D-84C8-8A43740B5FC1}"
-EndProject
-Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "PubSub", "src\DataStreamSuite\PubSub\PubSub.csproj", "{14140310-2233-430A-821B-F3058A25234B}"
-EndProject
-Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "BaseUtilities", "src\DataStreamSuite\BaseUtilities\BaseUtilities.csproj", "{43146EA8-6FCC-46B8-88B0-42F9BB5E6E0A}"
-EndProject
-Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "ExpressionCore", "src\DataStreamSuite\ExpressionCore\ExpressionCore.csproj", "{6F22DB96-6EAB-4034-99D9-287021C36F01}"
-EndProject
-Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "ArithmeticExpressions", "src\DataStreamSuite\ArithmeticExpressions\ArithmeticExpressions.csproj", "{A5C33956-A41F-49B3-A41D-1082CBA59D02}"
-EndProject
-Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "DataStreamExpressions", "src\DataStreamSuite\DataStreamExpressions\DataStreamExpressions.csproj", "{2FE745C3-1EDF-4421-8CD7-A63EE0C1AE43}"
-EndProject
-Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Bts.ColumnSortExpressions", "src\DataStreamSuite\Bts.ColumnSortExpressions\Bts.ColumnSortExpressions.csproj", "{FDA791E0-2B91-4AD5-B91D-CF920A4C4E38}"
-EndProject
-Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "ColumnMapExpressions", "src\DataStreamSuite\ColumnMapExpressions\ColumnMapExpressions.csproj", "{B945A183-2174-4ACB-B289-8A5C546656FE}"
-EndProject
-Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "BooleanExpressions", "src\DataStreamSuite\BooleanExpressions\BooleanExpressions.csproj", "{9F6D7DE8-185B-4C1D-B809-3314D092DF28}"
-EndProject
-Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Bts.ColumnExpressions", "src\DataStreamSuite\Bts.ColumnExpressions\Bts.ColumnExpressions.csproj", "{6BD3F6D8-B6FC-44E8-B8FE-06352F638B9B}"
-EndProject
-Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "DataStream", "src\DataStreamSuite\DataStream\DataStream.csproj", "{39AEA075-6457-46C0-8821-1158BE6EFA57}"
-EndProject
 Global
 	GlobalSection(SolutionConfigurationPlatforms) = preSolution
 		Debug|x64 = Debug|x64
@@ -98,102 +70,6 @@ Global
 		{89D666E6-B500-498F-B026-4D167F68ADFC}.Release|x64.Build.0 = Release|x64
 		{89D666E6-B500-498F-B026-4D167F68ADFC}.RelWithDebInfo|x64.ActiveCfg = Debug|x64
 		{89D666E6-B500-498F-B026-4D167F68ADFC}.RelWithDebInfo|x64.Build.0 = Debug|x64
-		{EF26A475-6101-4C9C-A717-D60BB92A2302}.Debug|x64.ActiveCfg = Debug|Any CPU
-		{EF26A475-6101-4C9C-A717-D60BB92A2302}.Debug|x64.Build.0 = Debug|Any CPU
-		{EF26A475-6101-4C9C-A717-D60BB92A2302}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
-		{EF26A475-6101-4C9C-A717-D60BB92A2302}.MinSizeRel|x64.Build.0 = Debug|Any CPU
-		{EF26A475-6101-4C9C-A717-D60BB92A2302}.Release|x64.ActiveCfg = Release|Any CPU
-		{EF26A475-6101-4C9C-A717-D60BB92A2302}.Release|x64.Build.0 = Release|Any CPU
-		{EF26A475-6101-4C9C-A717-D60BB92A2302}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
-		{EF26A475-6101-4C9C-A717-D60BB92A2302}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
-		{911B08A3-515A-441D-84C8-8A43740B5FC1}.Debug|x64.ActiveCfg = Debug|Any CPU
-		{911B08A3-515A-441D-84C8-8A43740B5FC1}.Debug|x64.Build.0 = Debug|Any CPU
-		{911B08A3-515A-441D-84C8-8A43740B5FC1}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
-		{911B08A3-515A-441D-84C8-8A43740B5FC1}.MinSizeRel|x64.Build.0 = Debug|Any CPU
-		{911B08A3-515A-441D-84C8-8A43740B5FC1}.Release|x64.ActiveCfg = Release|Any CPU
-		{911B08A3-515A-441D-84C8-8A43740B5FC1}.Release|x64.Build.0 = Release|Any CPU
-		{911B08A3-515A-441D-84C8-8A43740B5FC1}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
-		{911B08A3-515A-441D-84C8-8A43740B5FC1}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
-		{14140310-2233-430A-821B-F3058A25234B}.Debug|x64.ActiveCfg = Debug|Any CPU
-		{14140310-2233-430A-821B-F3058A25234B}.Debug|x64.Build.0 = Debug|Any CPU
-		{14140310-2233-430A-821B-F3058A25234B}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
-		{14140310-2233-430A-821B-F3058A25234B}.MinSizeRel|x64.Build.0 = Debug|Any CPU
-		{14140310-2233-430A-821B-F3058A25234B}.Release|x64.ActiveCfg = Release|Any CPU
-		{14140310-2233-430A-821B-F3058A25234B}.Release|x64.Build.0 = Release|Any CPU
-		{14140310-2233-430A-821B-F3058A25234B}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
-		{14140310-2233-430A-821B-F3058A25234B}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
-		{43146EA8-6FCC-46B8-88B0-42F9BB5E6E0A}.Debug|x64.ActiveCfg = Debug|Any CPU
-		{43146EA8-6FCC-46B8-88B0-42F9BB5E6E0A}.Debug|x64.Build.0 = Debug|Any CPU
-		{43146EA8-6FCC-46B8-88B0-42F9BB5E6E0A}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
-		{43146EA8-6FCC-46B8-88B0-42F9BB5E6E0A}.MinSizeRel|x64.Build.0 = Debug|Any CPU
-		{43146EA8-6FCC-46B8-88B0-42F9BB5E6E0A}.Release|x64.ActiveCfg = Release|Any CPU
-		{43146EA8-6FCC-46B8-88B0-42F9BB5E6E0A}.Release|x64.Build.0 = Release|Any CPU
-		{43146EA8-6FCC-46B8-88B0-42F9BB5E6E0A}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
-		{43146EA8-6FCC-46B8-88B0-42F9BB5E6E0A}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
-		{6F22DB96-6EAB-4034-99D9-287021C36F01}.Debug|x64.ActiveCfg = Debug|Any CPU
-		{6F22DB96-6EAB-4034-99D9-287021C36F01}.Debug|x64.Build.0 = Debug|Any CPU
-		{6F22DB96-6EAB-4034-99D9-287021C36F01}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
-		{6F22DB96-6EAB-4034-99D9-287021C36F01}.MinSizeRel|x64.Build.0 = Debug|Any CPU
-		{6F22DB96-6EAB-4034-99D9-287021C36F01}.Release|x64.ActiveCfg = Release|Any CPU
-		{6F22DB96-6EAB-4034-99D9-287021C36F01}.Release|x64.Build.0 = Release|Any CPU
-		{6F22DB96-6EAB-4034-99D9-287021C36F01}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
-		{6F22DB96-6EAB-4034-99D9-287021C36F01}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
-		{A5C33956-A41F-49B3-A41D-1082CBA59D02}.Debug|x64.ActiveCfg = Debug|Any CPU
-		{A5C33956-A41F-49B3-A41D-1082CBA59D02}.Debug|x64.Build.0 = Debug|Any CPU
-		{A5C33956-A41F-49B3-A41D-1082CBA59D02}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
-		{A5C33956-A41F-49B3-A41D-1082CBA59D02}.MinSizeRel|x64.Build.0 = Debug|Any CPU
-		{A5C33956-A41F-49B3-A41D-1082CBA59D02}.Release|x64.ActiveCfg = Release|Any CPU
-		{A5C33956-A41F-49B3-A41D-1082CBA59D02}.Release|x64.Build.0 = Release|Any CPU
-		{A5C33956-A41F-49B3-A41D-1082CBA59D02}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
-		{A5C33956-A41F-49B3-A41D-1082CBA59D02}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
-		{2FE745C3-1EDF-4421-8CD7-A63EE0C1AE43}.Debug|x64.ActiveCfg = Debug|Any CPU
-		{2FE745C3-1EDF-4421-8CD7-A63EE0C1AE43}.Debug|x64.Build.0 = Debug|Any CPU
-		{2FE745C3-1EDF-4421-8CD7-A63EE0C1AE43}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
-		{2FE745C3-1EDF-4421-8CD7-A63EE0C1AE43}.MinSizeRel|x64.Build.0 = Debug|Any CPU
-		{2FE745C3-1EDF-4421-8CD7-A63EE0C1AE43}.Release|x64.ActiveCfg = Release|Any CPU
-		{2FE745C3-1EDF-4421-8CD7-A63EE0C1AE43}.Release|x64.Build.0 = Release|Any CPU
-		{2FE745C3-1EDF-4421-8CD7-A63EE0C1AE43}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
-		{2FE745C3-1EDF-4421-8CD7-A63EE0C1AE43}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
-		{FDA791E0-2B91-4AD5-B91D-CF920A4C4E38}.Debug|x64.ActiveCfg = Debug|Any CPU
-		{FDA791E0-2B91-4AD5-B91D-CF920A4C4E38}.Debug|x64.Build.0 = Debug|Any CPU
-		{FDA791E0-2B91-4AD5-B91D-CF920A4C4E38}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
-		{FDA791E0-2B91-4AD5-B91D-CF920A4C4E38}.MinSizeRel|x64.Build.0 = Debug|Any CPU
-		{FDA791E0-2B91-4AD5-B91D-CF920A4C4E38}.Release|x64.ActiveCfg = Release|Any CPU
-		{FDA791E0-2B91-4AD5-B91D-CF920A4C4E38}.Release|x64.Build.0 = Release|Any CPU
-		{FDA791E0-2B91-4AD5-B91D-CF920A4C4E38}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
-		{FDA791E0-2B91-4AD5-B91D-CF920A4C4E38}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
-		{B945A183-2174-4ACB-B289-8A5C546656FE}.Debug|x64.ActiveCfg = Debug|Any CPU
-		{B945A183-2174-4ACB-B289-8A5C546656FE}.Debug|x64.Build.0 = Debug|Any CPU
-		{B945A183-2174-4ACB-B289-8A5C546656FE}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
-		{B945A183-2174-4ACB-B289-8A5C546656FE}.MinSizeRel|x64.Build.0 = Debug|Any CPU
-		{B945A183-2174-4ACB-B289-8A5C546656FE}.Release|x64.ActiveCfg = Release|Any CPU
-		{B945A183-2174-4ACB-B289-8A5C546656FE}.Release|x64.Build.0 = Release|Any CPU
-		{B945A183-2174-4ACB-B289-8A5C546656FE}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
-		{B945A183-2174-4ACB-B289-8A5C546656FE}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
-		{39AEA075-6457-46C0-8821-1158BE6EFA57}.Debug|x64.ActiveCfg = Debug|Any CPU
-		{39AEA075-6457-46C0-8821-1158BE6EFA57}.Debug|x64.Build.0 = Debug|Any CPU
-		{39AEA075-6457-46C0-8821-1158BE6EFA57}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
-		{39AEA075-6457-46C0-8821-1158BE6EFA57}.MinSizeRel|x64.Build.0 = Debug|Any CPU
-		{39AEA075-6457-46C0-8821-1158BE6EFA57}.Release|x64.ActiveCfg = Release|Any CPU
-		{39AEA075-6457-46C0-8821-1158BE6EFA57}.Release|x64.Build.0 = Release|Any CPU
-		{39AEA075-6457-46C0-8821-1158BE6EFA57}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
-		{39AEA075-6457-46C0-8821-1158BE6EFA57}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
-		{9F6D7DE8-185B-4C1D-B809-3314D092DF28}.Debug|x64.ActiveCfg = Debug|Any CPU
-		{9F6D7DE8-185B-4C1D-B809-3314D092DF28}.Debug|x64.Build.0 = Debug|Any CPU
-		{9F6D7DE8-185B-4C1D-B809-3314D092DF28}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
-		{9F6D7DE8-185B-4C1D-B809-3314D092DF28}.MinSizeRel|x64.Build.0 = Debug|Any CPU
-		{9F6D7DE8-185B-4C1D-B809-3314D092DF28}.Release|x64.ActiveCfg = Release|Any CPU
-		{9F6D7DE8-185B-4C1D-B809-3314D092DF28}.Release|x64.Build.0 = Release|Any CPU
-		{9F6D7DE8-185B-4C1D-B809-3314D092DF28}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
-		{9F6D7DE8-185B-4C1D-B809-3314D092DF28}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
-		{6BD3F6D8-B6FC-44E8-B8FE-06352F638B9B}.Debug|x64.ActiveCfg = Debug|Any CPU
-		{6BD3F6D8-B6FC-44E8-B8FE-06352F638B9B}.Debug|x64.Build.0 = Debug|Any CPU
-		{6BD3F6D8-B6FC-44E8-B8FE-06352F638B9B}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
-		{6BD3F6D8-B6FC-44E8-B8FE-06352F638B9B}.MinSizeRel|x64.Build.0 = Debug|Any CPU
-		{6BD3F6D8-B6FC-44E8-B8FE-06352F638B9B}.Release|x64.ActiveCfg = Release|Any CPU
-		{6BD3F6D8-B6FC-44E8-B8FE-06352F638B9B}.Release|x64.Build.0 = Release|Any CPU
-		{6BD3F6D8-B6FC-44E8-B8FE-06352F638B9B}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
-		{6BD3F6D8-B6FC-44E8-B8FE-06352F638B9B}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
 	EndGlobalSection
 	GlobalSection(SolutionProperties) = preSolution
 		HideSolutionNode = FALSE
@@ -204,19 +80,4 @@ Global
 	GlobalSection(Performance) = preSolution
 		HasPerformanceSessions = true
 	EndGlobalSection
-	GlobalSection(NestedProjects) = preSolution
-		{C09ED521-D2DF-40BC-A76F-EB3CB870B2F6} = {267F81EF-A8AB-49FD-9F56-D7EC3C1F7C7C}
-		{EF26A475-6101-4C9C-A717-D60BB92A2302} = {C09ED521-D2DF-40BC-A76F-EB3CB870B2F6}
-		{911B08A3-515A-441D-84C8-8A43740B5FC1} = {C09ED521-D2DF-40BC-A76F-EB3CB870B2F6}
-		{14140310-2233-430A-821B-F3058A25234B} = {C09ED521-D2DF-40BC-A76F-EB3CB870B2F6}
-		{43146EA8-6FCC-46B8-88B0-42F9BB5E6E0A} = {C09ED521-D2DF-40BC-A76F-EB3CB870B2F6}
-		{6F22DB96-6EAB-4034-99D9-287021C36F01} = {C09ED521-D2DF-40BC-A76F-EB3CB870B2F6}
-		{A5C33956-A41F-49B3-A41D-1082CBA59D02} = {C09ED521-D2DF-40BC-A76F-EB3CB870B2F6}
-		{2FE745C3-1EDF-4421-8CD7-A63EE0C1AE43} = {C09ED521-D2DF-40BC-A76F-EB3CB870B2F6}
-		{FDA791E0-2B91-4AD5-B91D-CF920A4C4E38} = {C09ED521-D2DF-40BC-A76F-EB3CB870B2F6}
-		{B945A183-2174-4ACB-B289-8A5C546656FE} = {C09ED521-D2DF-40BC-A76F-EB3CB870B2F6}
-		{39AEA075-6457-46C0-8821-1158BE6EFA57} = {C09ED521-D2DF-40BC-A76F-EB3CB870B2F6}
-		{9F6D7DE8-185B-4C1D-B809-3314D092DF28} = {C09ED521-D2DF-40BC-A76F-EB3CB870B2F6}
-		{6BD3F6D8-B6FC-44E8-B8FE-06352F638B9B} = {C09ED521-D2DF-40BC-A76F-EB3CB870B2F6}
-	EndGlobalSection
 EndGlobal

cmake/ProtobufGenerateFs.cmake

@@ -1,49 +0,0 @@
-include_guard(DIRECTORY)
-
-function(protobuf_generate_fs)
-  set(_options)
-  set(_one_value_args TARGET_NAME OUT_DIR PROTOC)
-  set(_multi_value_args PROTO_INCLUDE_DIRS PROTO_SOURCES)
-  cmake_parse_arguments(_ARGS "${_options}" "${_one_value_args}" "${_multi_value_args}" ${ARGN})
-
-  if (NOT DEFINED _ARGS_TARGET_NAME)
-    set(_ARGS_TARGET_NAME "${PROJECT_NAME}_generate_fs_protos")
-  endif()
-  if (NOT DEFINED _ARGS_PROTOC)
-    set(_ARGS_PROTOC "${SPARK_BIN}/protoc")
-  endif()
-
-  set(_proto_include_cli_args)
-  foreach(_dir ${_ARGS_PROTO_INCLUDE_DIRS})
-    list(APPEND _proto_include_cli_args -I ${_dir})
-  endforeach()
-
-  set(_outputs)
-  foreach(_proto_source ${_ARGS_PROTO_SOURCES})
-    # get_filename_component(_abs_proto_source ${_proto_source} ABSOLUTE)
-    get_filename_component(_proto_source_we ${_proto_source} NAME_WE)
-    # get_filename_component(_proto_source_dir ${_proto_source} DIRECTORY)
-    list(APPEND _outputs ${_ARGS_OUT_DIR}/${_proto_source_we}.fable.fs)
-  endforeach()
-
-  set(_venv_exec ${CMAKE_SOURCE_DIR}/tools/venv_exec.sh)
-  set(_plugin_py ${CMAKE_SOURCE_DIR}/src/lib/api/service_compiler/plugin.py)
-  set(_template_fs ${CMAKE_SOURCE_DIR}/src/lib/api/service_compiler/templates/fs/fable.fs)
-  add_custom_command(
-    OUTPUT ${_outputs}
-    WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}
-    COMMAND ${_venv_exec}
-      ${_ARGS_PROTOC}
-      --plugin=protoc-gen-service=${_plugin_py}
-      --service_out=lang=fs,strip_out_path=1:${_ARGS_OUT_DIR}
-      ${_proto_include_cli_args}
-      ${_ARGS_PROTO_SOURCES}
-    DEPENDS
-      ${_venv_exec}
-      ${_plugin_py}
-      ${_template_fs}
-      ${CMAKE_CURRENT_SOURCE_DIR}/package-lock.json
-      ${_ARGS_PROTO_SOURCES}
-  )
-  add_custom_target(${_ARGS_TARGET_NAME} ALL DEPENDS "${_outputs}")
-endfunction()

cmake/ProtobufGenerateJs.cmake

@@ -1,33 +0,0 @@
-include_guard(DIRECTORY)
-
-function(protobuf_generate_js)
-  set(_options)
-  set(_one_value_args TARGET_NAME OUTPUT)
-  set(_multi_value_args PROTO_INCLUDE_DIRS PROTO_SOURCES)
-  cmake_parse_arguments(_ARGS "${_options}" "${_one_value_args}" "${_multi_value_args}" ${ARGN})
-
-  if (NOT DEFINED _ARGS_TARGET_NAME)
-    set(_ARGS_TARGET_NAME "${PROJECT_NAME}_generate_js_protos")
-  endif()
-
-  set(_proto_include_cli_args)
-  foreach(_dir ${_ARGS_PROTO_INCLUDE_DIRS})
-    list(APPEND _proto_include_cli_args -p ${_dir})
-  endforeach()
-
-  add_custom_command(
-    OUTPUT ${_ARGS_OUTPUT}
-    WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}
-    COMMAND npx
-      --no-install pbjs
-      -t json-module
-      -w commonjs
-      -o ${_ARGS_OUTPUT}
-      ${_proto_include_cli_args}
-      ${_ARGS_PROTO_SOURCES}
-    DEPENDS
-      ${CMAKE_CURRENT_SOURCE_DIR}/package-lock.json
-      ${_ARGS_PROTO_SOURCES}
-  )
-  add_custom_target(${_ARGS_TARGET_NAME} ALL DEPENDS ${_ARGS_OUTPUT})
-endfunction()

cmake/SymLink.cmake

@@ -1,49 +0,0 @@
-include_guard(DIRECTORY)
-include(${CMAKE_SOURCE_DIR}/cmake/TernaryOperator.cmake)
-
-function(sym_link_dir)
-  set(_options)
-  set(_one_value_args TARGET_NAME SOURCE DEST COMMENT)
-  set(_multi_value_args)
-  cmake_parse_arguments(_ARGS "${_options}" "${_one_value_args}" "${_multi_value_args}" ${ARGN})
-  add_custom_command(
-    OUTPUT ${_ARGS_DEST}
-    COMMAND ln ARGS -sf ${_ARGS_SOURCE} ${_ARGS_DEST}
-  )
-  set_ternary(_comment _ARGS_COMMENT "${_ARGS_COMMENT}" "linking dir ${_ARGS_SOURCE} -> ${_ARGS_DEST}")
-  add_custom_target(
-    ${_ARGS_TARGET_NAME} ALL
-    DEPENDS ${_ARGS_DEST}
-    COMMENT "${_comment}"
-  )
-endfunction()
-
-function(sym_link_file)
-  set(_options)
-  set(_one_value_args SOURCE DEST INSTALL_DEST)
-  set(_multi_value_args)
-  cmake_parse_arguments(_ARGS "${_options}" "${_one_value_args}" "${_multi_value_args}" ${ARGN})
-  file(
-    CREATE_LINK
-      ${_ARGS_SOURCE}
-      ${_ARGS_DEST}
-    SYMBOLIC
-  )
-  if (DEFINED _ARGS_INSTALL_DEST)
-    install(
-      FILES
-      ${_ARGS_SOURCE}
-      DESTINATION ${_ARGS_INSTALL_DEST}
-    )
-  endif()
-endfunction()
-
-function(sym_link_local_lib LIB_NAME)
-  set(_lib_file_name "${CMAKE_SHARED_LIBRARY_PREFIX}${LIB_NAME}.so")
-  file(
-    CREATE_LINK
-      "${CMAKE_BINARY_DIR}/src/lib_${LIB_NAME}/${_lib_file_name}"
-      "${CMAKE_CURRENT_BINARY_DIR}/${_lib_file_name}"
-    SYMBOLIC
-  )
-endfunction()

cmake/TernaryOperator.cmake

@@ -1,9 +0,0 @@
-include_guard(GLOBAL)
-
-macro(set_ternary RESULT CONDITION TRUE_VALUE FALSE_VALUE)
-  if(${CONDITION})
-    set(${RESULT} ${TRUE_VALUE})
-  else()
-    set(${RESULT} ${FALSE_VALUE})
-  endif()
-endmacro()

src/DataStreamSuite/ArithmeticExpressions/ArithmeticExpressionGrammar.y

@@ -8,7 +8,6 @@
    public int Integer;
    public double Real;
    public string Identifier;
-   public DateTime DateTime; 
    public ImmutableList<IArithmeticExpression> Arguments;
    public ImmutableList<Assignment> Assignments;
    public IArithmeticExpression Argument;

src/DataStreamSuite/BlueRegistry/DoubleRegistryHandler.cs

@@ -1,4 +1,6 @@
 using System;
+using System.Runtime.Versioning;
+
 using Microsoft.Win32;
 
 namespace BCG
@@ -10,6 +12,7 @@ namespace BCG
          rk.SetValue (name, value.ToString ());
       }
 
+      [SupportedOSPlatform("windows")]
       public static double getValue (RegistryKey rk, string name)
       {
          object v = rk.GetValue (name);

src/DataStreamSuite/Bts.DataStreamWebApp.Client/.eslintrc.cjs

@@ -0,0 +1,20 @@
+module.exports = {
+  root: true,
+  env: { browser: true, es2020: true },
+  extends: [
+    'eslint:recommended',
+    'plugin:react/recommended',
+    'plugin:react/jsx-runtime',
+    'plugin:react-hooks/recommended',
+  ],
+  ignorePatterns: ['dist', '.eslintrc.cjs'],
+  parserOptions: { ecmaVersion: 'latest', sourceType: 'module' },
+  settings: { react: { version: '18.3.1' } },
+  plugins: ['react-refresh'],
+  rules: {
+    'react-refresh/only-export-components': [
+      'warn',
+      { allowConstantExport: true },
+    ],
+  },
+}

src/DataStreamSuite/Bts.DataStreamWebApp.Client/.gitignore

@@ -0,0 +1,24 @@
+# Logs
+logs
+*.log
+npm-debug.log*
+yarn-debug.log*
+yarn-error.log*
+pnpm-debug.log*
+lerna-debug.log*
+
+node_modules
+dist
+dist-ssr
+*.local
+
+# Editor directories and files
+.vscode/*
+!.vscode/extensions.json
+.idea
+.DS_Store
+*.suo
+*.ntvs*
+*.njsproj
+*.sln
+*.sw?

src/DataStreamSuite/Bts.DataStreamWebApp.Client/Bts.DataStreamWebApp.Client.esproj

@@ -0,0 +1,11 @@
+<Project Sdk="Microsoft.VisualStudio.JavaScript.Sdk/0.5.128-alpha">
+  <PropertyGroup>
+    <StartupCommand>npm run dev</StartupCommand>
+    <JavaScriptTestRoot>src\</JavaScriptTestRoot>
+    <JavaScriptTestFramework>Jest</JavaScriptTestFramework>
+    <!-- Allows the build (or compile) script located on package.json to run on Build -->
+    <ShouldRunBuildScript>true</ShouldRunBuildScript>
+    <!-- Folder where production build objects will be placed -->
+    <PublishAssetsDirectory>$(MSBuildProjectDirectory)\distill</PublishAssetsDirectory>
+  </PropertyGroup>
+</Project>

src/DataStreamSuite/Bts.DataStreamWebApp.Client/README.md

@@ -0,0 +1,8 @@
+# React + Vite
+
+This template provides a minimal setup to get React working in Vite with HMR and some ESLint rules.
+
+Currently, two official plugins are available:
+
+- [@vitejs/plugin-react](https://github.com/vitejs/vite-plugin-react/blob/main/packages/plugin-react/README.md) uses [Babel](https://babeljs.io/) for Fast Refresh
+- [@vitejs/plugin-react-swc](https://github.com/vitejs/vite-plugin-react-swc) uses [SWC](https://swc.rs/) for Fast Refresh

src/DataStreamSuite/Bts.DataStreamWebApp.Client/index.html

@@ -0,0 +1,13 @@
+<!doctype html>
+<html lang="en">
+  <head>
+    <meta charset="UTF-8" />
+    <link rel="icon" type="image/ico" href="/logo.ico" />
+    <meta name="viewport" content="width=device-width, initial-scale=1.0" />
+    <title>BTS Cloud Streams</title>
+  </head>
+  <body>
+    <div id="root"></div>
+    <script type="module" src="/src/main.jsx"></script>
+  </body>
+</html>

src/DataStreamSuite/Bts.DataStreamWebApp.Client/nuget.config

@@ -0,0 +1,10 @@
+<?xml version="1.0" encoding="utf-8"?>
+<configuration>
+  <packageSources>
+    <clear />
+    <add key="nuget.org" value="https://api.nuget.org/v3/index.json" />
+  </packageSources>
+  <disabledPackageSources>
+    <clear />
+  </disabledPackageSources>
+</configuration>

src/DataStreamSuite/Bts.DataStreamWebApp.Client/package.json

@@ -0,0 +1,38 @@
+{
+  "name": "bts.datastreamwebapp.client",
+  "private": true,
+  "version": "0.0.0",
+  "type": "module",
+  "scripts": {
+    "dev": "vite",
+    "build": "vite build",
+    "lint": "eslint . --ext js,jsx --report-unused-disable-directives --max-warnings 0",
+    "preview": "vite preview"
+  },
+  "dependencies": {
+    "@angular/material": "^18.0.4",
+    "@emotion/react": "^11.11.4",
+    "@emotion/styled": "^11.11.5",
+    "@fontsource/noto-sans": "^5.0.22",
+    "@material/data-table": "^15.0.0-canary.7f224ddd4.0",
+    "@mui/icons-material": "^5.15.20",
+    "@mui/material": "^5.15.20",
+    "@mui/x-data-grid": "^7.7.1",
+    "bulma": "^1.0.1",
+    "react": "^18.3.1",
+    "react-data-table-component": "^7.6.2",
+    "react-dom": "^18.3.1",
+    "react-use-websocket": "^4.8.1",
+    "rpc-websockets": "^7.11.2"
+  },
+  "devDependencies": {
+    "@types/react": "^18.3.3",
+    "@types/react-dom": "^18.3.0",
+    "@vitejs/plugin-react": "^4.3.1",
+    "eslint": "^8.57.0",
+    "eslint-plugin-react": "^7.34.2",
+    "eslint-plugin-react-hooks": "^4.6.2",
+    "eslint-plugin-react-refresh": "^0.4.7",
+    "vite": "^5.2.13"
+  }
+}

src/DataStreamSuite/Bts.DataStreamWebApp.Client/public/vite.svg

@@ -0,0 +1 @@
+<svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" class="iconify iconify--logos" width="31.88" height="32" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 257"><defs><linearGradient id="IconifyId1813088fe1fbc01fb466" x1="-.828%" x2="57.636%" y1="7.652%" y2="78.411%"><stop offset="0%" stop-color="#41D1FF"></stop><stop offset="100%" stop-color="#BD34FE"></stop></linearGradient><linearGradient id="IconifyId1813088fe1fbc01fb467" x1="43.376%" x2="50.316%" y1="2.242%" y2="89.03%"><stop offset="0%" stop-color="#FFEA83"></stop><stop offset="8.333%" stop-color="#FFDD35"></stop><stop offset="100%" stop-color="#FFA800"></stop></linearGradient></defs><path fill="url(#IconifyId1813088fe1fbc01fb466)" d="M255.153 37.938L134.897 252.976c-2.483 4.44-8.862 4.466-11.382.048L.875 37.958c-2.746-4.814 1.371-10.646 6.827-9.67l120.385 21.517a6.537 6.537 0 0 0 2.322-.004l117.867-21.483c5.438-.991 9.574 4.796 6.877 9.62Z"></path><path fill="url(#IconifyId1813088fe1fbc01fb467)" d="M185.432.063L96.44 17.501a3.268 3.268 0 0 0-2.634 3.014l-5.474 92.456a3.268 3.268 0 0 0 3.997 3.378l24.777-5.718c2.318-.535 4.413 1.507 3.936 3.838l-7.361 36.047c-.495 2.426 1.782 4.5 4.151 3.78l15.304-4.649c2.372-.72 4.652 1.36 4.15 3.788l-11.698 56.621c-.732 3.542 3.979 5.473 5.943 2.437l1.313-2.028l72.516-144.72c1.215-2.423-.88-5.186-3.54-4.672l-25.505 4.922c-2.396.462-4.435-1.77-3.759-4.114l16.646-57.705c.677-2.35-1.37-4.583-3.769-4.113Z"></path></svg>

src/DataStreamSuite/Bts.DataStreamWebApp.Client/src/App.css

@@ -0,0 +1,28 @@
+@use "@material/checkbox";
+@use "@material/icon-button";
+@use "@material/data-table/data-table";
+
+h1#main-header {
+  padding: 4px 8px 4px 8px;
+  text-align: center;
+}
+
+td:hover {
+  background-color: cadetblue;
+}
+
+td#cell-decimal {
+  text-align: right;
+}
+
+td#cell-integer {
+  text-align: right;
+}
+
+td#cell-string {
+  text-align: center;
+}
+
+td#cell-date-time {
+  text-align: center;
+}

src/DataStreamSuite/Bts.DataStreamWebApp.Client/src/App.jsx

@@ -0,0 +1,24 @@
+import { createTheme, ThemeProvider, CssBaseline } from '@mui/material';
+import useMediaQuery from '@mui/material/useMediaQuery';
+import QueryControl from './QueryControl';
+import getTheme from './Theme';
+import '@fontsource/noto-sans';
+import './App.css';
+
+const App = () => {
+  const prefersDarkMode = useMediaQuery('(prefers-color-scheme: dark)');
+  const theme = getTheme({ mode: prefersDarkMode ? 'dark' : 'light' });
+
+  return (
+    <ThemeProvider theme={theme}>
+      <CssBaseline />
+      <h1 id='main-header' style={{
+        color: theme.palette.text.primary,
+        fontFamily: 'Noto Sans',
+      }}>BTS Cloud Streams</h1>
+      <QueryControl websocketUrl="ws://localhost:5000/ws" theme={theme} />
+    </ThemeProvider>
+  );
+};
+
+export default App;

src/DataStreamSuite/Bts.DataStreamWebApp.Client/src/DataTable.css

@@ -0,0 +1,32 @@
+@use "@material/checkbox";
+@use "@material/icon-button";
+@use "@material/data-table/data-table";
+
+th {
+  font-size: large;  
+  padding: 15px;
+}
+
+td {
+  padding: 15px;
+}
+
+td:hover {
+  background-color: cadetblue;
+}
+
+td#cell-decimal {
+  text-align: right;
+}
+
+td#cell-integer {
+  text-align: right;
+}
+
+td#cell-string {
+  text-align: center;
+}
+
+td#cell-date-time {
+  text-align: center;
+}

src/DataStreamSuite/Bts.DataStreamWebApp.Client/src/DataTable.jsx

@@ -0,0 +1,52 @@
+import PropTypes from 'prop-types';
+import { DataGrid } from '@mui/x-data-grid';
+
+function getGridColDef(col) {
+  return {
+    field: col.name,
+    headerName: col.name,
+  };
+}
+
+export default function DataTable({ columns, rows }) {
+  return (
+    <DataGrid
+      columns={columns.map(getGridColDef)}
+      rows={rows}
+      sx={{
+        borderColor: 'primary.main',
+        backgroundColor: 'primary.dark',
+        color: 'text.primary',
+        '& .MuiDataGrid-columnHeader': {
+          color: 'primary.main',
+          fontWeight: 700,
+          backgroundColor: 'primary.dark'
+        },
+        '& .MuiDataGrid-filler': {
+          color: 'primary.main',
+          fontWeight: 700,
+          backgroundColor: 'primary.dark'
+        },
+        '& .MuiDataGrid-cell': {
+          color: 'text.primary',
+        },
+        '& .MuiDataGrid-cell:hover': {
+          color: 'primary.main',
+        },
+        '& .MuiDataGrid-row': {
+          '--DataGrid-rowBorderColor': 'secondary.dark'
+        }
+      }}
+    />
+  );
+}
+
+DataTable.propTypes = {
+  columns: PropTypes.arrayOf(
+    PropTypes.shape({
+      name: PropTypes.string.isRequired,
+      type: PropTypes.string.isRequired,
+    })
+  ).isRequired,
+  rows: PropTypes.arrayOf(PropTypes.array).isRequired,
+};

src/DataStreamSuite/Bts.DataStreamWebApp.Client/src/QueryControl.jsx

@@ -0,0 +1,100 @@
+import PropTypes from 'prop-types';
+import DataTable from './DataTable';
+import QueryInputForm from './QueryInputForm';
+import Container from '@mui/material/Container';
+import React from 'react';
+
+var rpcReqIdCounter = 1;
+
+function subscribeReq(query) {
+  return {
+    jsonrpc: "2.0",
+    id: rpcReqIdCounter++,
+    method: "subscribe",
+    params: {
+      subscription_id: "1",
+      query: query
+    }
+  };
+}
+
+function unsubscribeReq() {
+  return {
+    jsonrpc: "2.0",
+    id: rpcReqIdCounter++,
+    method: "unsubscribe",
+    params: {
+      subscription_id: "1",
+    }
+  };
+}
+
+class QueryControl extends React.Component {
+  constructor({ websocketUrl, theme }) {
+    super(websocketUrl);
+    console.log('Connecting to WebSocket:', websocketUrl);
+    this.state = {
+      rows: null,
+      ws: new WebSocket(websocketUrl),
+      columns: null,
+      isRunning: false,
+      theme: theme,
+    };
+    this.state.ws.onopen = () => {
+      console.info('WebSocket open');
+    };
+    this.state.ws.onclose = () => {
+      console.info('WebSocket closed');
+    };
+    this.state.ws.onmessage = e => {
+      console.info(`Received message: ${e.data}`);
+      const msg = JSON.parse(e.data);
+      if (msg.id != null) {
+        let newState = { ...this.state };
+        newState.isRunning = msg.result.status == 'Subscribed';
+        this.setState(newState);
+      } else {
+        let newState = { ...this.state };
+        newState.rows = msg.params.all;
+        console.info(`Received rows: ${newState.rows}`);
+        newState.columns = msg.params.columns;
+        this.setState(newState);
+      }
+    };
+
+    this.start = (query) => {
+      this.state.ws.send(JSON.stringify(subscribeReq(query)));
+    };
+
+    this.stop = () => {
+      this.state.ws.send(JSON.stringify(unsubscribeReq()));
+    };
+  }
+
+  componentWillUnmount() {
+    if (this.state.ws) {
+      console.warn('Closing websocket because component is unmounting');
+      this.state.ws.close();
+    }
+  }
+
+  render() {
+    console.info('Rendering');
+    return (
+      <Container>
+        <QueryInputForm isRunning={this.state.isRunning} onStart={this.start} onStop={this.stop} />
+        {
+          this.state.rows == null ? <p style={{ color: this.state.theme.palette.warning.main }}>No data received</p> :
+            <DataTable columns={this.state.columns} rows={this.state.rows} />
+        }
+      </Container>
+    );
+  }
+}
+
+QueryControl.propTypes = {
+  websocketUrl: PropTypes.string.isRequired,
+  theme: PropTypes.object.isRequired,
+};
+
+export default QueryControl;

src/DataStreamSuite/Bts.DataStreamWebApp.Client/src/QueryInputForm.jsx

@@ -0,0 +1,93 @@
+import { useRef } from 'react';
+import Box from '@mui/material/Box';
+import Button from '@mui/material/Button';
+import Grid from '@mui/material/Grid';
+import Paper from '@mui/material/Paper';
+import PlayArrowIcon from '@mui/icons-material/PlayArrow';
+import PropTypes from 'prop-types';
+import StopIcon from '@mui/icons-material/Stop';
+import TextField from '@mui/material/TextField';
+import Tooltip from '@mui/material/Tooltip';
+
+const QueryInputForm = ({ isRunning, onStart, onStop }) => {
+  let textInput = useRef(null);
+
+  const onClearClick = () => {
+    textInput.current.value = '';
+  };
+
+  const onRunClick = () => {
+    onStart(textInput.current.value);
+  };
+
+  const onStopClick = () => {
+    onStop();
+  };
+
+  const onKeyDown = (e) => {
+    // Detect keyboard shortcuts.
+    //   Ctrl+Enter : Run
+    if (e.ctrlKey && e.key === 'Enter') {
+      onRunClick();
+    }
+  };
+
+  return (
+    <Box sx={{ p: 2 }}>
+      <Paper elevation={3} sx={{ color: 'primary.main' }}>
+        <Box sx={{ p: 2 }}>
+          <TextField
+            fullWidth
+            multiline
+            rows={10}
+            variant="outlined"
+            onKeyDown={onKeyDown}
+            inputRef={textInput}
+            placeholder="Enter Query..."
+          />
+        </Box>
+        <Box sx={{ p: 2, borderTop: 1 }}>
+          <Grid container spacing={2} justifyContent="flex-start">
+            <Grid item xs>
+              <Tooltip title="Clear Query Text">
+                <Button onClick={onClearClick} sx={{ padding: '11px', color: 'text.dark' }}>
+                  Clear
+                </Button>
+              </Tooltip>
+            </Grid>
+            <Grid item>
+              <Tooltip title="Run (Ctrl+Enter)">
+                <Button onClick={onRunClick} sx={{
+                  color: `query_start_button.${isRunning ? 'active' : 'inactive'}.color`,
+                  bgcolor: `query_start_button.${isRunning ? 'active' : 'inactive'}.background_color`,
+                  padding: '11px'
+                }}>
+                  <PlayArrowIcon />
+                </Button>
+              </Tooltip>
+            </Grid>
+            <Grid item>
+              <Tooltip title="Stop">
+                <Button onClick={onStopClick} sx={{
+                  color: `query_stop_button.${isRunning ? 'inactive' : 'active'}.color`,
+                  bgcolor: `query_stop_button.${isRunning ? 'inactive' : 'active'}.background_color`,
+                  padding: '11px'
+                }}>
+                  <StopIcon />
+                </Button>
+              </Tooltip>
+            </Grid>
+          </Grid>
+        </Box>
+      </Paper>
+    </Box>
+  );
+};
+
+QueryInputForm.propTypes = {
+  isRunning: PropTypes.bool.isRequired,
+  onStart: PropTypes.func.isRequired,
+  onStop: PropTypes.func.isRequired,
+};
+
+export default QueryInputForm;

src/DataStreamSuite/Bts.DataStreamWebApp.Client/src/Theme.js

@@ -0,0 +1,88 @@
+import { createTheme } from '@mui/material/styles';
+import PropTypes from 'prop-types';
+import '@fontsource/noto-sans';
+import './App.css';
+
+const getLightPalette = () => {
+  return {
+    mode: 'light',
+    primary: {
+      light: '#6573c3',
+      main: '#3f51b5',
+      dark: '#2c387e',
+      contrastText: '#ffffff',
+    },
+    secondary: {
+      light: '#f73378',
+      main: '#f50057',
+      dark: '#ab003c',
+      contrastText: '#ffffff',
+    },
+    text: {
+      primary: '#000000c4',
+      secondary: '#00000086',
+      disabled: '#0000005b',
+      hint: '#0000005b',
+    }
+  };
+};
+
+const getDarkPalette = () => {
+  return {
+    mode: 'dark',
+    primary: {
+      light: '#bcc5ff',
+      main: '#33baf9',
+      dark: '#202334',
+      contrastText: '#78a2ac',
+    },
+    secondary: {
+      light: '#9fd0b8',
+      main: '#5eb289',
+      dark: '#ffffff12',
+      contrastText: '#c8d4f7',
+    },
+    text: {
+      primary: '#eaf8f8',
+      dark: '#eaf8f8a0'
+    },
+    query_start_button: {
+      active: {
+        color: '#33baf9',
+        background_color: '#ffffff40',
+      },
+      inactive: {
+        color: '#ffffff40',
+        background_color: '#ffffff00',
+      }
+    },
+    warning: {
+      main: '#c4823f'
+    },
+    query_stop_button: {
+      active: {
+        color: '#ff4444a0',
+        background_color: '#ffffff40',
+      },
+      inactive: {
+        color: '#ff444450',
+        background_color: '#ffffff00',
+      }
+    },
+  };
+};
+
+const getTheme = ({ mode }) => {
+  return createTheme({
+    typography: {
+      fontFamily: '"Helvetica"'
+    },
+    palette: mode == 'dark' ? getDarkPalette() : getLightPalette()
+  });
+};
+
+export default getTheme;
+
+getTheme.propTypes = {
+  mode: PropTypes.string.isRequired,
+};

src/DataStreamSuite/Bts.DataStreamWebApp.Client/src/assets/react.svg

@@ -0,0 +1 @@
+<svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" class="iconify iconify--logos" width="35.93" height="32" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 228"><path fill="#00D8FF" d="M210.483 73.824a171.49 171.49 0 0 0-8.24-2.597c.465-1.9.893-3.777 1.273-5.621c6.238-30.281 2.16-54.676-11.769-62.708c-13.355-7.7-35.196.329-57.254 19.526a171.23 171.23 0 0 0-6.375 5.848a155.866 155.866 0 0 0-4.241-3.917C100.759 3.829 77.587-4.822 63.673 3.233C50.33 10.957 46.379 33.89 51.995 62.588a170.974 170.974 0 0 0 1.892 8.48c-3.28.932-6.445 1.924-9.474 2.98C17.309 83.498 0 98.307 0 113.668c0 15.865 18.582 31.778 46.812 41.427a145.52 145.52 0 0 0 6.921 2.165a167.467 167.467 0 0 0-2.01 9.138c-5.354 28.2-1.173 50.591 12.134 58.266c13.744 7.926 36.812-.22 59.273-19.855a145.567 145.567 0 0 0 5.342-4.923a168.064 168.064 0 0 0 6.92 6.314c21.758 18.722 43.246 26.282 56.54 18.586c13.731-7.949 18.194-32.003 12.4-61.268a145.016 145.016 0 0 0-1.535-6.842c1.62-.48 3.21-.974 4.76-1.488c29.348-9.723 48.443-25.443 48.443-41.52c0-15.417-17.868-30.326-45.517-39.844Zm-6.365 70.984c-1.4.463-2.836.91-4.3 1.345c-3.24-10.257-7.612-21.163-12.963-32.432c5.106-11 9.31-21.767 12.459-31.957c2.619.758 5.16 1.557 7.61 2.4c23.69 8.156 38.14 20.213 38.14 29.504c0 9.896-15.606 22.743-40.946 31.14Zm-10.514 20.834c2.562 12.94 2.927 24.64 1.23 33.787c-1.524 8.219-4.59 13.698-8.382 15.893c-8.067 4.67-25.32-1.4-43.927-17.412a156.726 156.726 0 0 1-6.437-5.87c7.214-7.889 14.423-17.06 21.459-27.246c12.376-1.098 24.068-2.894 34.671-5.345a134.17 134.17 0 0 1 1.386 6.193ZM87.276 214.515c-7.882 2.783-14.16 2.863-17.955.675c-8.075-4.657-11.432-22.636-6.853-46.752a156.923 156.923 0 0 1 1.869-8.499c10.486 2.32 22.093 3.988 34.498 4.994c7.084 9.967 14.501 19.128 21.976 27.15a134.668 134.668 0 0 1-4.877 4.492c-9.933 8.682-19.886 14.842-28.658 17.94ZM50.35 144.747c-12.483-4.267-22.792-9.812-29.858-15.863c-6.35-5.437-9.555-10.836-9.555-15.216c0-9.322 13.897-21.212 37.076-29.293c2.813-.98 5.757-1.905 8.812-2.773c3.204 10.42 7.406 21.315 12.477 32.332c-5.137 11.18-9.399 22.249-12.634 32.792a134.718 134.718 0 0 1-6.318-1.979Zm12.378-84.26c-4.811-24.587-1.616-43.134 6.425-47.789c8.564-4.958 27.502 2.111 47.463 19.835a144.318 144.318 0 0 1 3.841 3.545c-7.438 7.987-14.787 17.08-21.808 26.988c-12.04 1.116-23.565 2.908-34.161 5.309a160.342 160.342 0 0 1-1.76-7.887Zm110.427 27.268a347.8 347.8 0 0 0-7.785-12.803c8.168 1.033 15.994 2.404 23.343 4.08c-2.206 7.072-4.956 14.465-8.193 22.045a381.151 381.151 0 0 0-7.365-13.322Zm-45.032-43.861c5.044 5.465 10.096 11.566 15.065 18.186a322.04 322.04 0 0 0-30.257-.006c4.974-6.559 10.069-12.652 15.192-18.18ZM82.802 87.83a323.167 323.167 0 0 0-7.227 13.238c-3.184-7.553-5.909-14.98-8.134-22.152c7.304-1.634 15.093-2.97 23.209-3.984a321.524 321.524 0 0 0-7.848 12.897Zm8.081 65.352c-8.385-.936-16.291-2.203-23.593-3.793c2.26-7.3 5.045-14.885 8.298-22.6a321.187 321.187 0 0 0 7.257 13.246c2.594 4.48 5.28 8.868 8.038 13.147Zm37.542 31.03c-5.184-5.592-10.354-11.779-15.403-18.433c4.902.192 9.899.29 14.978.29c5.218 0 10.376-.117 15.453-.343c-4.985 6.774-10.018 12.97-15.028 18.486Zm52.198-57.817c3.422 7.8 6.306 15.345 8.596 22.52c-7.422 1.694-15.436 3.058-23.88 4.071a382.417 382.417 0 0 0 7.859-13.026a347.403 347.403 0 0 0 7.425-13.565Zm-16.898 8.101a358.557 358.557 0 0 1-12.281 19.815a329.4 329.4 0 0 1-23.444.823c-7.967 0-15.716-.248-23.178-.732a310.202 310.202 0 0 1-12.513-19.846h.001a307.41 307.41 0 0 1-10.923-20.627a310.278 310.278 0 0 1 10.89-20.637l-.001.001a307.318 307.318 0 0 1 12.413-19.761c7.613-.576 15.42-.876 23.31-.876H128c7.926 0 15.743.303 23.354.883a329.357 329.357 0 0 1 12.335 19.695a358.489 358.489 0 0 1 11.036 20.54a329.472 329.472 0 0 1-11 20.722Zm22.56-122.124c8.572 4.944 11.906 24.881 6.52 51.026c-.344 1.668-.73 3.367-1.15 5.09c-10.622-2.452-22.155-4.275-34.23-5.408c-7.034-10.017-14.323-19.124-21.64-27.008a160.789 160.789 0 0 1 5.888-5.4c18.9-16.447 36.564-22.941 44.612-18.3ZM128 90.808c12.625 0 22.86 10.235 22.86 22.86s-10.235 22.86-22.86 22.86s-22.86-10.235-22.86-22.86s10.235-22.86 22.86-22.86Z"></path></svg>

src/DataStreamSuite/Bts.DataStreamWebApp.Client/src/css/dark-hc.css

@@ -0,0 +1,51 @@
+.dark-high-contrast {
+  --md-sys-color-primary: rgb(251 250 255);
+  --md-sys-color-surface-tint: rgb(165 200 255);
+  --md-sys-color-on-primary: rgb(0 0 0);
+  --md-sys-color-primary-container: rgb(173 204 255);
+  --md-sys-color-on-primary-container: rgb(0 0 0);
+  --md-sys-color-secondary: rgb(250 250 255);
+  --md-sys-color-on-secondary: rgb(0 0 0);
+  --md-sys-color-secondary-container: rgb(159 208 255);
+  --md-sys-color-on-secondary-container: rgb(0 0 0);
+  --md-sys-color-tertiary: rgb(255 249 251);
+  --md-sys-color-on-tertiary: rgb(0 0 0);
+  --md-sys-color-tertiary-container: rgb(222 193 231);
+  --md-sys-color-on-tertiary-container: rgb(0 0 0);
+  --md-sys-color-error: rgb(255 249 249);
+  --md-sys-color-on-error: rgb(0 0 0);
+  --md-sys-color-error-container: rgb(255 186 177);
+  --md-sys-color-on-error-container: rgb(0 0 0);
+  --md-sys-color-background: rgb(17 19 24);
+  --md-sys-color-on-background: rgb(225 226 233);
+  --md-sys-color-surface: rgb(17 19 24);
+  --md-sys-color-on-surface: rgb(255 255 255);
+  --md-sys-color-surface-variant: rgb(67 71 78);
+  --md-sys-color-on-surface-variant: rgb(251 250 255);
+  --md-sys-color-outline: rgb(200 202 212);
+  --md-sys-color-outline-variant: rgb(200 202 212);
+  --md-sys-color-shadow: rgb(0 0 0);
+  --md-sys-color-scrim: rgb(0 0 0);
+  --md-sys-color-inverse-surface: rgb(225 226 233);
+  --md-sys-color-inverse-on-surface: rgb(0 0 0);
+  --md-sys-color-inverse-primary: rgb(0 42 83);
+  --md-sys-color-primary-fixed: rgb(219 231 255);
+  --md-sys-color-on-primary-fixed: rgb(0 0 0);
+  --md-sys-color-primary-fixed-dim: rgb(173 204 255);
+  --md-sys-color-on-primary-fixed-variant: rgb(0 22 49);
+  --md-sys-color-secondary-fixed: rgb(214 233 255);
+  --md-sys-color-on-secondary-fixed: rgb(0 0 0);
+  --md-sys-color-secondary-fixed-dim: rgb(159 208 255);
+  --md-sys-color-on-secondary-fixed-variant: rgb(0 24 42);
+  --md-sys-color-tertiary-fixed: rgb(249 222 255);
+  --md-sys-color-on-tertiary-fixed: rgb(0 0 0);
+  --md-sys-color-tertiary-fixed-dim: rgb(222 193 231);
+  --md-sys-color-on-tertiary-fixed-variant: rgb(33 14 42);
+  --md-sys-color-surface-dim: rgb(17 19 24);
+  --md-sys-color-surface-bright: rgb(55 57 62);
+  --md-sys-color-surface-container-lowest: rgb(12 14 19);
+  --md-sys-color-surface-container-low: rgb(25 28 32);
+  --md-sys-color-surface-container: rgb(29 32 36);
+  --md-sys-color-surface-container-high: rgb(40 42 47);
+  --md-sys-color-surface-container-highest: rgb(50 53 58);
+}

src/DataStreamSuite/Bts.DataStreamWebApp.Client/src/css/dark-mc.css

@@ -0,0 +1,51 @@
+.dark-medium-contrast {
+  --md-sys-color-primary: rgb(173 204 255);
+  --md-sys-color-surface-tint: rgb(165 200 255);
+  --md-sys-color-on-primary: rgb(0 22 49);
+  --md-sys-color-primary-container: rgb(112 146 198);
+  --md-sys-color-on-primary-container: rgb(0 0 0);
+  --md-sys-color-secondary: rgb(159 208 255);
+  --md-sys-color-on-secondary: rgb(0 24 42);
+  --md-sys-color-secondary-container: rgb(100 149 193);
+  --md-sys-color-on-secondary-container: rgb(0 0 0);
+  --md-sys-color-tertiary: rgb(222 193 231);
+  --md-sys-color-on-tertiary: rgb(33 14 42);
+  --md-sys-color-tertiary-container: rgb(162 136 171);
+  --md-sys-color-on-tertiary-container: rgb(0 0 0);
+  --md-sys-color-error: rgb(255 186 177);
+  --md-sys-color-on-error: rgb(55 0 1);
+  --md-sys-color-error-container: rgb(255 84 73);
+  --md-sys-color-on-error-container: rgb(0 0 0);
+  --md-sys-color-background: rgb(17 19 24);
+  --md-sys-color-on-background: rgb(225 226 233);
+  --md-sys-color-surface: rgb(17 19 24);
+  --md-sys-color-on-surface: rgb(251 250 255);
+  --md-sys-color-surface-variant: rgb(67 71 78);
+  --md-sys-color-on-surface-variant: rgb(200 202 212);
+  --md-sys-color-outline: rgb(160 163 171);
+  --md-sys-color-outline-variant: rgb(128 131 139);
+  --md-sys-color-shadow: rgb(0 0 0);
+  --md-sys-color-scrim: rgb(0 0 0);
+  --md-sys-color-inverse-surface: rgb(225 226 233);
+  --md-sys-color-inverse-on-surface: rgb(40 42 47);
+  --md-sys-color-inverse-primary: rgb(35 73 120);
+  --md-sys-color-primary-fixed: rgb(212 227 255);
+  --md-sys-color-on-primary-fixed: rgb(0 17 40);
+  --md-sys-color-primary-fixed-dim: rgb(165 200 255);
+  --md-sys-color-on-primary-fixed-variant: rgb(10 55 100);
+  --md-sys-color-secondary-fixed: rgb(206 229 255);
+  --md-sys-color-on-secondary-fixed: rgb(0 18 34);
+  --md-sys-color-secondary-fixed-dim: rgb(154 203 250);
+  --md-sys-color-on-secondary-fixed-variant: rgb(0 57 91);
+  --md-sys-color-tertiary-fixed: rgb(247 216 255);
+  --md-sys-color-on-tertiary-fixed: rgb(28 9 37);
+  --md-sys-color-tertiary-fixed-dim: rgb(218 189 226);
+  --md-sys-color-on-tertiary-fixed-variant: rgb(67 46 76);
+  --md-sys-color-surface-dim: rgb(17 19 24);
+  --md-sys-color-surface-bright: rgb(55 57 62);
+  --md-sys-color-surface-container-lowest: rgb(12 14 19);
+  --md-sys-color-surface-container-low: rgb(25 28 32);
+  --md-sys-color-surface-container: rgb(29 32 36);
+  --md-sys-color-surface-container-high: rgb(40 42 47);
+  --md-sys-color-surface-container-highest: rgb(50 53 58);
+}

src/DataStreamSuite/Bts.DataStreamWebApp.Client/src/css/dark.css

@@ -0,0 +1,51 @@
+.dark {
+  --md-sys-color-primary: rgb(165 200 255);
+  --md-sys-color-surface-tint: rgb(165 200 255);
+  --md-sys-color-on-primary: rgb(0 49 94);
+  --md-sys-color-primary-container: rgb(34 72 118);
+  --md-sys-color-on-primary-container: rgb(212 227 255);
+  --md-sys-color-secondary: rgb(154 203 250);
+  --md-sys-color-on-secondary: rgb(0 51 83);
+  --md-sys-color-secondary-container: rgb(12 74 114);
+  --md-sys-color-on-secondary-container: rgb(206 229 255);
+  --md-sys-color-tertiary: rgb(218 189 226);
+  --md-sys-color-on-tertiary: rgb(61 41 70);
+  --md-sys-color-tertiary-container: rgb(85 63 93);
+  --md-sys-color-on-tertiary-container: rgb(247 216 255);
+  --md-sys-color-error: rgb(255 180 171);
+  --md-sys-color-on-error: rgb(105 0 5);
+  --md-sys-color-error-container: rgb(147 0 10);
+  --md-sys-color-on-error-container: rgb(255 218 214);
+  --md-sys-color-background: rgb(17 19 24);
+  --md-sys-color-on-background: rgb(225 226 233);
+  --md-sys-color-surface: rgb(17 19 24);
+  --md-sys-color-on-surface: rgb(225 226 233);
+  --md-sys-color-surface-variant: rgb(67 71 78);
+  --md-sys-color-on-surface-variant: rgb(195 198 207);
+  --md-sys-color-outline: rgb(141 145 153);
+  --md-sys-color-outline-variant: rgb(67 71 78);
+  --md-sys-color-shadow: rgb(0 0 0);
+  --md-sys-color-scrim: rgb(0 0 0);
+  --md-sys-color-inverse-surface: rgb(225 226 233);
+  --md-sys-color-inverse-on-surface: rgb(46 48 53);
+  --md-sys-color-inverse-primary: rgb(60 96 144);
+  --md-sys-color-primary-fixed: rgb(212 227 255);
+  --md-sys-color-on-primary-fixed: rgb(0 28 58);
+  --md-sys-color-primary-fixed-dim: rgb(165 200 255);
+  --md-sys-color-on-primary-fixed-variant: rgb(34 72 118);
+  --md-sys-color-secondary-fixed: rgb(206 229 255);
+  --md-sys-color-on-secondary-fixed: rgb(0 29 50);
+  --md-sys-color-secondary-fixed-dim: rgb(154 203 250);
+  --md-sys-color-on-secondary-fixed-variant: rgb(12 74 114);
+  --md-sys-color-tertiary-fixed: rgb(247 216 255);
+  --md-sys-color-on-tertiary-fixed: rgb(39 20 48);
+  --md-sys-color-tertiary-fixed-dim: rgb(218 189 226);
+  --md-sys-color-on-tertiary-fixed-variant: rgb(85 63 93);
+  --md-sys-color-surface-dim: rgb(17 19 24);
+  --md-sys-color-surface-bright: rgb(55 57 62);
+  --md-sys-color-surface-container-lowest: rgb(12 14 19);
+  --md-sys-color-surface-container-low: rgb(25 28 32);
+  --md-sys-color-surface-container: rgb(29 32 36);
+  --md-sys-color-surface-container-high: rgb(40 42 47);
+  --md-sys-color-surface-container-highest: rgb(50 53 58);
+}

src/DataStreamSuite/Bts.DataStreamWebApp.Client/src/css/light-hc.css

@@ -0,0 +1,51 @@
+.light-high-contrast {
+  --md-sys-color-primary: rgb(0 34 70);
+  --md-sys-color-surface-tint: rgb(60 96 144);
+  --md-sys-color-on-primary: rgb(255 255 255);
+  --md-sys-color-primary-container: rgb(29 68 114);
+  --md-sys-color-on-primary-container: rgb(255 255 255);
+  --md-sys-color-secondary: rgb(0 36 61);
+  --md-sys-color-on-secondary: rgb(255 255 255);
+  --md-sys-color-secondary-container: rgb(3 70 110);
+  --md-sys-color-on-secondary-container: rgb(255 255 255);
+  --md-sys-color-tertiary: rgb(46 26 55);
+  --md-sys-color-on-tertiary: rgb(255 255 255);
+  --md-sys-color-tertiary-container: rgb(81 59 89);
+  --md-sys-color-on-tertiary-container: rgb(255 255 255);
+  --md-sys-color-error: rgb(78 0 2);
+  --md-sys-color-on-error: rgb(255 255 255);
+  --md-sys-color-error-container: rgb(140 0 9);
+  --md-sys-color-on-error-container: rgb(255 255 255);
+  --md-sys-color-background: rgb(249 249 255);
+  --md-sys-color-on-background: rgb(25 28 32);
+  --md-sys-color-surface: rgb(249 249 255);
+  --md-sys-color-on-surface: rgb(0 0 0);
+  --md-sys-color-surface-variant: rgb(224 226 236);
+  --md-sys-color-on-surface-variant: rgb(32 36 43);
+  --md-sys-color-outline: rgb(63 67 74);
+  --md-sys-color-outline-variant: rgb(63 67 74);
+  --md-sys-color-shadow: rgb(0 0 0);
+  --md-sys-color-scrim: rgb(0 0 0);
+  --md-sys-color-inverse-surface: rgb(46 48 53);
+  --md-sys-color-inverse-on-surface: rgb(255 255 255);
+  --md-sys-color-inverse-primary: rgb(228 236 255);
+  --md-sys-color-primary-fixed: rgb(29 68 114);
+  --md-sys-color-on-primary-fixed: rgb(255 255 255);
+  --md-sys-color-primary-fixed-dim: rgb(0 45 88);
+  --md-sys-color-on-primary-fixed-variant: rgb(255 255 255);
+  --md-sys-color-secondary-fixed: rgb(3 70 110);
+  --md-sys-color-on-secondary-fixed: rgb(255 255 255);
+  --md-sys-color-secondary-fixed-dim: rgb(0 47 77);
+  --md-sys-color-on-secondary-fixed-variant: rgb(255 255 255);
+  --md-sys-color-tertiary-fixed: rgb(81 59 89);
+  --md-sys-color-on-tertiary-fixed: rgb(255 255 255);
+  --md-sys-color-tertiary-fixed-dim: rgb(57 37 66);
+  --md-sys-color-on-tertiary-fixed-variant: rgb(255 255 255);
+  --md-sys-color-surface-dim: rgb(217 218 224);
+  --md-sys-color-surface-bright: rgb(249 249 255);
+  --md-sys-color-surface-container-lowest: rgb(255 255 255);
+  --md-sys-color-surface-container-low: rgb(243 243 250);
+  --md-sys-color-surface-container: rgb(237 237 244);
+  --md-sys-color-surface-container-high: rgb(231 232 238);
+  --md-sys-color-surface-container-highest: rgb(225 226 233);
+}

src/DataStreamSuite/Bts.DataStreamWebApp.Client/src/css/light-mc.css

@@ -0,0 +1,51 @@
+.light-medium-contrast {
+  --md-sys-color-primary: rgb(29 68 114);
+  --md-sys-color-surface-tint: rgb(60 96 144);
+  --md-sys-color-on-primary: rgb(255 255 255);
+  --md-sys-color-primary-container: rgb(83 118 167);
+  --md-sys-color-on-primary-container: rgb(255 255 255);
+  --md-sys-color-secondary: rgb(3 70 110);
+  --md-sys-color-on-secondary: rgb(255 255 255);
+  --md-sys-color-secondary-container: rgb(71 121 163);
+  --md-sys-color-on-secondary-container: rgb(255 255 255);
+  --md-sys-color-tertiary: rgb(81 59 89);
+  --md-sys-color-on-tertiary: rgb(255 255 255);
+  --md-sys-color-tertiary-container: rgb(133 108 141);
+  --md-sys-color-on-tertiary-container: rgb(255 255 255);
+  --md-sys-color-error: rgb(140 0 9);
+  --md-sys-color-on-error: rgb(255 255 255);
+  --md-sys-color-error-container: rgb(218 52 46);
+  --md-sys-color-on-error-container: rgb(255 255 255);
+  --md-sys-color-background: rgb(249 249 255);
+  --md-sys-color-on-background: rgb(25 28 32);
+  --md-sys-color-surface: rgb(249 249 255);
+  --md-sys-color-on-surface: rgb(25 28 32);
+  --md-sys-color-surface-variant: rgb(224 226 236);
+  --md-sys-color-on-surface-variant: rgb(63 67 74);
+  --md-sys-color-outline: rgb(91 95 103);
+  --md-sys-color-outline-variant: rgb(119 123 131);
+  --md-sys-color-shadow: rgb(0 0 0);
+  --md-sys-color-scrim: rgb(0 0 0);
+  --md-sys-color-inverse-surface: rgb(46 48 53);
+  --md-sys-color-inverse-on-surface: rgb(240 240 247);
+  --md-sys-color-inverse-primary: rgb(165 200 255);
+  --md-sys-color-primary-fixed: rgb(83 118 167);
+  --md-sys-color-on-primary-fixed: rgb(255 255 255);
+  --md-sys-color-primary-fixed-dim: rgb(58 93 141);
+  --md-sys-color-on-primary-fixed-variant: rgb(255 255 255);
+  --md-sys-color-secondary-fixed: rgb(71 121 163);
+  --md-sys-color-on-secondary-fixed: rgb(255 255 255);
+  --md-sys-color-secondary-fixed-dim: rgb(43 96 137);
+  --md-sys-color-on-secondary-fixed-variant: rgb(255 255 255);
+  --md-sys-color-tertiary-fixed: rgb(133 108 141);
+  --md-sys-color-on-tertiary-fixed: rgb(255 255 255);
+  --md-sys-color-tertiary-fixed-dim: rgb(107 84 116);
+  --md-sys-color-on-tertiary-fixed-variant: rgb(255 255 255);
+  --md-sys-color-surface-dim: rgb(217 218 224);
+  --md-sys-color-surface-bright: rgb(249 249 255);
+  --md-sys-color-surface-container-lowest: rgb(255 255 255);
+  --md-sys-color-surface-container-low: rgb(243 243 250);
+  --md-sys-color-surface-container: rgb(237 237 244);
+  --md-sys-color-surface-container-high: rgb(231 232 238);
+  --md-sys-color-surface-container-highest: rgb(225 226 233);
+}

src/DataStreamSuite/Bts.DataStreamWebApp.Client/src/css/light.css

@@ -0,0 +1,51 @@
+.light {
+  --md-sys-color-primary: rgb(60 96 144);
+  --md-sys-color-surface-tint: rgb(60 96 144);
+  --md-sys-color-on-primary: rgb(255 255 255);
+  --md-sys-color-primary-container: rgb(212 227 255);
+  --md-sys-color-on-primary-container: rgb(0 28 58);
+  --md-sys-color-secondary: rgb(46 98 140);
+  --md-sys-color-on-secondary: rgb(255 255 255);
+  --md-sys-color-secondary-container: rgb(206 229 255);
+  --md-sys-color-on-secondary-container: rgb(0 29 50);
+  --md-sys-color-tertiary: rgb(110 86 118);
+  --md-sys-color-on-tertiary: rgb(255 255 255);
+  --md-sys-color-tertiary-container: rgb(247 216 255);
+  --md-sys-color-on-tertiary-container: rgb(39 20 48);
+  --md-sys-color-error: rgb(186 26 26);
+  --md-sys-color-on-error: rgb(255 255 255);
+  --md-sys-color-error-container: rgb(255 218 214);
+  --md-sys-color-on-error-container: rgb(65 0 2);
+  --md-sys-color-background: rgb(249 249 255);
+  --md-sys-color-on-background: rgb(25 28 32);
+  --md-sys-color-surface: rgb(249 249 255);
+  --md-sys-color-on-surface: rgb(25 28 32);
+  --md-sys-color-surface-variant: rgb(224 226 236);
+  --md-sys-color-on-surface-variant: rgb(67 71 78);
+  --md-sys-color-outline: rgb(116 119 127);
+  --md-sys-color-outline-variant: rgb(195 198 207);
+  --md-sys-color-shadow: rgb(0 0 0);
+  --md-sys-color-scrim: rgb(0 0 0);
+  --md-sys-color-inverse-surface: rgb(46 48 53);
+  --md-sys-color-inverse-on-surface: rgb(240 240 247);
+  --md-sys-color-inverse-primary: rgb(165 200 255);
+  --md-sys-color-primary-fixed: rgb(212 227 255);
+  --md-sys-color-on-primary-fixed: rgb(0 28 58);
+  --md-sys-color-primary-fixed-dim: rgb(165 200 255);
+  --md-sys-color-on-primary-fixed-variant: rgb(34 72 118);
+  --md-sys-color-secondary-fixed: rgb(206 229 255);
+  --md-sys-color-on-secondary-fixed: rgb(0 29 50);
+  --md-sys-color-secondary-fixed-dim: rgb(154 203 250);
+  --md-sys-color-on-secondary-fixed-variant: rgb(12 74 114);
+  --md-sys-color-tertiary-fixed: rgb(247 216 255);
+  --md-sys-color-on-tertiary-fixed: rgb(39 20 48);
+  --md-sys-color-tertiary-fixed-dim: rgb(218 189 226);
+  --md-sys-color-on-tertiary-fixed-variant: rgb(85 63 93);
+  --md-sys-color-surface-dim: rgb(217 218 224);
+  --md-sys-color-surface-bright: rgb(249 249 255);
+  --md-sys-color-surface-container-lowest: rgb(255 255 255);
+  --md-sys-color-surface-container-low: rgb(243 243 250);
+  --md-sys-color-surface-container: rgb(237 237 244);
+  --md-sys-color-surface-container-high: rgb(231 232 238);
+  --md-sys-color-surface-container-highest: rgb(225 226 233);
+}

src/DataStreamSuite/Bts.DataStreamWebApp.Client/src/index.css

@@ -0,0 +1,107 @@
+:root {
+  font-family: Inter, system-ui, Avenir, Helvetica, Arial, sans-serif;
+  line-height: 1.5;
+  font-weight: 400;
+
+  color-scheme: light dark;
+  color: rgba(255, 255, 255, 0.87);
+  background-color: #242424;
+
+  font-synthesis: none;
+  text-rendering: optimizeLegibility;
+  -webkit-font-smoothing: antialiased;
+  -moz-osx-font-smoothing: grayscale;
+}
+
+a {
+  font-weight: 500;
+  color: #646cff;
+  text-decoration: inherit;
+}
+a:hover {
+  color: #535bf2;
+}
+
+body {
+  margin: 0;
+  display: flex;
+  place-items: center;
+  min-width: 320px;
+  min-height: 100vh;
+}
+
+button {
+  border-radius: 8px;
+  border: 1px solid transparent;
+  padding: 0.6em 1.2em;
+  font-size: 1em;
+  font-weight: 500;
+  font-family: inherit;
+  background-color: #1a1a1a;
+  cursor: pointer;
+  transition: border-color 0.25s;
+}
+button:hover {
+  border-color: #646cff;
+}
+button:focus,
+button:focus-visible {
+  outline: 4px auto -webkit-focus-ring-color;
+}
+
+button#query-button {
+  height: 5em;
+  vertical-align: top;
+}
+
+button#play-button {
+  padding: '10px 20px';
+  font-size: '16px';
+  cursor: 'pointer';
+  background-color: '#28a745';
+  color: '#fff';
+  border: 'none';
+  border-radius: '5px';
+};
+
+button#pause-button {
+  background-color: #ffc107;
+}
+
+button#stop-button {
+  background-color: #dc3545;
+}
+
+h1 {
+  font-size: 3.2em;
+  line-height: 1.1;
+}
+
+p#footer {
+  float: none;
+  clear: both;
+  vertical-align: bottom;
+}
+
+table#query-output {
+  float: none;
+  clear: both;
+}
+
+textarea#query-input {
+  height: 5em;
+  width: 50em;
+}
+
+@media (prefers-color-scheme: light) {
+  :root {
+    color: #213547;
+    background-color: #ffffff;
+  }
+  a:hover {
+    color: #747bff;
+  }
+  button {
+    background-color: #f9f9f9;
+  }
+}

src/DataStreamSuite/Bts.DataStreamWebApp.Client/src/main.jsx

@@ -0,0 +1,9 @@
+import React from 'react';
+import ReactDOM from 'react-dom/client';
+import App from './App';
+
+ReactDOM.createRoot(document.getElementById('root')).render(
+  <React.StrictMode>
+    <App />
+  </React.StrictMode>
+)

src/DataStreamSuite/Bts.DataStreamWebApp.Client/vite.config.js

@@ -0,0 +1,69 @@
+import { fileURLToPath, URL } from 'node:url';
+import { defineConfig } from 'vite';
+import plugin from '@vitejs/plugin-react';
+import fs from 'fs';
+import path from 'path';
+import child_process from 'child_process';
+import process, { env } from 'process';
+
+const baseFolder =
+  process.env.APPDATA !== undefined && process.env.APPDATA !== ''
+    ? `${process.env.APPDATA}/ASP.NET/https`
+    : `${process.env.HOME}/.aspnet/https`;
+
+const certificateName = "bts.datastreamwebapp.client";
+const certFilePath = path.join(baseFolder, `${certificateName}.pem`);
+const keyFilePath = path.join(baseFolder, `${certificateName}.key`);
+
+if (!fs.existsSync(certFilePath) || !fs.existsSync(keyFilePath)) {
+  if (0 !== child_process.spawnSync('dotnet', [
+    'dev-certs',
+    'https',
+    '--export-path',
+    certFilePath,
+    '--format',
+    'Pem',
+    '--no-password',
+  ], { stdio: 'inherit', }).status) {
+    throw new Error("Could not create certificate.");
+  }
+}
+
+const target = env.ASPNETCORE_HTTPS_PORT
+  ? `https://localhost:${env.ASPNETCORE_HTTPS_PORT}`
+  : env.ASPNETCORE_URLS
+    ? env.ASPNETCORE_URLS.split(';')[0]
+    : 'https://localhost:5001';
+
+export default defineConfig(({ $$command, mode, $$isSsrBuild, $$isPreview }) => {
+  return {
+    plugins: [plugin()],
+    resolve: {
+      alias: {
+        '@': fileURLToPath(new URL('./src', import.meta.url))
+      }
+    },
+    build: {
+      minify: false,
+      sourcemap: true,
+      outDir: `../../../build/Bts.DataStreamWebApp.Client/${mode}/dist`
+    },
+    server: {
+      proxy: {
+        '^/api': {
+          target,
+          secure: false
+        },
+        '^/ws': {
+          target,
+          secure: false
+        }
+      },
+      port: 5173,
+      https: {
+        key: fs.readFileSync(keyFilePath),
+        cert: fs.readFileSync(certFilePath),
+      }
+    }
+  };
+});

src/DataStreamSuite/Bts.DataStreamWebApp.Server/Bts.DataStreamWebApp.Server.csproj

@@ -0,0 +1,35 @@
+<Project Sdk="Microsoft.NET.Sdk.Web">
+
+  <PropertyGroup>
+    <TargetFramework>net7.0</TargetFramework>
+    <Nullable>enable</Nullable>
+    <ImplicitUsings>enable</ImplicitUsings>
+    <InvariantGlobalization>true</InvariantGlobalization>
+    <SpaRoot>..\Bts.DataStreamWebApp.Client</SpaRoot>
+    <SpaProxyLaunchCommand>npm run dev</SpaProxyLaunchCommand>
+    <SpaProxyServerUrl>https://localhost:5173</SpaProxyServerUrl>
+  </PropertyGroup>
+
+  <ItemGroup>
+    <PackageReference Include="CommandLineParser" Version="2.9.1" />
+    <PackageReference Include="Microsoft.AspNetCore.SpaProxy" Version="7.0.20" />
+    <PackageReference Include="Microsoft.AspNetCore.SpaServices.Extensions" Version="7.0.20" />
+    <PackageReference Include="Newtonsoft.Json" Version="13.0.3" />
+    <PackageReference Include="Swashbuckle.AspNetCore" Version="6.4.0" />
+  </ItemGroup>
+
+  <ItemGroup>
+    <ProjectReference Include="..\Bts.DataStreamWebApp.Client\Bts.DataStreamWebApp.Client.esproj">
+      <ReferenceOutputAssembly>false</ReferenceOutputAssembly>
+    </ProjectReference>
+    <ProjectReference Include="..\DataStream\DataStream.csproj" />
+    <ProjectReference Include="..\SharedStreams\SharedStreams.csproj" />
+    <ProjectReference Include="..\..\client\spark_api\spark_api.csproj" />
+    <ProjectReference Include="..\..\client\spark_core\spark_core.csproj" />
+    <ProjectReference Include="..\..\client\spark_websocket\spark_websocket.csproj" />
+  </ItemGroup>
+
+  <!-- <ItemGroup>
+    <Content Update="wwwroot\alt-index.html" CopyToPublishDirectory="Never" />
+  </ItemGroup> -->
+</Project>

src/DataStreamSuite/Bts.DataStreamWebApp.Server/Configurations/DevServer.json

@@ -0,0 +1,106 @@
+{
+  "MasterWorker": {
+    "Interval": "00:00:00.1"
+  },
+  "ViewStreamDatabase": {
+    "Interval": "00:00:00.1"
+  },
+  "Sparkd": {
+    "Config": {
+      "ConfigServer": "http://localhost:8000/config"
+    },
+    "Credentials": {
+      "User": "bts",
+      "Password": ""
+    },
+    "EnableAdmin": true,
+    "EnableOrders": true,
+    "EnablePosition": true
+  },
+  "PubSub": {
+    "MillisecondsBetweenSubInfoUpdates": 1000,
+    "MillisecondsBetweenTimeSliceUpdates": 5000,
+    "MillisecondsBetweenNoBufferSpaceRetries": 100,
+    "RetriesOnNoBufferSpace": 1,
+    "DoItAllSecondsBetweenLogs": 0,
+    "HeartbeatInterval": 15000,
+    "MissingHeartbeatsBeforeGc": 50,
+    "UseManagedSockets": true,
+    "StateUpdatePeriod": 5000,
+    "StatisticsUpdatePeriod": 5000,
+    "DisposedConnectionCheckPeriod": 1000,
+    "MessageProcessingWarningThreshold": 1000,
+    "ServerPort": 12345,
+    "Routing": {
+      "PublisherRoutes": [
+        {
+          "Name": "default",
+          "ServerAddress": "bts-cloud://127.0.0.1:12345",
+          "TagRegExp": ".*"
+        }
+      ],
+      "SubscriberRoutes": [
+        {
+          "Name": "default",
+          "ServerAddress": "bts-cloud://127.0.0.1:12345",
+          "TagRegExp": ".*"
+        }
+      ]
+    },
+    "MaxPacketSize": 500000000,
+    "BlueCommMustDie": true,
+    "CrushableTags": [],
+    "TagTypeToAssemblyMap": {
+      "Entries": [
+        {
+          "HandlerClass": "BCG.PubSubServerTagHandler",
+          "Assembly": "DataStreamCore.dll"
+        }
+      ]
+    },
+    "ServerQueue": "PubSubQueue",
+    "PortRange": {
+      "First": 30400,
+      "Last": 31700
+    },
+    "BindToRandomPort": false,
+    "DebugTags": "^NOTAGS$",
+    "LatencyReportThresholdMs": 1500,
+    "UseSubstations": false,
+    "UseDynamicClients": false,
+    "IsSubstation": false,
+    "InternallyPublishSubInfo": true,
+    "UseManagedClientsInNativeCode": false,
+    "CompressedTags": "^NOTAGS$",
+    "TagsToReportCompressionRatio": "^NOTAGS$",
+    "DisableMsmq": false,
+    "UsePrepublishedServerState": false,
+    "IpTranslation": null,
+    "GcActivationThreshold": 0,
+    "WaitPeriodAfterGc": 1000,
+    "TagHandlerFactory": {
+      "Handlers": [
+        {
+          "TagType": "DataStream",
+          "Assembly": "DataStreamCore",
+          "Class": "bts.PubsubServerStreamTagHandler"
+        },
+        {
+          "TagType": "System_Sub",
+          "Assembly": "PubSub",
+          "Class": "BCG.SystemSubscriptionMapPublisher"
+        },
+        {
+          "TagType": "System_Stats",
+          "Assembly": "PubSub",
+          "Class": "BCG.SystemStatsPublisher"
+        },
+        {
+          "TagType": ".*",
+          "Assembly": "DataStreamCore",
+         "Class": "bts.PubsubServerGenericDataHandler"
+        }
+      ]
+    }
+  }
+}

src/DataStreamSuite/Bts.DataStreamWebApp.Server/Configurations/ProdServer.json

@@ -0,0 +1,4 @@
+{
+  "content_root_path": "build/Bts.DataStreamWebApp.Client/production",
+  "web_root_path": "dist"
+}

src/DataStreamSuite/Bts.DataStreamWebApp.Server/Controllers/DataStreamSubscriptionController.cs

@@ -0,0 +1,30 @@
+namespace Bts.DataStreamWebApp.Server.Controllers;
+
+using Microsoft.AspNetCore.Mvc;
+
+[Route("/ws")]
+public class DataStreamSubscriptionController : ControllerBase {
+   readonly ILogger logger;
+
+   public DataStreamSubscriptionController(ILogger<DataStreamSubscriptionController> logger) {
+      this.logger = logger;
+      logger.LogInformation("Creating {ControllerType}", GetType().Name);
+   }
+
+   [Route("/ws")]
+   public async Task Get(CancellationToken cancellationToken) {
+      if (!HttpContext.WebSockets.IsWebSocketRequest) {
+         logger.LogError("Expected websocket request on path '{}'", HttpContext.Request.Path);
+         return;
+      }
+      logger.LogInformation("Establishing websocket connection...");
+      try {
+         using var webSocket = await HttpContext.WebSockets.AcceptWebSocketAsync();
+         logger.LogInformation("Websocket connection established.");
+         using var server = new DataStreamWebSocketServer(webSocket, logger);
+         await server.Run(cancellationToken);
+      } catch (Exception ex) {
+         logger.LogError("DataStreamSubscriptionController.Run threw exception: {Exception}", ex);
+      }
+   }
+}

src/DataStreamSuite/Bts.DataStreamWebApp.Server/DataStreamSubscription.cs

@@ -0,0 +1,125 @@
+namespace Bts.DataStreamWebApp.Server.Controllers;
+
+using BCG;
+
+using bts.websocket_core.json_rpc;
+
+using Bts.DataStreamCore;
+using Bts.DataStreamWebApp.Server.Messages;
+
+using System.Net.WebSockets;
+using System.Text;
+using System.Text.Json;
+using System.Text.Json.Serialization;
+
+class DataStreamSubscription : IDisposable, IHostedService {
+   public readonly ViewStream ViewStream;
+   public DataPacket? LastFired;
+
+   readonly ILogger logger;
+   readonly WebSocket webSocket;
+
+   public DataStreamSubscription(string id, JsonRpcId requestId, ViewStream viewStream, WebSocket webSocket, ILogger logger) {
+      ViewStream = viewStream;
+      RequestId = requestId;
+      Id = id;
+      this.webSocket = webSocket;
+      this.logger = logger;
+   }
+
+   public string Id { get; }
+   public JsonRpcId RequestId { get; }
+   public SubscriptionStatus Status { get; private set; }
+
+   public void Start() {
+      if (Status == SubscriptionStatus.Subscribed) {
+         logger.LogWarning("Ignoring redundant call to 'Start'. Id = {Id} Status = {Status}", Id, Status);
+         return;
+      }
+      logger.LogInformation("Subscribing to ViewStream. SubscriptionId = {SubscriptionId}", Id);
+      Status = SubscriptionStatus.Subscribed;
+      ViewStream.onChange += onChange;
+   }
+
+   public Task StartAsync(CancellationToken cancellationToken) {
+      cancellationToken.ThrowIfCancellationRequested();
+      Start();
+      return Task.CompletedTask;
+   }
+
+   public Task Clear() {
+      if (LastFired is null) {
+         return Task.CompletedTask;
+      }
+      logger.LogInformation("Clearing subscription (firing DataPacket to delete all rows). Id = {Id} Status = {Status}", Id, Status);
+      return sendDataPacket("clear", new(new BlueTable(LastFired.schema), new BlueTable(LastFired.schema), LastFired.newRows));
+   }
+
+   public void Stop() {
+      if (Status == SubscriptionStatus.Unsubscribed) {
+         logger.LogWarning("Ignoring redundant call to 'Stop'. Id = {Id} Status = {Status}", Id, Status);
+         return;
+      }
+      logger.LogInformation("Unsubscribing from ViewStream. SubscriptionId = {SubscriptionId}", Id);
+      Status = SubscriptionStatus.Unsubscribed;
+      ViewStream.onChange -= onChange;
+   }
+
+   public Task StopAsync(CancellationToken cancellationToken) {
+      cancellationToken.ThrowIfCancellationRequested();
+      Stop();
+      return Task.CompletedTask;
+   }
+
+   public void Dispose() =>
+      Stop();
+
+   async void onChange(object newValue) {
+      if (newValue is not DataPacket dp) {
+         logger.LogError("Expected DataPacket in stream callback (actually received {}).", newValue?.GetType());
+         return;
+      }
+      if (webSocket.State != WebSocketState.Open) {
+         logger.LogError(
+            "Unsubscribing from stream early because WebSocket state = {WebSocketState}", webSocket.State);
+         Dispose();
+         return;
+      }
+      await sendDataPacket("subscribe", dp);
+   }
+
+   async Task sendDataPacket(string method, DataPacket dp) {
+      var serializerOpts = new JsonSerializerOptions {
+         NumberHandling = JsonNumberHandling.AllowNamedFloatingPointLiterals
+      };
+      serializerOpts.Converters.Add(new DataPacketJsonConverter() {
+         IncludeCurrent = true,
+         UseObjectRows = true,
+      });
+      var res = JsonRpcRequest.CreateNotification(method, dp);
+      var str = serialize(res, serializerOpts);
+      if (str is null) {
+         logger.LogError("Unsubscribing from stream early because we failed to serialize an outgoing DataPacket");
+         Dispose();
+         return;
+      }
+      logger.LogInformation(
+         "Sending DataPacket. NumRows = {NumRows} NumCols = {NumCols} MessageLength = {MessageLength}",
+         dp.newRows.Count,
+         dp.schema.columnNames.Length,
+         str.Length);
+      var bytes = Encoding.UTF8.GetBytes(str);
+      await webSocket.SendAsync(bytes, WebSocketMessageType.Text, true, CancellationToken.None);
+      LastFired = dp;
+   }
+
+   string? serialize<T>(T message, JsonSerializerOptions? opts) {
+      opts ??= JsonSerializerOptions.Default;
+      try {
+         return JsonSerializer.Serialize(message, opts);
+      } catch (Exception ex) {
+         logger.LogError("Failed to serialize message of type {MessageType}: {Exception}", typeof(T), ex);
+      }
+      return null;
+   }
+}

src/DataStreamSuite/Bts.DataStreamWebApp.Server/DataStreamWebAppServerConfig.cs

@@ -0,0 +1,89 @@
+namespace Bts.DataStreamWebApp.Server;
+
+using Bts.PubSub;
+
+using CommandLine;
+
+using System.Text.Json;
+using System.Text.Json.Serialization;
+
+class DataStreamWebAppServerConfig {
+   /// <summary>
+   /// The login user for sparkd
+   /// </summary>
+   [JsonPropertyName("user")]
+   [Option("user", HelpText = "The sparkd login user")]
+   public string? User { get; set; }
+
+   /// <summary>
+   /// The login password for sparkd
+   /// </summary>
+   [JsonPropertyName("password")]
+   [Option("pass", HelpText = "The sparkd login password")]
+   public string? Password { get; set; }
+
+   /// <summary>
+   /// The config server url for sparkd
+   /// </summary>
+   [JsonPropertyName("sparkd")]
+   [Option("sparkd", HelpText = "The sparkd config server")]
+   public string? Sparkd { get; set; }
+
+   [JsonPropertyName("pubsub")]
+   public PubSubConfig? PubSub { get; set; }
+
+   [Option("content-root-path")]
+   [JsonPropertyName("content_root_path")]
+   public string? ContentRootPath { get; set; }
+
+   [Option("web-root-path")]
+   [JsonPropertyName("web_root_path")]
+   public string? WebRootPath { get; set; }
+
+   [Option("environment-name")]
+   [JsonPropertyName("environment_name")]
+   public string? EnvironmentName { get; set; }
+
+   public DirectoryBrowserOptions? DirectoryBrowser { get; set; }
+
+   [Option(
+      "config",
+      MetaValue = "PATH",
+      HelpText = "Path to json file representing these options (DataStreamPublisherConfig)")]
+   [JsonIgnore]
+   public string? ConfigFile { get; set; }
+
+   public static DataStreamWebAppServerConfig Default() =>
+      new() {
+         User = "bts",
+         PubSub = PubSubConfig.Default(),
+      };
+
+   public void MergeWith(DataStreamWebAppServerConfig other) {
+      User ??= other.User;
+      Password ??= other.Password;
+      PubSub ??= other.PubSub;
+      Sparkd ??= other.Sparkd;
+      ContentRootPath ??= other.ContentRootPath;
+      WebRootPath ??= other.WebRootPath;
+      ConfigFile ??= other.ConfigFile;
+   }
+
+   /// <summary>
+   /// Merges options with those defined in `ConfigFile` (does not overwrite
+   /// non-null existing values).
+   /// </summary>
+   /// <exception cref="InvalidOperationException">
+   /// JsonSerializer.Deserialize returns null
+   /// </exception>
+   public void MergeWithConfigFile() {
+      if (ConfigFile is null) {
+         return;
+      }
+      var cfgString = File.ReadAllText(ConfigFile);
+      var parsedCfg = JsonSerializer.Deserialize<DataStreamWebAppServerConfig>(cfgString) ??
+         throw new InvalidOperationException($"Failed to parse json from {ConfigFile}");
+      MergeWith(parsedCfg);
+      ConfigFile = null;
+   }
+}

src/DataStreamSuite/Bts.DataStreamWebApp.Server/DataStreamWebSocketServer.cs

@@ -0,0 +1,121 @@
+namespace Bts.DataStreamWebApp.Server.Controllers;
+
+using bts.websocket_core.json_rpc;
+
+using Bts.DataStreamWebApp.Server.Messages;
+
+using System.Net.WebSockets;
+using System.Text;
+using System.Text.Json;
+using System.Text.Json.Nodes;
+
+public class DataStreamWebSocketServer : IDisposable, IAsyncDisposable {
+   readonly ILogger logger;
+   readonly WebSocket webSocket;
+   readonly Dictionary<string, DataStreamSubscription> subscriptions = new();
+
+   public DataStreamWebSocketServer(WebSocket webSocket, ILogger logger) {
+      this.webSocket = webSocket;
+      this.logger = logger;
+   }
+
+   public async Task<int> Run(CancellationToken cancellationToken) {
+      logger.LogInformation("DataStreamWebSocketServer running");
+      await foreach (var req in JsonRpcRequestAsyncEnumerable.Create(webSocket, logger, cancellationToken)) {
+         await (req.Method switch {
+            "subscribe" => onSubscribe(req, cancellationToken),
+            "unsubscribe" => onUnsubscribe(req.Deserialize<UnsubscribeRequest>(), cancellationToken),
+            _ => Task.CompletedTask,
+         });
+      }
+      logger.LogInformation("DataStreamWebSocketServer stopped");
+      return 0;
+   }
+
+   /// <summary>
+   /// Disposes internal subscriptions. To additionally notify the client of the subscription status
+   /// change(s), use <see cref="DisposeAsync"/>.
+   /// </summary>
+   public void Dispose() {
+      GC.SuppressFinalize(this);
+      disposeSubscriptions();
+   }
+
+   /// <summary>
+   /// Disposes internal subscriptions (like <see cref="Dispose"/>), then sends 'subscription status change'
+   /// notifications to the client (unlike <see cref="Dispose"/>).
+   /// </summary>
+   public async ValueTask DisposeAsync() {
+      GC.SuppressFinalize(this);
+      var finalSubs = subscriptions.Values.Select(s => (s.RequestId, s.Id)).ToList();
+      disposeSubscriptions();
+      // Send a 'subscription status change' msg to the client for each one.
+      foreach (var (requestId, id) in finalSubs) {
+         await sendSubscriptionStatus(requestId, id, "unsubscribe", SubscriptionStatus.Unsubscribed, CancellationToken.None);
+      }
+   }
+
+   async Task onSubscribe(JsonRpcRequest<JsonNode> req, CancellationToken cancellationToken) {
+      ArgumentNullException.ThrowIfNull(req.Id);
+      ArgumentException.ThrowIfNullOrEmpty(req.Method);
+      if (!SubscribeRequest.ParseQuery(req, out var vs, out var subId, out var errMsg, out var errCode)) {
+         logger.LogError("Invalid 'subscribe' request: {Reason}", errMsg);
+         var rpcError = new JsonRpcError<JsonNode>() {
+            Code = (long)errCode,
+            Message = $"{errCode}: {errMsg}",
+         };
+         await send(rpcError, cancellationToken);
+         return;
+      }
+      if (subscriptions.TryGetValue(subId, out var oldSub)) {
+         oldSub.Dispose();
+      }
+      var sub = new DataStreamSubscription(subId, req.Id.Value, vs, webSocket, logger);
+      subscriptions[subId] = sub;
+      await sendSubscriptionStatus(req.Id.Value, subId, req.Method, SubscriptionStatus.Subscribed, cancellationToken);
+      await sub.StartAsync(cancellationToken);
+   }
+
+   async Task onUnsubscribe(JsonRpcRequest<UnsubscribeRequest> req, CancellationToken cancellationToken) {
+      ArgumentNullException.ThrowIfNull(req.Id);
+      ArgumentException.ThrowIfNullOrEmpty(req.Method);
+      var subId = req.Params?.SubscriptionId ?? string.Empty;
+      if (!subscriptions.TryGetValue(subId, out var sub)) {
+         logger.LogInformation("Ignoring '{Method}' request because no subscription was found with id {Id}.", req.Method, subId);
+         return;
+      }
+      logger.LogInformation("Disposing subscription");
+      sub.Dispose();
+      subscriptions.Remove(subId);
+      await sendSubscriptionStatus(req.Id.Value, subId, req.Method, SubscriptionStatus.Unsubscribed, cancellationToken);
+   }
+
+   async Task unsubscribe(JsonRpcId requestId, string subscriptionId, CancellationToken cancellationToken) {
+      await sendSubscriptionStatus(requestId, subscriptionId, "unsubscribe", SubscriptionStatus.Unsubscribed, cancellationToken);
+   }
+
+   Task sendSubscriptionStatus(JsonRpcId reqId, string subId, string method, SubscriptionStatus status, CancellationToken cancellationToken) {
+      logger.LogInformation("Sending {Method} ack. SubscriptionId = {SubscriptionId}", method, subId);
+      var response = JsonRpcResponse.Create(reqId, new SubscriptionStatusMessage() {
+         SubscriptionId = subId,
+         Status = status,
+      });
+      return send(response, cancellationToken);
+   }
+
+   Task send<T>(T message, CancellationToken cancellationToken) {
+      if (message is null) {
+         return Task.CompletedTask;
+      }
+      var str = JsonSerializer.Serialize(message);
+      var bytes = Encoding.UTF8.GetBytes(str);
+      logger.LogTrace("Sending data. Length = {Length}", bytes.Length);
+      return webSocket.SendAsync(bytes, WebSocketMessageType.Text, true, cancellationToken);
+   }
+
+   void disposeSubscriptions() {
+      logger.LogInformation("Disposing DataStreamWebSocketServer subscriptions. Count: {Count}", subscriptions.Count);
+      subscriptions.Values.DisposeAll();
+      subscriptions.Clear();
+   }
+}

src/DataStreamSuite/Bts.DataStreamWebApp.Server/Messages/ErrorCode.cs

@@ -0,0 +1,6 @@
+namespace Bts.DataStreamWebApp.Server.Messages;
+
+enum ErrorCode {
+   None = 0,
+   InvalidSubscriptionRequest,
+}

src/DataStreamSuite/Bts.DataStreamWebApp.Server/Messages/SubscribeRequest.cs

@@ -0,0 +1,67 @@
+namespace Bts.DataStreamWebApp.Server.Messages;
+
+using BCG;
+
+using bts.websocket_core.json_rpc;
+
+using Bts.DataStreamExpressions;
+
+using System.Diagnostics.CodeAnalysis;
+using System.Text.Json;
+using System.Text.Json.Nodes;
+using System.Text.Json.Serialization;
+
+public class SubscribeRequest {
+   [JsonPropertyName("query")]
+   public string? Query { get; set; }
+
+   [JsonPropertyName("subscription_id")]
+   public string? SubscriptionId { get; set; }
+
+   internal static bool ParseQuery(JsonRpcRequest<JsonNode> req, [NotNullWhen(true)] out ViewStream? stream, [NotNullWhen(true)] out string? subscriptionId, [NotNullWhen(false)] out string? errorMessage, out ErrorCode errorCode) {
+      stream = null;
+      subscriptionId = null;
+      errorCode = ErrorCode.None;
+      var paramsJsonObj = req.Params?.AsObject();
+      if (paramsJsonObj is null) {
+         errorCode = ErrorCode.InvalidSubscriptionRequest;
+         errorMessage = "'params' object not found";
+         return false;
+      }
+      var subReq = paramsJsonObj.Deserialize<SubscribeRequest>();
+      if (subReq is null) {
+         errorCode = ErrorCode.InvalidSubscriptionRequest;
+         errorMessage = "'params' object invalid";
+         return false;
+      }
+      subscriptionId = subReq.SubscriptionId ?? string.Empty;
+      var query = subReq.Query;
+      if (query is null) {
+         errorCode = ErrorCode.InvalidSubscriptionRequest;
+         errorMessage = "'params.query' field not found (expected a string)";
+         return false;
+      }
+      errorMessage = null;
+      var exp = DataStreamExpressionsInterpreter.Evaluate(query);
+      if (exp is null) {
+         errorCode = ErrorCode.InvalidSubscriptionRequest;
+         errorMessage = "'params.query' expression has invalid syntax";
+         return false;
+      }
+      string? exceptionMsg = null;
+      try {
+         stream = exp.getView(new Dictionary<string, StatementExpression>(0));
+      } catch (Exception ex) {
+         exceptionMsg = ex.ToString();
+      }
+      if (stream is null) {
+         errorCode = ErrorCode.InvalidSubscriptionRequest;
+         errorMessage = "Failed to create ViewStream from 'params.query' expression (check for typos in arguments/typenames)";
+         if (exceptionMsg is not null) {
+            errorMessage += ". Error" + exceptionMsg;
+         }
+         return false;
+      }
+      return true;
+   }
+}

src/DataStreamSuite/Bts.DataStreamWebApp.Server/Messages/SubscriptionStatusMessage.cs

@@ -0,0 +1,18 @@
+namespace Bts.DataStreamWebApp.Server.Messages;
+
+using System.Text.Json.Serialization;
+
+enum SubscriptionStatus {
+   None,
+   Subscribed,
+   Unsubscribed,
+}
+
+class SubscriptionStatusMessage {
+   [JsonPropertyName("subscription_id")]
+   public string? SubscriptionId { get; set; }
+
+   [JsonPropertyName("status")]
+   [JsonConverter(typeof(JsonStringEnumConverter))]
+   public SubscriptionStatus Status { get; set; } = SubscriptionStatus.None;
+}

src/DataStreamSuite/Bts.DataStreamWebApp.Server/Messages/UnsubscribeRequest.cs

@@ -0,0 +1,8 @@
+namespace Bts.DataStreamWebApp.Server.Messages;
+
+using System.Text.Json.Serialization;
+
+public class UnsubscribeRequest {
+   [JsonPropertyName("subscription_id")]
+   public string? SubscriptionId { get; set; }
+}

src/DataStreamSuite/Bts.DataStreamWebApp.Server/Program.cs

@@ -0,0 +1,59 @@
+using Bts.DataStreamCore.Services;
+using Bts.PubSub.Services;
+
+using Spark.Api.Services;
+
+using System.Reactive.Linq;
+
+var builder = WebApplication.CreateBuilder(new WebApplicationOptions {
+   Args = args,
+   ApplicationName = "Bts.DataStreamWebApp.Server",
+});
+ConfigureApp(builder.Configuration);
+ConfigureServices(builder.Services, builder.Configuration);
+
+var app = builder.Build();
+if (app.Environment.IsDevelopment()) {
+   app.UseDirectoryBrowser();
+}
+var wsOpts = new WebSocketOptions {
+   KeepAliveInterval = TimeSpan.FromMinutes(2)
+};
+app.UseWebSockets(wsOpts);
+app.UseDefaultFiles();
+app.UseStaticFiles();
+if (app.Environment.IsDevelopment()) {
+   app.UseSwagger();
+   app.UseSwaggerUI();
+}
+app.UseHttpsRedirection();
+app.UseAuthorization();
+app.MapControllers();
+try {
+   Console.Write($"Starting Application");
+   app.Run();
+   Console.Write($"Application exited gracefully.");
+} catch (Exception ex) {
+   Console.Error.Write($"Application crashed: {ex}");
+   return 1;
+}
+return 0;
+
+static void ConfigureApp(ConfigurationManager configBuilder) {
+   configBuilder.AddCommandLine(Environment.GetCommandLineArgs());
+   var cfgRoot = configBuilder.GetSection("config");
+   if (cfgRoot?.Value is not null) {
+      configBuilder.AddJsonFile(cfgRoot.Value);
+   }
+}
+
+static void ConfigureServices(IServiceCollection services, IConfiguration config) {
+   services.AddDirectoryBrowser();
+   services.AddControllers();
+   services.AddEndpointsApiExplorer();
+   services.AddSwaggerGen();
+   services.AddMasterWorker(config.GetSection("MasterWorker"));
+   services.AddViewStreamDatabase(config.GetSection("ViewStreamDatabase"));
+   services.AddPubSubClient(config.GetSection("PubSub"));
+   services.AddSparkd(config.GetSection("Sparkd"));
+}

src/DataStreamSuite/Bts.DataStreamWebApp.Server/Properties/launchSettings.json

@@ -0,0 +1,26 @@
+{
+  "$schema": "http://json.schemastore.org/launchsettings.json",
+  "profiles": {
+    "http": {
+      "commandName": "Project",
+      "dotnetRunMessages": true,
+      "launchBrowser": true,
+      "launchUrl": "swagger",
+      "applicationUrl": "http://localhost:5000",
+      "environmentVariables": {
+        "ASPNETCORE_ENVIRONMENT": "Development",
+        "ASPNETCORE_HOSTINGSTARTUPASSEMBLIES": "Microsoft.AspNetCore.SpaProxy",
+      }
+    },
+    "https": {
+      "commandName": "Project",
+      "dotnetRunMessages": true,
+      "launchBrowser": true,
+      "applicationUrl": "http://localhost:5000;https://localhost:5001",
+      "environmentVariables": {
+        "ASPNETCORE_ENVIRONMENT": "Development",
+        "ASPNETCORE_HOSTINGSTARTUPASSEMBLIES": "Microsoft.AspNetCore.SpaProxy",
+      }
+    },
+  },
+}

src/DataStreamSuite/Bts.DataStreamWebApp.Server/appsettings.json

@@ -0,0 +1,9 @@
+{
+  "Logging": {
+    "LogLevel": {
+      "Default": "Information",
+      "Microsoft.AspNetCore": "Warning"
+    }
+  },
+  "AllowedHosts": "*"
+}

src/DataStreamSuite/Bts.DataStreamWebApp.Server/wwwroot/alt-index.html

@@ -0,0 +1,168 @@
+<!DOCTYPE html>
+<html>
+<head>
+    <meta charset="utf-8" />
+    <title></title>
+    <style>
+        table {
+            border: 0
+        }
+
+        .commslog-data {
+            font-family: Consolas, Courier New, Courier, monospace;
+        }
+
+        .commslog-server {
+            background-color: red;
+            color: white
+        }
+
+        .commslog-client {
+            background-color: green;
+            color: white
+        }
+    </style>
+</head>
+<body>
+    <h1>ALT WebSocket Sample Application</h1>
+    <p id="stateLabel">Ready to connect...</p>
+    <div>
+        <label for="connectionUrl">WebSocket Server URL:</label>
+        <input id="connectionUrl" />
+        <button id="connectButton" type="submit">Connect</button>
+    </div>
+    <p></p>
+    <div>
+        <label for="sendMessage">Message to send:</label>
+        <input id="sendMessage" disabled />
+        <button id="sendButton" type="submit" disabled>Send</button>
+        <button id="closeButton" disabled>Close Socket</button>
+    </div>
+
+    <h2>Communication Log</h2>
+    <table style="width: 800px">
+        <thead>
+            <tr>
+                <td style="width: 100px">From</td>
+                <td style="width: 100px">To</td>
+                <td>Data</td>
+            </tr>
+        </thead>
+        <tbody id="commsLog">
+        </tbody>
+    </table>
+
+    <script>
+        var connectionUrl = document.getElementById("connectionUrl");
+        var connectButton = document.getElementById("connectButton");
+        var stateLabel = document.getElementById("stateLabel");
+        var sendMessage = document.getElementById("sendMessage");
+        var sendButton = document.getElementById("sendButton");
+        var commsLog = document.getElementById("commsLog");
+        var closeButton = document.getElementById("closeButton");
+        var socket;
+
+        var scheme = document.location.protocol === "https:" ? "wss" : "ws";
+        var port = document.location.port ? (":" + document.location.port) : "";
+
+        connectionUrl.value = scheme + "://" + document.location.hostname + port + "/ws" ;
+
+        function updateState() {
+            function disable() {
+                sendMessage.disabled = true;
+                sendButton.disabled = true;
+                closeButton.disabled = true;
+            }
+            function enable() {
+                sendMessage.disabled = false;
+                sendButton.disabled = false;
+                closeButton.disabled = false;
+            }
+
+            connectionUrl.disabled = true;
+            connectButton.disabled = true;
+
+            if (!socket) {
+                disable();
+            } else {
+                switch (socket.readyState) {
+                    case WebSocket.CLOSED:
+                        stateLabel.innerHTML = "Closed";
+                        disable();
+                        connectionUrl.disabled = false;
+                        connectButton.disabled = false;
+                        break;
+                    case WebSocket.CLOSING:
+                        stateLabel.innerHTML = "Closing...";
+                        disable();
+                        break;
+                    case WebSocket.CONNECTING:
+                        stateLabel.innerHTML = "Connecting...";
+                        disable();
+                        break;
+                    case WebSocket.OPEN:
+                        stateLabel.innerHTML = "Open";
+                        enable();
+                        break;
+                    default:
+                        stateLabel.innerHTML = "Unknown WebSocket State: " + htmlEscape(socket.readyState);
+                        disable();
+                        break;
+                }
+            }
+        }
+
+        closeButton.onclick = function () {
+            if (!socket || socket.readyState !== WebSocket.OPEN) {
+                alert("socket not connected");
+            }
+            socket.close(1000, "Closing from client");
+        };
+
+        sendButton.onclick = function () {
+            if (!socket || socket.readyState !== WebSocket.OPEN) {
+                alert("socket not connected");
+            }
+            var data = sendMessage.value;
+            socket.send(data);
+            commsLog.innerHTML += '<tr>' +
+                '<td class="commslog-client">Client</td>' +
+                '<td class="commslog-server">Server</td>' +
+                '<td class="commslog-data">' + htmlEscape(data) + '</td></tr>';
+        };
+
+        connectButton.onclick = function() {
+            stateLabel.innerHTML = "Connecting";
+            socket = new WebSocket(connectionUrl.value);
+            socket.onopen = function (event) {
+                updateState();
+                commsLog.innerHTML += '<tr>' +
+                    '<td colspan="3" class="commslog-data">Connection opened</td>' +
+                '</tr>';
+            };
+            socket.onclose = function (event) {
+                updateState();
+                commsLog.innerHTML += '<tr>' +
+                    '<td colspan="3" class="commslog-data">Connection closed. Code: ' + htmlEscape(event.code) + '. Reason: ' + htmlEscape(event.reason) + '</td>' +
+                '</tr>';
+            };
+            socket.onerror = updateState;
+            socket.onmessage = function (event) {
+                commsLog.innerHTML += '<tr>' +
+                    '<td class="commslog-server">Server</td>' +
+                    '<td class="commslog-client">Client</td>' +
+                    '<td class="commslog-data">' + htmlEscape(event.data) + '</td></tr>';
+            };
+        };
+
+        function htmlEscape(str) {
+            return str.toString()
+                .replace(/&/g, '&amp;')
+                .replace(/"/g, '&quot;')
+                .replace(/'/g, '&#39;')
+                .replace(/</g, '&lt;')
+                .replace(/>/g, '&gt;');
+        }
+    </script>
+</body>
+</html>

src/DataStreamSuite/Bts.DataStreamWebApp.Server/wwwroot/index.html

@@ -0,0 +1,176 @@
+<!DOCTYPE html>
+<html>
+<head>
+    <meta charset="utf-8" />
+    <title></title>
+    <style>
+        table {
+            border: 0
+        }
+
+        .commslog-data {
+            font-family: Consolas, Courier New, Courier, monospace;
+        }
+
+        .commslog-server {
+            background-color: red;
+            color: white
+        }
+
+        .commslog-client {
+            background-color: green;
+            color: white
+        }
+    </style>
+</head>
+<body>
+    <h1>DataStreams via WebSocket</h1>
+    <p id="stateLabel">Ready to connect...</p>
+    <div>
+        <label for="connectionUrl">WebSocket Server URL:</label>
+        <input id="connectionUrl" />
+        <button id="connectButton" type="submit">Connect</button>
+    </div>
+    <p></p>
+    <div>
+        <label for="sendMessage">Query:</label>
+        <input id="sendMessage" disabled />
+        <button id="sendButton" type="submit" disabled>Send</button>
+        <button id="closeButton" disabled>Close Socket</button>
+    </div>
+
+    <h2>Communication Log</h2>
+    <table style="width: 800px">
+        <thead>
+            <tr>
+                <td style="width: 100px">From</td>
+                <td style="width: 100px">To</td>
+                <td>Data</td>
+            </tr>
+        </thead>
+        <tbody id="commsLog">
+        </tbody>
+    </table>
+
+    <script>
+        var connectionUrl = document.getElementById("connectionUrl");
+        var connectButton = document.getElementById("connectButton");
+        var stateLabel = document.getElementById("stateLabel");
+        var sendMessage = document.getElementById("sendMessage");
+        var sendButton = document.getElementById("sendButton");
+        var commsLog = document.getElementById("commsLog");
+        var closeButton = document.getElementById("closeButton");
+        var socket;
+
+        var scheme = document.location.protocol === "https:" ? "wss" : "ws";
+        var port = document.location.port ? (":" + document.location.port) : "";
+
+        connectionUrl.value = scheme + "://" + document.location.hostname + port + "/ws" ;
+
+        function updateState() {
+            function disable() {
+                sendMessage.disabled = true;
+                sendButton.disabled = true;
+                closeButton.disabled = true;
+            }
+            function enable() {
+                sendMessage.disabled = false;
+                sendButton.disabled = false;
+                closeButton.disabled = false;
+            }
+
+            connectionUrl.disabled = true;
+            connectButton.disabled = true;
+
+            if (!socket) {
+                disable();
+            } else {
+                switch (socket.readyState) {
+                    case WebSocket.CLOSED:
+                        stateLabel.innerHTML = "Closed";
+                        disable();
+                        connectionUrl.disabled = false;
+                        connectButton.disabled = false;
+                        break;
+                    case WebSocket.CLOSING:
+                        stateLabel.innerHTML = "Closing...";
+                        disable();
+                        break;
+                    case WebSocket.CONNECTING:
+                        stateLabel.innerHTML = "Connecting...";
+                        disable();
+                        break;
+                    case WebSocket.OPEN:
+                        stateLabel.innerHTML = "Open";
+                        enable();
+                        break;
+                    default:
+                        stateLabel.innerHTML = "Unknown WebSocket State: " + htmlEscape(socket.readyState);
+                        disable();
+                        break;
+                }
+            }
+        }
+
+        closeButton.onclick = function () {
+            if (!socket || socket.readyState !== WebSocket.OPEN) {
+                alert("socket not connected");
+            }
+            socket.close(1000, "Closing from client");
+        };
+
+        sendButton.onclick = function () {
+            if (!socket || socket.readyState !== WebSocket.OPEN) {
+                alert("socket not connected");
+            }
+            const rpcReq = {
+              jsonrpc: "2.0",
+              method: "subscribe",
+              params: {
+                query: sendMessage.value,
+              },
+              id: 1,
+            };
+            const rpcReqStr = JSON.stringify(rpcReq);
+            socket.send(rpcReqStr);
+            commsLog.innerHTML += '<tr>' +
+                '<td class="commslog-client">Client</td>' +
+                '<td class="commslog-server">Server</td>' +
+                '<td class="commslog-data">' + htmlEscape(rpcReqStr) + '</td></tr>';
+        };
+
+        connectButton.onclick = function() {
+            stateLabel.innerHTML = "Connecting";
+            socket = new WebSocket(connectionUrl.value);
+            socket.onopen = function (event) {
+                updateState();
+                commsLog.innerHTML += '<tr>' +
+                    '<td colspan="3" class="commslog-data">Connection opened</td>' +
+                '</tr>';
+            };
+            socket.onclose = function (event) {
+                updateState();
+                commsLog.innerHTML += '<tr>' +
+                    '<td colspan="3" class="commslog-data">Connection closed. Code: ' + htmlEscape(event.code) + '. Reason: ' + htmlEscape(event.reason) + '</td>' +
+                '</tr>';
+            };
+            socket.onerror = updateState;
+            socket.onmessage = function (event) {
+                commsLog.innerHTML += '<tr>' +
+                    '<td class="commslog-server">Server</td>' +
+                    '<td class="commslog-client">Client</td>' +
+                    '<td class="commslog-data">' + htmlEscape(event.data) + '</td></tr>';
+            };
+        };
+
+        function htmlEscape(str) {
+            return str.toString()
+                .replace(/&/g, '&amp;')
+                .replace(/"/g, '&quot;')
+                .replace(/'/g, '&#39;')
+                .replace(/</g, '&lt;')
+                .replace(/>/g, '&gt;');
+        }
+    </script>
+</body>
+</html>

src/DataStreamSuite/DataStream/DataStream.csproj

@@ -18,11 +18,8 @@
 
   <ItemGroup>
     <PackageReference Include="HtmlAgilityPack" Version="1.11.57" />
-    <PackageReference Include="Microsoft.Data.Analysis" Version="0.21.1" />
     <PackageReference Include="MongoDB.Bson" Version="2.23.1" />
     <PackageReference Include="MongoDB.Driver" Version="2.23.1" />
-    <PackageReference Include="Parquet.Net" Version="5.0.1" />
-    <PackageReference Include="System.Data.SQLite" Version="1.0.118" />
   </ItemGroup>
 
 </Project>

src/DataStreamSuite/DataStream/Misc/DSLStream.cs

@@ -1,32 +0,0 @@
-namespace bts.SilverPine {
-   using System;
-   using BCG;
-   using System.Collections.Generic;
-   using Bts.DataStreamExpressions;
-
-   // All this does is take in a dsl expression, parse it and fire the data that come out of running it.
-   // So for example, if an expresison is passed 
-   [DataStreamUserRespresentation("dsl")]
-   public class DSLStream : ViewStream {
-      private readonly string _dsl;
-      public DSLStream(string dsl) {
-         Log.Gen.info($"DSLStream.cons dsl: {dsl}");
-         _dsl = dsl;
-         ViewStream rs = build_stream(dsl);
-         rs = rs ?? new LiteralStream("error", $"msg:invalid dsl \"{dsl}\"");
-         mySetRealSource(rs);
-      }
-
-      static ViewStream? build_stream(string stream_expression) {
-         try {
-            var exp = DataStreamExpressionsInterpreter.Evaluate(stream_expression);
-            var rv = exp.getView(new Dictionary<string, StatementExpression>(0));
-            return rv;
-
-         } catch (Exception ex) {
-            Log.Gen.error($"Could not build stream: {stream_expression} ex: {ex}");
-            return null;
-         }
-      }
-   }
-}

src/DataStreamSuite/DataStream/Offloading/RemoteQueryManager.cs

@@ -4,6 +4,8 @@ using System.Collections.Generic;
 using Bts.DataStreamExpressions;
 using BCG.Utilities.General;
 using bts;
+using Bts.PubSub;
+using Bts.DataStreamCore;
 
 namespace BCG {
    // listens to the requests posted on "cloud_streams" request table, and 
@@ -40,13 +42,15 @@ namespace BCG {
       private IDictionary<string, StreamPubUnit>
          _stream_pub_map = new Dictionary<string, StreamPubUnit>();
       private ViewStream _stream = new TableStream("cloud_streams");
-      private PubTableBroker _broker = new PubTableBroker();
-      private ManagedSubscriber _sub = ManagedSubscriber.instance;
+      readonly IPubTableBroker _broker;
+      readonly IManagedSubscriber _sub;
 
 
-      public RemoteQueryManager() {
+      public RemoteQueryManager(IManagedSubscriber managedSubscriber, IPubTableBroker pubTableBroker) {
+         _sub = managedSubscriber;
+         _broker = pubTableBroker;
          _stream.onChange += _on_str;
-         _sub.subscribe("RemoteRequest", (string tag, byte[] data) => {
+         _sub.Subscribe("RemoteRequest", (string tag, byte[] data) => {
             RemoteRequest req = RemoteRequest.construct_from_bytes(data);
             _spawn_remote_stream(req.published_table_name, req.stream_expression, false);
          });

src/DataStreamSuite/DataStream/Primitives/Distinct/DistinctRowStream.cs

@@ -1,24 +1,23 @@
-using bts;
-
 using System;
 using System.Collections;
 
-namespace BCG {
+namespace BCG
+{
 	/// <summary>
 	/// Provides a stream that with distinct rows with respect to the columns
 	/// the user specifies in the argument
 	/// </summary>
-	[DataStreamUserRespresentation("distinct")]
-	public class DistinctRowStream : ViewStream {
-		public DistinctRowStream(ViewStream stream, params string[] cols) {
-			var g_by_eval = new DistinctRowEvaluator(cols);
-			ViewStream r = new GroupByStream(stream, g_by_eval);
-			var proj_exp = String.Empty;
-			foreach (var c in cols)
-				proj_exp += $"{c} ";
-			proj_exp = proj_exp.Trim();
-			r = new NewProjectStream(r, proj_exp);
-			mySetRealSource(r);
+	[DataStreamUserRespresentation ("distinct")]
+	public class DistinctRowStream : ViewStream
+	{
+		public DistinctRowStream (ViewStream stream, params string[] cols)
+		{
+         ArrayList groupByCols = new ArrayList ();
+         foreach (string col in cols)
+            foreach (string c in col.Split (' '))
+               groupByCols.Add (c);
+         mySetRealSource (new GroupByStream (stream, 
+            new DistinctRowEvaluator (groupByCols.ToArray (typeof(string)) as string[]), false));
 		}
 	}
 }

src/DataStreamSuite/DataStream/Primitives/FileStreams/DumpLoaderStream.cs

@@ -0,0 +1,49 @@
+// using System;
+// using System.Collections.Generic;
+// using System.Text;
+// using System.IO;
+
+// using System.Runtime.Serialization.Formatters.Binary;
+
+// namespace BCG
+// {
+//    public class DumpLoaderStream : ViewStream, ISideEffectStream
+//    {
+//       //wtf
+//       //public class Exception : System.Exception
+//       //{public Exception(string msg) : base(msg) { }}
+
+//       public DumpLoaderStream(string fileName, int secondsBetweenPolls)
+//       {
+//          mySetRealSource(
+//             new PollStream(() => new LoaderStream(fileName).rows, secondsBetweenPolls, 
+//                ViewStreamDatabase.instance));
+//       }
+
+//       public DumpLoaderStream(string fileName)
+//       {
+//         if (fileName == null)
+//            throw new LoaderStream.Exception("File name cannot be null");
+
+//         for (int i = 0; i < 5; i++)
+//            try
+//            {
+//               using (FileStream fs = new FileStream(fileName, FileMode.Open, FileAccess.Read))
+//               {
+//                  IClientDeserializer deser = new ClientUniversalDeserializer();
+//                  byte[] the_ops;
+//                  Array the_rows;
+//                  deser.deserialize(new Stream[] { fs }, out the_ops, out the_rows);
+
+//                  mySetRealSource(new BlueRowCollectionStaticStream((IBlueRowCollection)the_rows.GetValue(0)));
+//               }
+//               break;
+//            }
+//            catch (IOException)
+//            {
+//               System.Threading.Thread.Sleep(50);
+//            }
+
+//       }
+//    }
+// }

src/DataStreamSuite/DataStream/Primitives/FileStreams/LoaderStream.cs

@@ -1,7 +1,6 @@
 using BCG;
 
 namespace bts {
-   // This is the other side of the SaverStream, its "loads" the stream and fires once
    [DataStreamUserRespresentation("loader")]
    public class LoaderStream : ViewStream {
 

src/DataStreamSuite/DataStream/Primitives/FileStreams/PersistorStream.cs

@@ -5,11 +5,10 @@ using System.Text;
 using System.IO;
 using Google.Protobuf;
 using System.Globalization;
-using BCG;
+using bts;
 
-namespace bts {
-   // Writes all the udpates to a stream in a file or multiples files dependong on the rollover size (as opposed to Saver
-   // which just writes one update, as a create)
+namespace BCG {
+   // Writes all the udpates to a stream in a file.
    // File naming:
    // token_epoch_yyyyMMdd_HHmmss
    // File location: directory(passed)\token\
@@ -95,10 +94,10 @@ namespace bts {
       void _on_change(object new_value) {
          DataPacket dp = new_value as DataPacket;
          byte[] serialized_update = null;
-         bool first_write = _serializer == null || _n_updates_in_current_file == 0;
+         bool _write_create = _serializer == null || _n_updates_in_current_file == 0;
          _serializer = _serializer ?? new UniversalStreamIOSerializer();
-         if (first_write) {
-            // first file always starts with create
+         if (_write_create) {
+            // new files always starts with create
             serialized_update = _serializer.serialize(new bts.StreamIOMessage(new bts.StreamIOMessage.CreateMessage(dp.newRows)));
          } else
             serialized_update = _serializer.serialize(new bts.StreamIOMessage(new bts.StreamIOMessage.MultipleRemoveAddMessage(dp.deletedRows, dp.addedRows)));
@@ -106,8 +105,8 @@ namespace bts {
          long utc_epoch = epoch_now();
 
          //time, length of data, data
-         //bw.Write(utc_epoch);
-         //bw.Write(serialized_update.Length);
+         bw.Write(utc_epoch);
+         bw.Write(serialized_update.Length);
          bw.Write(serialized_update);
          _out_fs.Flush();
          myFireEvent(dp);

src/DataStreamSuite/DataStream/Primitives/FileStreams/PlaybackStream.cs

@@ -0,0 +1,190 @@
+// using System;
+// using System.Collections.Generic;
+// using System.Linq;
+// using System.Text;
+// using System.IO;
+// using System.Runtime.Serialization.Formatters.Binary;
+// using BCG.Utilities.General;
+// using bts.collections;
+// using System.Reflection.Metadata;
+
+
+// namespace BCG {
+
+//    public class PlaybackStream : ViewStream, IWorker {
+//       private readonly PlaybackType _type;
+//       private BlueTable _current_table;
+//       private IClientDeserializer _deserilializer = new ClientUniversalDeserializer();
+//       // these are the packets already downloaded from the current file
+//       private List<PlaybackPacket> _packets = new List<PlaybackPacket>();
+//       //private IEnumerable<FileStream> _file_streams = new List<FileStream>();
+//       private IEnumerable<string> _file_names;
+//       private Tuple<int, FileStream> _current_fs = new Tuple<int, FileStream>(-1, null);
+//       private DateTime _playback_t0, _playback_t1;
+//       private bool _all_files_read = false;
+//       SnapshotData _snapshot_data = null; // only initialized when in snapshot mode
+
+//       public PlaybackStream(string datetime_start, string datetime_end, string directory_name, string token) {
+//          _type = PlaybackType.Streaming;
+//          _playback_t0 = PersistorStream.parse_datetime(datetime_start);
+//          _playback_t1 = PersistorStream.parse_datetime(datetime_end);
+//          string token_directory_name = $"{directory_name}\\{token}";
+//          bts.Log.Gen.info($"PlaybackStream t0: {_playback_t0} t1: {_playback_t1} dir: {token_directory_name}");
+//          List<string> files_in_range = new List<string>();
+//          if (Directory.Exists(token_directory_name)) {
+//             _file_names =
+//          Directory.GetFiles(token_directory_name).Where(f => {
+//             DateTime file_time = PersistorStream.parse_datetime_from_filename(f);
+//             return file_time > _playback_t0 && file_time < _playback_t1;
+//          });
+//          }
+//       }
+
+//       protected override void myOnFirstSubscriber() {
+//          FileStream fs = new FileStream(_snapshot_data.snapshot_file, FileMode.Open, FileAccess.Read);
+//          _snapshot_data.packets = StreamPersistorUtils.load_packets_from_file(fs);
+//          PlaybackPacket snapshot_packet = StreamPersistorUtils.find_packet_at_time(_snapshot_data.packets, _snapshot_data.snapshot_time);
+//          if (snapshot_packet != null) {
+//             foreach (PlaybackPacket p in _snapshot_data.packets) {
+//                byte[] ops;
+//                Array row_array;
+//                _deserilializer.deserialize(new Stream[] { new MemoryStream(p.data) }, out ops, out row_array);
+//                for (int i = 0; i < ops.Length; i++) {
+//                   IBlueRowCollection[] rows_args = row_array.GetValue(i) as IBlueRowCollection[];
+//                   byte op = ops[i];
+//                   if (op == StreamOpCode.create) {
+//                      IBlueRowCollection full_table = rows_args[0];
+//                      _current_table = new BlueTable(full_table);
+//                   } else if (op == StreamOpCode.multipleRemoveAdd) {
+//                      //IBlueRowCollection[] del_added = rows_args.GetValue(0) as IBlueRowCollection[];
+//                      IBlueRowCollection dels = rows_args.GetValue(0) as IBlueRowCollection;
+//                      IBlueRowCollection adds = rows_args.GetValue(1) as IBlueRowCollection;
+//                      BlueDataUtility.absorbChanges(_current_table, dels, adds);
+//                   } else {
+//                      bts.Log.assert(false, $"Oprator: {op} is not expected");
+//                   }
+//                   if (p.idx == snapshot_packet.idx) {
+//                      myFireEvent(new DataPacket(_current_table, _current_table, new BlueTable(_current_table.schema)));
+//                      break;
+//                   }
+//                }
+//             }
+//          }
+//       }
+
+
+//       private void _load_packets_from_current_file() {
+//          int curr_idx = _current_fs.Item1;
+//          FileStream curr_fs = _current_fs.Item2;
+//          _packets = new List<PlaybackPacket>();
+//          _packets.AddRange(StreamPersistorUtils.load_packets_from_file(curr_fs));
+//       }
+
+//       private bool current_file_all_replayed {
+//          get {
+//             bool rv = true;
+//             _packets.ForEach(p => rv = rv && p.replayed);
+//             return rv;
+//          }
+//       }
+
+//       protected override void myOnLastSubscriber() {
+//          var fs = _current_fs.Item2;
+//          if (fs != null) {
+//             fs.Close();
+//          }
+//          ViewStreamDatabase.instance.removeWorker(this);
+//       }
+
+//       // public event PlaybackStreamProgress onProgress;
+//       // public event Action onFileExhausted;
+//       private PlaybackPacket _locate_packet_to_replay() {
+//          foreach (var p in _packets) {
+//             // returns the first non replayed packet
+//             if (!p.replayed)
+//                return p;
+//          }
+//          //else
+//          {
+//             double secondsElapsed = new BinaryReader(myFileReader).ReadDouble();
+//             myNextPacketTime = myPlaybackStart + new TimeSpan(0, 0, 0, 0, (int)(1000 * secondsElapsed / myPlaybackSpeed));
+//             if (onProgress != null)
+//                onProgress(myFileReader.Position, myFileReader.Length);
+//          }
+//       }
+
+//       #region IWorker Members
+
+//       public void doWork() {
+//          if (_all_files_read)
+//             return;
+//          if (current_file_all_replayed) {
+//             _load_next_file();
+//          }
+//          PlaybackPacket curr_packet = _locate_packet_to_replay();
+//          if (curr_packet.idx == 0) { // first packet in the file, fire it rightaway 
+//             byte[] ops;
+//             Array row_array;
+//             // what a stupid interface!
+//             _deserilializer.deserialize(new Stream[] { new MemoryStream(curr_packet.data) }, out ops, out row_array);
+//             _fire_packet(ops, row_array);
+//             curr_packet.replay_timestamp = PersistorStream.epoch_now();
+//          } else {
+//             PlaybackPacket prev_packet = _get_previous_packet(curr_packet);
+//             // this is the time we need to fire (replay) this packet
+//             long time_to_fire = prev_packet.replay_timestamp + curr_packet.offset_from_previous;
+//             long now_epoch = PersistorStream.epoch_now();
+//             // fire when we are past the 
+//             if (now_epoch >= time_to_fire) {
+//                byte[] ops;
+//                Array row_array;
+//                _deserilializer.deserialize(new Stream[] { new MemoryStream(curr_packet.data) }, out ops, out row_array);
+//                _fire_packet(ops, row_array);
+//                curr_packet.replay_timestamp = PersistorStream.epoch_now();
+//             }
+//          }
+//       }
+
+//       private void _fire_packet(byte[] ops, Array row_array) {
+//          for (int i = 0; i < ops.Length; i++) {
+//             IBlueRowCollection[] rows_args = row_array.GetValue(i) as IBlueRowCollection[];
+//             byte op = ops[i];
+//             if (op == StreamOpCode.create) {
+//                IBlueRowCollection full_table = rows_args[0];
+//                _current_table = new BlueTable(full_table);
+//                //_current_table.addRows(full_table);
+//                myFireEvent(new DataPacket(full_table, full_table, myLastValue == null ? new BlueTable(full_table.schema) :
+//                                           (myLastValue as DataPacket).newRows));
+//             } else if (op == StreamOpCode.multipleRemoveAdd) {
+//                //IBlueRowCollection[] del_added = rows_args.GetValue(0) as IBlueRowCollection[];
+//                IBlueRowCollection dels = rows_args.GetValue(0) as IBlueRowCollection;
+//                IBlueRowCollection adds = rows_args.GetValue(1) as IBlueRowCollection;
+//                BlueDataUtility.absorbChanges(_current_table, dels, adds);
+//                myFireEvent(new DataPacket(_current_table, adds, dels));
+//             } else {
+//                bts.Log.assert(false, $"Oprator: {op} is not expected");
+//             }
+
+//          }
+//       }
+
+//       private void _load_next_file() {
+//          var new_idx = _current_fs.Item1 + 1;
+//          if (new_idx >= _file_names.Count()) {
+//             _all_files_read = true;
+//             bts.Log.Gen.info("All files read");
+//             return;
+//          }
+//          var prev_fs = _current_fs.Item2;
+//          if (prev_fs != null)
+//             prev_fs.Close();
+//          var new_filename = _file_names.ElementAt(new_idx);
+//          bts.Log.Gen.info("PlaybackStream_load_next_file new_filename=: {new_filename}");
+//          var new_fs = new FileStream(new_filename, FileMode.Open, FileAccess.Read);
+//          _current_fs = new Tuple<int, FileStream>(new_idx, new_fs);
+//          _load_packets_from_current_file();
+//       }
+
+//       #endregion
+//    }
+// }

src/DataStreamSuite/DataStream/Primitives/FileStreams/RecorderStream.cs

@@ -0,0 +1,68 @@
+// using System;
+// using System.Collections.Generic;
+// using System.Linq;
+// using System.Text;
+// using System.IO;
+
+// namespace BCG
+// {
+//    public class RecorderStream : ViewStream
+//    {
+//       private ViewStream mySource;
+//       private DateTime myStartTime;
+//       private FileStream myOutFile;
+//       private string myFileName;
+//       private IBlueDataSerializer mySerializer;
+
+//       public RecorderStream(ViewStream src, string recordFile)
+//       {
+//          mySource = src;
+//          myFileName = recordFile;
+
+//          myFileName = myFileName.Replace("%DATE%", DateTime.Now.ToString("yyyyMMdd_HHmmss")); 
+//       }
+
+//       protected override void myOnFirstSubscriber()
+//       {
+//          myStartTime = DateTime.Now;         
+//          mySource.onChange += new ChangeDelegate(mySource_onChange);
+//          myOutFile = new FileStream(myFileName, FileMode.OpenOrCreate);
+//       }
+
+//       protected override void myOnLastSubscriber()
+//       {
+//          mySource.onChange -= mySource_onChange;
+//          myOutFile.Close();
+//       }
+
+//       void mySource_onChange(object newValue)
+//       {
+//          DataPacket dp = (DataPacket)newValue;         
+
+//          BlueTable toWrite = new BlueTable(dp.schema);
+
+//          if (mySerializer == null)
+//          {
+//             mySerializer = new UniversalSerializer(dp.schema);
+//          }
+//          foreach (IBlueRow r in dp.newRows)
+//             toWrite.addRow(r);
+
+//          byte[] result;
+
+//          if (myLastValue == null)
+//             mySerializer.serialize(StreamOpCode.create, new IBlueRowCollection[] { dp.newRows }, out result);
+//          else
+//             mySerializer.serialize(StreamOpCode.multipleRemoveAdd, new IBlueRowCollection[] { dp.deletedRows, dp.addedRows }, out result);
+
+//          BinaryWriter bw = new BinaryWriter(myOutFile);
+//          //write down what time this happened
+//          bw.Write((DateTime.Now - myStartTime).TotalSeconds);
+//          bw.Write(result.Length);
+//          bw.Write(result);         
+//          myOutFile.Flush();
+
+//          myFireEvent(dp);
+//       }
+//    }
+// }

src/DataStreamSuite/DataStream/Primitives/FileStreams/SaverStream.cs

@@ -5,8 +5,7 @@ using System.IO;
 using BCG;
 
 namespace bts {
-   // this saves the updates as a Create on the same file, each update calls -save, which
-   // reqrites the whole data to the packet. The other side of this stream is the Loader
+   // this saves the total state in the filr provided in the cons
    [DataStreamUserRespresentation("saver")]
    public class SaverStream : ViewStream {
       private ViewStream _stream;
@@ -32,6 +31,7 @@ namespace bts {
          StreamIOMessage.CreateMessage cr_msg = new StreamIOMessage.CreateMessage(dp.newRows);
          byte[] str_bytes = _serializer.serialize(new StreamIOMessage(cr_msg));
          StreamSerializationUtils.write_to_file(str_bytes, _filename);
+
       }
    }
 }

src/DataStreamSuite/DataStream/Primitives/FileStreams/StreamPersistorUtils.cs

@@ -3,7 +3,6 @@ using System.Collections.Generic;
 using System.Linq;
 using System.Globalization;
 using System.IO;
-using bts;
 
 
 namespace BCG {

src/DataStreamSuite/DataStream/Primitives/Filter/NullFilterStream.cs

@@ -1,24 +0,0 @@
-namespace bts.SilverPine {
-   using BCG;
-
-   using System;
-
-   public class NullFilterStream : ViewStream {
-      public NullFilterStream(ViewStream parent_str, string col_name)
-      : this(parent_str, col_name, true) { }
-      public NullFilterStream(ViewStream parent_str, string col_name, bool filter_out_doublenans = false) {
-         ViewStream rs = new FilterStream(parent_str,
-         r => {
-            bool is_dbnull = r.is_null(col_name);
-            if (is_dbnull) {
-               return false;
-            }
-            if (r.schema[col_name].type_code() == TypeCode.Double) {
-               return !Double.IsNaN(r.get<double>(col_name));
-            }
-            return true;
-         });
-         mySetRealSource(rs);
-      }
-   }
-}

src/DataStreamSuite/DataStream/Primitives/GroupBy/AverageGroupsStream.cs

@@ -1,102 +1,102 @@
-// using System;
-// using System.Collections.Generic;
-// using System.Linq;
-// using System.Text;
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
 
-// namespace BCG
-// {
+namespace BCG
+{
    
-//     public class AverageGroupsStream : ViewStream
-//     {
-//         public AverageGroupsStream(ViewStream input, string colToAverage, string keyColumn)
-//         {
+    public class AverageGroupsStream : ViewStream
+    {
+        public AverageGroupsStream(ViewStream input, string colToAverage, string keyColumn)
+        {
             
-//             myColToAverage = colToAverage;
-//             myInput = input;
-//             myKeyColumn = keyColumn;
-//         }
+            myColToAverage = colToAverage;
+            myInput = input;
+            myKeyColumn = keyColumn;
+        }
 
-//         private ViewStream myInput;
-//         private string myColToAverage;
-//         private BlueTableSchema mySchema;
-//         private string myKeyColumn;
+        private ViewStream myInput;
+        private string myColToAverage;
+        private BlueTableSchema mySchema;
+        private string myKeyColumn;
 
-//         protected override void myOnFirstSubscriber()
-//         {
-//             myInput.onChange += new ChangeDelegate(myInput_onChange);
-//         }
-//         protected override void myOnLastSubscriber()
-//         {
-//             myInput.onChange -= myInput_onChange;
-//         }
-//         private void myInput_onChange(object newValue)
-//         {
-//             if (mySchema == null)
-//                 myBuildSchema(((DataPacket)newValue).schema);
+        protected override void myOnFirstSubscriber()
+        {
+            myInput.onChange += new ChangeDelegate(myInput_onChange);
+        }
+        protected override void myOnLastSubscriber()
+        {
+            myInput.onChange -= myInput_onChange;
+        }
+        private void myInput_onChange(object newValue)
+        {
+            if (mySchema == null)
+                myBuildSchema(((DataPacket)newValue).schema);
 
-//             IBlueRowCollection newRows = myEvaluate((DataPacket)newValue);
+            IBlueRowCollection newRows = myEvaluate((DataPacket)newValue);
 
-//             if (myLastValue == null)
-//                 myFireEvent(new DataPacket(newRows, newRows, new BlueTable(mySchema)));
-//             else
-//             {
-//                 IBlueRowCollection added, deleted;
-//                 BlueDataUtility.computeDifferences(newRows, ((DataPacket) myLastValue).newRows, out deleted, out added);
-//                 myFireEvent(new DataPacket(newRows, added, deleted));
-//             }
-//         }
+            if (myLastValue == null)
+                myFireEvent(new DataPacket(newRows, newRows, new BlueTable(mySchema)));
+            else
+            {
+                IBlueRowCollection added, deleted;
+                BlueDataUtility.computeDifferences(newRows, ((DataPacket) myLastValue).newRows, out deleted, out added);
+                myFireEvent(new DataPacket(newRows, added, deleted));
+            }
+        }
 
-//         private void myBuildSchema(IBlueTableSchema schema)
-//         {
-//             List<ColumnInfo> columns = new List<ColumnInfo>();
+        private void myBuildSchema(IBlueTableSchema schema)
+        {
+            List<ColumnInfo> columns = new List<ColumnInfo>();
 
-//             foreach (ColumnInfo ci in schema.columnInfo)
-//                 columns.Add(ci);
+            foreach (ColumnInfo ci in schema.columnInfo)
+                columns.Add(ci);
 
-//             columns.Add(new ColumnInfo("average_"+myColToAverage, columns.Count, typeof(double)));
-//             mySchema = new BlueTableSchema("average_of_" + schema.tableName, columns.ToArray());
-//         }
+            columns.Add(new ColumnInfo("average_"+myColToAverage, columns.Count, typeof(double)));
+            mySchema = new BlueTableSchema("average_of_" + schema.tableName, columns.ToArray());
+        }
 
 
 
-//         private IBlueRowCollection myEvaluate(DataPacket packet)
-//         {
+        private IBlueRowCollection myEvaluate(DataPacket packet)
+        {
          
-//             IDictionary<string, double> colToRunningTotal = new Dictionary<string, double>();
-//             IDictionary<string, int> colToCount = new Dictionary<string, int>();
-//             IDictionary<string, IBlueRow> colToLastRow = new Dictionary<string, IBlueRow>();
+            IDictionary<string, double> colToRunningTotal = new Dictionary<string, double>();
+            IDictionary<string, int> colToCount = new Dictionary<string, int>();
+            IDictionary<string, IBlueRow> colToLastRow = new Dictionary<string, IBlueRow>();
 
-//             foreach (IBlueRow r in packet.newRows)
-//             {
-//                 double thisVal = Convert.ToDouble(r[myColToAverage]);
+            foreach (IBlueRow r in packet.newRows)
+            {
+                double thisVal = Convert.ToDouble(r[myColToAverage]);
 
-//                 string keyVal = r[myKeyColumn].ToString();
-//                 if (!colToCount.ContainsKey(keyVal))
-//                 {
-//                     colToRunningTotal[keyVal] = thisVal;
-//                     colToCount[keyVal] = 1;
-//                 }
-//                 else
-//                 {
-//                     colToRunningTotal[keyVal] += thisVal;
-//                     colToCount[keyVal] += 1;
-//                 }
-//                 colToLastRow[keyVal] = r;
-//             }
+                string keyVal = r[myKeyColumn].ToString();
+                if (!colToCount.ContainsKey(keyVal))
+                {
+                    colToRunningTotal[keyVal] = thisVal;
+                    colToCount[keyVal] = 1;
+                }
+                else
+                {
+                    colToRunningTotal[keyVal] += thisVal;
+                    colToCount[keyVal] += 1;
+                }
+                colToLastRow[keyVal] = r;
+            }
 
-//             BlueTable result = new BlueTable(mySchema);
-//             foreach (string key in colToCount.Keys)
-//             {
-//                 object[] values = new object[mySchema.columnInfo.Length];
+            BlueTable result = new BlueTable(mySchema);
+            foreach (string key in colToCount.Keys)
+            {
+                object[] values = new object[mySchema.columnInfo.Length];
 
-//                 colToLastRow[key].CopyTo(values, 0);
-//                 values[values.Length - 1] = packet.newRows.Count > 0 ? colToRunningTotal[key] / packet.newRows.Count : 0.0;
-//                 result.addRow(new BlueRow(mySchema, values));
-//             }
+                colToLastRow[key].CopyTo(values, 0);
+                values[values.Length - 1] = packet.newRows.Count > 0 ? colToRunningTotal[key] / packet.newRows.Count : 0.0;
+                result.addRow(new BlueRow(mySchema, values));
+            }
 
-//             return result;
-//         }
+            return result;
+        }
 
       
-//     }
-// }
+    }
+}

src/DataStreamSuite/DataStream/Primitives/GroupBy/AverageNonIncGroupsStream.cs

@@ -1,31 +0,0 @@
-namespace bts.SilverPine {
-   using BCG;
-
-   using System;
-   using System.Collections.Generic;
-   using System.Linq;
-
-   [DataStreamUserRespresentation("Average")]
-   public class AverageNonIncGroupsStream : ViewStream {
-      public AverageNonIncGroupsStream(ViewStream parent_stream, string inp_col_name, params string[] group_by_cols) {
-         NonIncGroupByStream rs = new NonIncGroupByStream(parent_stream, () => group_by_cols,
-                                                         (g_rows) => _find_mean(inp_col_name, g_rows));
-         mySetRealSource(rs);
-      }
-
-      private List<IBlueRow> _find_mean(string inp_col_name, List<IBlueRow> g_rows) {
-         IBlueRow inp_row = g_rows.ElementAt(0);
-         IEnumerable<double> inp_vals = BlueDataUtility.get_colvalues<double>(g_rows, inp_col_name);
-         double sum = 0.0;
-         inp_vals.ToList().ForEach(d => sum += d);
-         string out_col_name = $"average_{inp_col_name}";
-         double average = sum / inp_vals.Count();
-         IBlueTableSchema schema_out = inp_row.schema.add_column(TypeCode.Double, out_col_name);
-         MutableBlueRow out_row = new MutableBlueRow(schema_out, inp_row);
-         out_row.setValue(out_col_name, average);
-         List<IBlueRow> rv = new List<IBlueRow>();
-         rv.Add(out_row);
-         return rv;
-      }
-   }
-}

src/DataStreamSuite/DataStream/Primitives/GroupBy/GroupByBundle.cs

@@ -1,91 +0,0 @@
-namespace bts.SilverPine {
-   using BCG;
-
-   using System;
-   using System.Collections.Generic;
-   using System.Linq;
-
-   using ZstdSharp.Unsafe;
-
-   public enum GroupByBundleOp {
-      Mean,
-      Median,
-      Mode,
-      StdDev,
-      Max,
-      Min,
-      First,
-      Last,
-      Sum
-   }
-   public class GroupByBundle {
-      public readonly List<Tuple<string, GroupByBundleOp>> operations;
-
-      public static GroupByBundle parse(string text) {
-         // text is inp_col: op, inp_col: op, inp_col:op
-         List<Tuple<string, GroupByBundleOp>> rv = new List<Tuple<string, GroupByBundleOp>>();
-         string[] calcs = text.Split(',');
-         foreach (string c in calcs) {
-            string[] toks = c.Split(':');
-            string inp_col = toks[0].Trim();
-            string op_txt = toks[1].Trim();
-            GroupByBundleOp op = Enum.Parse<GroupByBundleOp>(op_txt, true);
-            rv.Add(Tuple.Create(inp_col, op));
-         }
-         return new GroupByBundle(rv);
-      }
-
-      public GroupByBundle(List<Tuple<string, GroupByBundleOp>> operations_) {
-         operations = operations_;
-      }
-
-      public static IBlueRow group_by(List<IBlueRow> rows, GroupByBundle parameters) {
-         if (rows.Count() == 0) {
-            return null;
-         }
-
-         IBlueRow ir = rows.ElementAt(0);
-         IBlueTableSchema in_schema = ir.schema;
-         IBlueTableSchema out_schema = in_schema;
-         IDictionary<string, Tuple<string, GroupByBundleOp>> out_col_map =
-            new Dictionary<string, Tuple<string, GroupByBundleOp>>();
-         foreach (var t in parameters.operations) {
-            string out_col_name = $"{t.Item1}_{t.Item2}".ToLower();
-            out_schema = out_schema.add_column(TypeCode.Double, out_col_name);
-            out_col_map[out_col_name] = t;
-         }
-         MutableBlueRow or = new MutableBlueRow(out_schema, ir);
-         foreach (var kv in out_col_map) {
-            string out_col_name = kv.Key;
-            string in_col_name = kv.Value.Item1;
-            GroupByBundleOp op = kv.Value.Item2;
-            IEnumerable<double> col_vals = BlueDataUtility.get_colvalues<double>(rows, in_col_name);
-            double comp_res = compute_op(col_vals, op);
-            or.setValue(out_col_name, comp_res);
-         }
-         return or;
-      }
-
-      public static double compute_op(IEnumerable<double> col_vals, GroupByBundleOp op) {
-         switch (op) {
-            case GroupByBundleOp.Max:
-               return col_vals.Max<double>();
-            case GroupByBundleOp.Min:
-               return col_vals.Min<double>();
-            case GroupByBundleOp.Mean:
-               return col_vals.Sum() / col_vals.Count();
-            case GroupByBundleOp.Sum:
-               return col_vals.Sum();
-            case GroupByBundleOp.First:
-               return col_vals.First();
-            case GroupByBundleOp.Last:
-               return col_vals.Last();
-            default:
-               // TODO
-               Log.assert(false);
-               return Double.NaN;
-         }
-      }
-
-   }
-}

src/DataStreamSuite/DataStream/Primitives/GroupBy/MaxStream.cs

@@ -10,6 +10,6 @@ namespace BCG {
         }
 
         public MaxStream(ViewStream stream, string outputCol)
-           : this(new AddColumnsStream(stream, "fake_col: 0"), outputCol, new string[] { "fake_col" }) { }
+           : this(new AddColumnsStream(stream, "fakeCol=0"), outputCol, new string[] { "fakeCol" }) { }
     }
 }

src/DataStreamSuite/DataStream/Primitives/GroupBy/NonIncremental/NonIncGroupByStream.cs

@@ -1,40 +1,13 @@
 using System;
 using System.Collections.Generic;
 using System.Diagnostics;
-using System.Linq;
-using System.Text.Json;
 
 using bts;
-using bts.SilverPine;
 
-namespace BCG {
-   [DataStreamUserRespresentation("aggregate")]
-   public class NonIncGroupByStream : ViewStream {
-      private class DelNonIncGroupByEvaluator : INonIncGroupByEvaluator {
-         Func<string[]> _groupby_cols_func;
-         Func<List<IBlueRow>, List<IBlueRow>> _evaluate_func;
-         public DelNonIncGroupByEvaluator(Func<string[]> groupby_cols_func,
-                                          Func<List<IBlueRow>, List<IBlueRow>> evaluate_func) {
-            _groupby_cols_func = groupby_cols_func;
-            _evaluate_func = evaluate_func;
-         }
-         public IBlueRowCollection evaluate(IDictionary<int, List<IBlueRow>> grouped_rows) {
-            BlueTable rv = null;
-            foreach (var kv in grouped_rows) {
-               List<IBlueRow> g_rows = kv.Value;
-               List<IBlueRow> out_rows = _evaluate_func.Invoke(g_rows);
-               if (out_rows != null && out_rows.Count > 0) {
-                  rv = rv ?? new BlueTable(out_rows.ElementAt(0).schema);
-               }
-               rv.addRows(out_rows);
-            }
-            return rv;
-         }
-         public string[] get_group_by_cols() {
-            return _groupby_cols_func();
-         }
-
-      }
+namespace BCG
+{
+   public class NonIncGroupByStream : ViewStream
+   {
       private ViewStream _main_stream;
       private INonIncGroupByEvaluator _groupby_func;
       private BlueRowHasher _row_hasher;
@@ -42,47 +15,24 @@ namespace BCG {
       private Stopwatch _timer = new();
       private int _n_total_fires = 0, _n_expensive_fires = 0;
 
-      public NonIncGroupByStream(ViewStream main_stream, INonIncGroupByEvaluator groupby_func) {
+      public NonIncGroupByStream(ViewStream main_stream, INonIncGroupByEvaluator groupby_func)
+      {
          _main_stream = main_stream;
          _groupby_func = groupby_func;
          _groupby_cols = _groupby_func.get_group_by_cols();
-         _row_hasher = new BlueRowHasher(_groupby_cols == null ? 0 : _groupby_cols.Length);
+         _row_hasher = new BlueRowHasher(_groupby_cols.Length);
       }
 
-      public NonIncGroupByStream(ViewStream main_stream, Func<string[]> groupby_cols_func,
-                                 Func<List<IBlueRow>, List<IBlueRow>> evaluate_func)
-                                 : this(main_stream, new DelNonIncGroupByEvaluator(groupby_cols_func, evaluate_func)) { }
-
-      public NonIncGroupByStream(ViewStream parent_str, string groupby_col, string standard_group_by_text, string optional_sort_col = null) {
-         GroupByBundle std_group_by = GroupByBundle.parse(standard_group_by_text);
-         Func<string[]> groupby_cols_func = () => {
-            return String.IsNullOrEmpty(groupby_col) ? null : new string[] { groupby_col };
-         };
-         Func<List<IBlueRow>, List<IBlueRow>> evaluate_func = (group_rows) => {
-            if (!String.IsNullOrEmpty(optional_sort_col))
-               group_rows.Sort((r1, r2) => BlueDataUtility.compare_rows(r2, r1, optional_sort_col));
-            IBlueRow rv = GroupByBundle.group_by(group_rows, std_group_by);
-            return new IBlueRow[] { rv }.ToList();
-         };
-         ViewStream rs = new NonIncGroupByStream(parent_str, groupby_cols_func, evaluate_func);
-         mySetRealSource(rs);
-      }
-      public NonIncGroupByStream(ViewStream parent_str, string standard_group_by_text) :
-      this(parent_str, String.Empty, standard_group_by_text) {
-      }
-
-
       protected override void myOnFirstSubscriber() { _main_stream.onChange += _main_stream_onchange; }
 
       protected override void myOnLastSubscriber() { _main_stream.onChange -= _main_stream_onchange; }
 
-      private void _main_stream_onchange(object nv) {
+      private void _main_stream_onchange(object nv)
+      {
          DataPacket dp = nv as DataPacket;
          ///grouping together
          _timer.Restart();
-         IDictionary<int, List<IBlueRow>> grouped_rows = (_groupby_cols == null || _groupby_cols.Length == 0) ?
-         new Dictionary<int, List<IBlueRow>>() { { 0, dp.newRows.to_list() } } :
-         BlueDataUtility.group_together(dp.newRows, _groupby_cols, ref _row_hasher);
+         IDictionary<int, List<IBlueRow>> grouped_rows = BlueDataUtility.group_together (dp.newRows, _groupby_cols, ref _row_hasher);
          _timer.Stop();
          var groupElapsedMs = _timer.ElapsedMilliseconds;
          if (groupElapsedMs >= 50) {
@@ -94,15 +44,14 @@ namespace BCG {
          IBlueRowCollection total_state = _groupby_func.evaluate(grouped_rows);
          _timer.Stop();
          var applyElapsedMs = _timer.ElapsedMilliseconds;
-         _n_total_fires++;
-         if (applyElapsedMs >= 50) {
+         _n_total_fires++;         
+         if (applyElapsedMs >= 50)
+         {
             _n_expensive_fires++;
             Log.DataFlow.info(
                $"Time to evaluate {grouped_rows.Count} groups {_timer.ElapsedMilliseconds}ms, ratio {_n_expensive_fires}/{_n_total_fires}");
-         }
-         // TODO,
-         if (total_state != null)
-            myFireEvent(new DataPacket(total_state, total_state, myLastValue == null ? new BlueTable(total_state.schema) : (myLastValue as DataPacket).newRows));
+         }         
+         myFireEvent(new DataPacket(total_state, total_state, myLastValue == null ? new BlueTable(total_state.schema) : (myLastValue as DataPacket).newRows));             
       }
    }
 }

src/DataStreamSuite/DataStream/Primitives/Join/NonIncremental/NonIncJoinBookkeeper.cs

@@ -1,16 +1,15 @@
 using bts;
-using bts.collections;
 
 using mbts;
 
 using System;
 using System.Collections.Generic;
 using System.Diagnostics;
-using System.Linq;
-using System.Security.Cryptography;
 
-namespace BCG {
-   public class NonIncJoinBookkeeper : IDisposable {
+namespace BCG
+{
+   public class NonIncJoinBookkeeper : IDisposable
+   {
       public Action on_new_data;
       private readonly NonIncJoinSpec _join_spec;
       private readonly int _join_position;
@@ -27,71 +26,41 @@ namespace BCG {
 
       private int[] _join_col_indices = null,
                      _data_col_indices = null;
-
-      public NonIncJoinBookkeeper(NonIncJoinSpec the_join_spec, int join_position, int start_index_in_output, BlueRowHasher row_hasher) {
+      
+      public NonIncJoinBookkeeper(NonIncJoinSpec the_join_spec, int join_position, int start_index_in_output, BlueRowHasher row_hasher)
+      {
          _join_spec = the_join_spec;
          _join_position = join_position;
          _start_index_in_output = start_index_in_output;
          _row_hasher = row_hasher;
          _is_left_most = _join_position == 0;
          _is_tag_along = _join_spec.join_col_names.Length == 0;
-      }
-
-      private bool _filled_missing_data = false;
-      private void _fill_missing_data(DataPacket dp) {
-         if (_filled_missing_data)
-            return;
-         Func<string[], bool> null_or_empty = (str_arr) => {
-            if (str_arr == null || str_arr.Length == 0)
-               return true;
-            foreach (string s in str_arr) {
-               if (!String.IsNullOrEmpty(s)) {
-                  return false;
-               }
-            }
-            return true;
-         };
-         Func<string[], string[]> remove_bad_cols = (col_names) => {
-            HashSet<string> schema_col_map = new HashSet<string>(dp.schema.columnNames);
-            List<string> cols_list = new List<string>(col_names);
-            cols_list.RemoveWhere(c => !schema_col_map.Contains(c));
-            return cols_list.ToArray();
-         };
-         if (null_or_empty(_join_spec.data_col_names)) {
-            _join_spec.data_col_names = dp.schema.columnNames;
-         }
-         _join_spec.data_col_names = remove_bad_cols(_join_spec.data_col_names);
-         if (null_or_empty(_join_spec.data_col_names)) {
-            _join_spec.data_col_names = dp.schema.columnNames;
-         }
-         if (null_or_empty(_join_spec.projected_col_names)) {
-            _join_spec.projected_col_names = _join_spec.data_col_names;
-         }
-      }
-
-      public void init() {
          _join_spec.stream.onChange += _on_stream_cb;
       }
 
-      public void _on_stream_cb(object dp_obj) {
-         Log.Gen.info($"NonIncJoinBookkeeper._on_stream_cb dp_obj: {dp_obj}");
+      public void _on_stream_cb(object dp_obj)
+      {
          var timer = Stopwatch.StartNew();
          DataPacket dp = (DataPacket)dp_obj;
-         _fill_missing_data(dp);
-         if (_schema == null) {
+         if (_schema == null)
+         {
             _schema = dp.schema;
             _join_col_indices = BlueDataUtility.get_column_indices(_schema, _join_spec.join_col_names);
             _data_col_indices = BlueDataUtility.get_column_indices(_schema, _join_spec.data_col_names);
          }
 
-         if (_is_left_most) {
+         if (_is_left_most)
+         {
             _row_list.Clear();
             foreach (IBlueRow r in dp.newRows)
-               _row_list.Add(new Pair<int, IBlueRow>(_row_hasher.get_hash(r, _join_col_indices), r));
-         } else {
+               _row_list.Add(new Pair<int, IBlueRow>(_row_hasher.get_hash(r, _join_col_indices), r));            
+         }
+         else
+         {
             if (_is_tag_along)
                _solitary_row = BlueDataUtility.extractRow(dp.newRows, 0);
-            else {
+            else
+            {
                _row_map.Clear();
                foreach (IBlueRow r in dp.newRows)
                   _row_map[_row_hasher.get_hash(r, _join_col_indices)] = r;
@@ -110,25 +79,31 @@ namespace BCG {
 
       public bool is_ready { get { return _schema != null; } }
 
-      public void constribute_cols_to_schema(List<ColumnInfo> cis) {
+      public void constribute_cols_to_schema(List<ColumnInfo> cis)
+      {
          for (int i = 0; i < _join_spec.data_col_names.Length; i++)
             cis.Add(new ColumnInfo(_join_spec.projected_col_names[i], cis.Count, _schema[_join_spec.data_col_names[i]].colType));
       }
 
       public List<Pair<int, IBlueRow>> get_list() { return _row_list; }
 
-      public void copy_master_row_to_output_row(int row_index, object[] output_row) {
+      public void copy_master_row_to_output_row(int row_index, object[] output_row)
+      {
          IBlueRow this_row = _row_list[row_index].second;
          BlueDataUtility.copy_to_row(this_row, _data_col_indices, output_row, _start_index_in_output);
       }
 
-      public void copy_matching_row_to_output_row(int row_hash, object[] output_row) {
-         if (_is_tag_along) {
+      public void copy_matching_row_to_output_row(int row_hash, object[] output_row)
+      {
+         if (_is_tag_along)
+         {
             if (_solitary_row != null)
                BlueDataUtility.copy_to_row(_solitary_row, _data_col_indices, output_row, _start_index_in_output);
             else
                copy_dbnull_to_output_row(output_row);
-         } else {
+         }
+         else
+         {
             IBlueRow this_row = null;
             if (_row_map.TryGetValue(row_hash, out this_row))
                BlueDataUtility.copy_to_row(this_row, _data_col_indices, output_row, _start_index_in_output);
@@ -137,7 +112,8 @@ namespace BCG {
          }
       }
 
-      private void copy_dbnull_to_output_row(object[] output_row) {
+      private void copy_dbnull_to_output_row(object[] output_row)
+      {
          for (int i = 0; i < _data_col_indices.Length; i++)
             output_row[i + _start_index_in_output] = DBNull.Value;
       }

src/DataStreamSuite/DataStream/Primitives/Join/NonIncremental/NonIncJoinSpec.cs

@@ -3,63 +3,63 @@ using System.Collections.Generic;
 using System.Linq;
 using System.Text;
 using System.Threading.Tasks;
-
 using mbts;
 
-namespace BCG {
-   // this a collection of data, theres no joining logic in here
-   public class NonIncJoinSpec {
+namespace BCG
+{
+   public class NonIncJoinSpec
+   {
       public ViewStream stream;
-      // join col names are the names on which the streams are joined (for equality),
-      // data col names are the names in the original stream
-      // projected col names are their new names, ie if its x:l_x, then x is in the 
-      // data_col_names and l_x in projected cols
       public string[] join_col_names, data_col_names, projected_col_names;
 
       public NonIncJoinSpec() { }
 
       public NonIncJoinSpec(ViewStream the_stream, string[] the_join_col_names, string[] the_data_col_names)
-         : this(the_stream, the_join_col_names, the_data_col_names, the_data_col_names) { }
+         : this (the_stream, the_join_col_names, the_data_col_names, the_data_col_names)
+      { }
 
-      public NonIncJoinSpec(ViewStream the_stream, string[] the_join_col_names, string[] the_data_col_names, string[] the_projected_col_names) {
+      public NonIncJoinSpec(ViewStream the_stream, string[] the_join_col_names, string[] the_data_col_names, string[] the_projected_col_names)
+      {
          stream = the_stream;
          join_col_names = the_join_col_names;
          data_col_names = the_data_col_names;
          projected_col_names = the_projected_col_names;
       }
 
-      public static NonIncJoinSpec[] from_triplet(params Triplet<ViewStream, string, string>[] join_specs) {
+      public static NonIncJoinSpec[] from_triplet(params Triplet<ViewStream, string, string>[] join_specs)
+      {
          List<NonIncJoinSpec> rv = new List<NonIncJoinSpec>();
-         foreach (Triplet<ViewStream, string, string> r in join_specs) {
-            string[] keys = String.IsNullOrEmpty(r.second) ? new string[] { } : r.second.Split(' ');
+         foreach (Triplet<ViewStream, string, string> r in join_specs)
+         {
+            string[] keys = String.IsNullOrEmpty (r.second) ? new string[] {} : r.second.Split(' ');
             string[] visible_cols = r.third.Split(' ');
             List<string> data_col_names = new List<string>(), mapped_col_names = new List<string>();
-            foreach (string s in visible_cols) {
+            foreach (string s in visible_cols)
+            {
                string[] toks = s.Split(':');
-               if (toks.Length == 1) {
+               if (toks.Length == 1)
+               {
                   data_col_names.Add(toks[0]);
                   mapped_col_names.Add(toks[0]);
-               } else if (toks.Length == 2) {
+               }
+               else if (toks.Length == 2)
+               {
                   data_col_names.Add(toks[0]);
                   mapped_col_names.Add(toks[1]);
                }
             }
-            rv.Add(new NonIncJoinSpec(r.first, keys, data_col_names.ToArray(), mapped_col_names.ToArray()));
+            rv.Add (new NonIncJoinSpec (r.first, keys, data_col_names.ToArray (), mapped_col_names.ToArray()));
          }
-         return rv.ToArray();
-      }
-
-      public override string ToString() {
-         Func<string[], string> arr_txt = (a) => {
-            string rv = "";
-            foreach (string s in a) {
-               rv += "{s} ";
-            }
-            rv = rv.Trim();
-            return rv;
-         };
-         string rv = $"stream: {stream} join_col_names: {arr_txt(join_col_names)} data_col_names: {arr_txt(data_col_names)} projected_col_names: {arr_txt(projected_col_names)}";
-         return rv;
+         return rv.ToArray ();
       }
    }
+
+   //[DataStreamUserRespresentation("ninctest")]
+   //public class NonIncJoinTestStream : ViewStream
+   //{
+   //   public NonIncJoinTestStream (string key_cols, string data_cols)
+   //   {
+   //      NonIncJoinSpec.from_triplet(new Triplet<ViewStream, string, string>(null, key_cols, data_cols));
+   //   }
+   //}
 }

src/DataStreamSuite/DataStream/Primitives/Join/NonIncremental/NonIncJoinStream.cs

@@ -6,9 +6,11 @@ using System;
 using System.Collections.Generic;
 using System.Diagnostics;
 
-namespace BCG {
+namespace BCG
+{
    [DataStreamUserRespresentation("fastJoin")]
-   public class NonIncJoinStream : ViewStream, BCG.Utilities.General.IWorker {
+   public class NonIncJoinStream : ViewStream, BCG.Utilities.General.IWorker
+   {
       private readonly NonIncJoinSpec[] _join_specs;
       private readonly INonIncJoinInlineMapper _output_mapper;
       private readonly BlueRowHasher _row_hasher;
@@ -26,19 +28,24 @@ namespace BCG {
                {
                   new Triplet<ViewStream, string, string> (left, left_join_cols, left_proj),
                   new Triplet<ViewStream, string, string> (right, right_join_cols, right_proj)
-               }) { }
+               })
+      { }
 
       public NonIncJoinStream(params Triplet<ViewStream, string, string>[] join_specs)
-         : this(NonIncJoinSpec.from_triplet(join_specs), null, 0) { }
+         : this(NonIncJoinSpec.from_triplet(join_specs), null, 0)
+      { }
 
       public NonIncJoinStream(ViewStreamDatabase vsd, INonIncJoinInlineMapper output_mapper, int throttles_ms,
                               params Triplet<ViewStream, string, string>[] join_specs)
-         : this(vsd, NonIncJoinSpec.from_triplet(join_specs), output_mapper, throttles_ms) { }
+         : this(vsd, NonIncJoinSpec.from_triplet(join_specs), output_mapper, throttles_ms)
+      { }
 
       public NonIncJoinStream(NonIncJoinSpec[] join_specs, INonIncJoinInlineMapper output_mapper, int throttles_ms = 0)
-         : this(ViewStreamDatabase.instance, join_specs, output_mapper, throttles_ms) { }
+         : this(ViewStreamDatabase.instance, join_specs, output_mapper, throttles_ms)
+      { }
 
-      public NonIncJoinStream(ViewStreamDatabase vsd, NonIncJoinSpec[] join_specs, INonIncJoinInlineMapper output_mapper, int throttles_ms = 0) {
+      public NonIncJoinStream(ViewStreamDatabase vsd, NonIncJoinSpec[] join_specs, INonIncJoinInlineMapper output_mapper, int throttles_ms = 0)
+      {
          _join_specs = join_specs;
          _output_mapper = output_mapper;
          _row_hasher = new BlueRowHasher(_join_specs[0].join_col_names.Length);
@@ -46,11 +53,12 @@ namespace BCG {
          _vsd = vsd;
       }
 
-      protected override void myOnFirstSubscriber() {
+      protected override void myOnFirstSubscriber()
+      {
          int data_col_no = 1;
-         for (int jp = 0; jp < _join_specs.Length; jp++) {
+         for (int jp = 0; jp < _join_specs.Length; jp++)
+         {
             NonIncJoinBookkeeper s = new NonIncJoinBookkeeper(_join_specs[jp], jp, data_col_no, _row_hasher);
-            Log.Gen.info($"NonIncJoinStream.myOnFirstSubscriber jp: {jp} spec: {_join_specs[jp]}");
             data_col_no += _join_specs[jp].data_col_names.Length;
             s.on_new_data += _stream_callback;
             _book_keeping_data.Add(s);
@@ -58,11 +66,10 @@ namespace BCG {
          if (_output_mapper != null)
             _output_mapper.onMapperChange += _on_mapper_change;
          _vsd.addWorker(this);
-         foreach (var bkd in _book_keeping_data)
-            bkd.init();
       }
 
-      protected override void myOnLastSubscriber() {
+      protected override void myOnLastSubscriber()
+      {
          foreach (NonIncJoinBookkeeper d in _book_keeping_data)
             d.Dispose();
          if (_output_mapper != null)
@@ -70,26 +77,30 @@ namespace BCG {
          _vsd.removeWorker(this);
       }
 
-      private void _stream_callback() {
-         Log.Gen.info("NonIncJoinStream._stream_callback received callback");
+      private void _stream_callback()
+      {
          _is_dirty = true;
          doWork();
       }
 
-      private void _on_mapper_change() {
+      private void _on_mapper_change()
+      {
          _is_dirty = true;
          doWork();
       }
 
-      private bool _is_ready() {
+      private bool _is_ready()
+      {
          bool rv = true;
          foreach (NonIncJoinBookkeeper d in _book_keeping_data)
             rv &= d.is_ready;
          return rv;
       }
 
-      private void _construct_schema() {
-         if (_output_schema == null) {
+      private void _construct_schema()
+      {
+         if (_output_schema == null)
+         {
             List<ColumnInfo> cis = new List<ColumnInfo>(new ColumnInfo[] { new ColumnInfo("id", 0, typeof(int)) });
             foreach (NonIncJoinBookkeeper d in _book_keeping_data)
                d.constribute_cols_to_schema(cis);
@@ -98,23 +109,27 @@ namespace BCG {
       }
 
       private int _n_total_fires = 0, _n_expensive_fires = 0;
-      private void _form_and_fire() {
-         Log.Gen.info($"NonIncJoinStream._form_and_fire _is_ready(): {_is_ready()}");
-         if (_is_ready()) {
+      private void _form_and_fire()
+      {
+         if (_is_ready())
+         {
             _construct_schema();
             var timer = Stopwatch.StartNew();
             IList<Pair<int, IBlueRow>> left_row_list = _book_keeping_data[0].get_list();
             List<object[]> output_rows = new List<object[]>();
 
-            for (int i = 0; i < left_row_list.Count; i++) {
+            for (int i = 0; i < left_row_list.Count; i++)
+            {
                object[] or = new object[_output_schema.columnInfo.Length];
                or[0] = i;
                output_rows.Add(or);
             }
 
-            for (int ji = 0; ji < _book_keeping_data.Count; ji++) {
+            for (int ji = 0; ji < _book_keeping_data.Count; ji++)
+            {
                NonIncJoinBookkeeper bk_data = _book_keeping_data[ji];
-               for (int i = 0; i < left_row_list.Count; i++) {
+               for (int i = 0; i < left_row_list.Count; i++)
+               {
                   if (ji == 0)
                      bk_data.copy_master_row_to_output_row(i, output_rows[i]);
                   else
@@ -124,20 +139,19 @@ namespace BCG {
             IBlueRowCollection dp_output_rows = _apply_map(new SequentialIDBlueTable(_output_schema, output_rows));
             timer.Stop();
             _n_total_fires++;
-            //if (timer.ElapsedMilliseconds >= 250) {
-            _n_expensive_fires++;
-            string log_msg = $"Join and map time for {_book_keeping_data.Count} "
-               + $"tables {timer.Elapsed.TotalMilliseconds} msecs, ratio {_n_expensive_fires}/{_n_total_fires}";
-            Log.Gen.info(log_msg);
-            //}
-            var dp_out = BlueDataUtility.build_dp_inc(dp_output_rows, myLastValue as DataPacket);
-            myFireEvent(dp_out);
-            //myFireEvent(new DataPacket(dp_output_rows, dp_output_rows, myLastValue == null ? new BlueTable(_output_schema) : (myLastValue as DataPacket).newRows));
+            if (timer.ElapsedMilliseconds >= 250) {
+               _n_expensive_fires++;
+               string log_msg = $"Join and map time for {_book_keeping_data.Count} "
+                  + $"tables {timer.Elapsed} ratio {_n_expensive_fires}/{_n_total_fires}";
+            }
+            myFireEvent(new DataPacket(dp_output_rows, dp_output_rows, myLastValue == null ? new BlueTable(_output_schema) : (myLastValue as DataPacket).newRows));
          }
       }
 
-      private IBlueRowCollection _apply_map(IBlueRowCollection unmapped_rows) {
-         if (_output_mapper != null) {
+      private IBlueRowCollection _apply_map(IBlueRowCollection unmapped_rows)
+      {
+         if (_output_mapper != null)
+         {
             IBlueRowCollection output_rows;
             var timer = Stopwatch.StartNew();
             output_rows = _output_mapper.map_rows(unmapped_rows);
@@ -146,14 +160,17 @@ namespace BCG {
                Log.Gen.info($"Time to map {unmapped_rows.Count} rows into {output_rows.Count} rows {timer.Elapsed}");
             }
             return output_rows;
-         } else {
+         }
+         else
+         {
             return unmapped_rows;
          }
       }
 
-      public void doWork() {
-         Log.Gen.info($"NonIncJoinStream.doWork _last_fire_time: {_last_fire_time} _is_dirty: {_is_dirty}");
-         if (DateTime.Now - _last_fire_time >= _throttle_period && _is_dirty) {
+      public void doWork()
+      {
+         if (DateTime.Now - _last_fire_time >= _throttle_period && _is_dirty)
+         {
             _is_dirty = false;
             _last_fire_time = DateTime.Now;
             _form_and_fire();

src/DataStreamSuite/DataStream/Primitives/Join/ParallelJoinTest.cs

@@ -1,9 +1,7 @@
-using System;
+#nullable disable
+using System;
 using System.Collections;
 using System.Collections.Generic;
-using System.Text;
-using BCG.Utilities.General;
-using BCG.Utilities;
 
 namespace BCG
 {

src/DataStreamSuite/DataStream/Primitives/Join/Proximity/ProximityJoinStream.cs

@@ -1,68 +0,0 @@
-namespace bts {
-   using BCG;
-   using BCG.Utilities.General;
-
-   using System;
-   using System.Diagnostics;
-
-   using ZstdSharp.Unsafe;
-
-   // functon that joins the 2 rows and returns the outputput row. The function
-   // can return a null which means the 2 rows dont need to be joined
-   using JoinFunc = System.Func<BCG.IBlueRow, BCG.IBlueRow, BCG.IBlueRow>;
-   public class ProximityJoinStream : ViewStream, IWorker {
-      private ViewStream _lvs, _rvs;
-      private JoinFunc _join_func;
-      private bool _dirty = false;
-      private Stopwatch _stopwatch = new Stopwatch();
-      private ViewStreamDatabase _vsd = ViewStreamDatabase.instance;
-      public ProximityJoinStream(ViewStream lvs, ViewStream rvs, JoinFunc join_func) {
-         _lvs = lvs;
-         _rvs = rvs;
-         _join_func = join_func;
-      }
-
-      protected override void myOnFirstSubscriber() {
-         _lvs.onChange += _handle_data;
-         _rvs.onChange += _handle_data;
-         _vsd.addWorker(this);
-      }
-
-      protected override void myOnLastSubscriber() {
-         _lvs.onChange -= _handle_data;
-         _rvs.onChange -= _handle_data;
-         _vsd.removeWorker(this);
-      }
-
-      private void _handle_data(object dp_obj) {
-         _dirty = true;
-         doWork();
-      }
-
-      public void doWork() {
-         Func<bool> all_data_revcd = () => {
-            return _lvs.rows != null && _rvs.rows != null;
-         };
-         _stopwatch.Start();
-         if (_dirty && all_data_revcd()) {
-            _dirty = false;
-            BlueTable rv = null;
-            foreach (IBlueRow lr in _lvs.rows) {
-               foreach (IBlueRow rr in _rvs.rows) {
-                  IBlueRow resr = _join_func(lr, rr);
-                  if (resr == null)
-                     continue;
-                  rv = rv ?? new BlueTable(resr.schema);
-                  rv.addRow(resr);
-               }
-            }
-            _stopwatch.Stop();
-            Log.Gen.info($"ProximityJoinStream left_count: {_lvs.rows.Count} right_count: {_rvs.rows.Count} result_count: {rv.Count} time: {_stopwatch.ElapsedMilliseconds} msecs");
-            var dp = BlueDataUtility.build_dp_inc(rv, myLastValue as DataPacket);
-            myFireEvent(dp);
-         }
-
-      }
-
-   }
-}

src/DataStreamSuite/DataStream/Primitives/Join/Proximity/TimeProximityJoinStream.cs

@@ -1,47 +0,0 @@
-namespace bts.SilverPine {
-   using BCG;
-
-   using System;
-   using System.CodeDom;
-   using System.Configuration;
-   using System.Reactive.Disposables;
-
-   public class TimeProximityJoinStream : ViewStream {
-      private int _id = 0;
-
-      public TimeProximityJoinStream(ViewStream ls, string l_time_col, ViewStream rs, string r_time_col,
-                                       string pre_join_period, string post_join_period) :
-            this(ls, l_time_col, rs, r_time_col, TimeBasedFilter.parse_duration(pre_join_period),
-            TimeBasedFilter.parse_duration(post_join_period)) { }
-
-
-      public TimeProximityJoinStream(ViewStream ls, string l_time_col, ViewStream rs, string r_time_col,
-                                       TimeSpan pre_join_period, TimeSpan post_join_period) {
-         Func<IBlueRow, IBlueRow, IBlueRow> create_row = (lr, rr) => {
-            var id = 0;
-            lock (this) {
-               id = _id++;
-            }
-            IBlueRow rv = BlueDataUtility.create_composite_row(lr, rr, id);
-            return rv;
-         };
-         Func<IBlueRow, IBlueRow, IBlueRow> join_func = (lr, rr) => {
-            DateTime ldt = lr.get_datetime(l_time_col);
-            DateTime rdt = rr.get_datetime(r_time_col);
-            TimeSpan diff = ldt - rdt;
-            if (ldt < rdt) {
-               if (diff.Duration() > pre_join_period) {
-                  return null;
-               }
-            } else {
-               if (diff > post_join_period)
-                  return null;
-            }
-            return create_row(lr, rr);
-         };
-
-         ViewStream rstr = new ProximityJoinStream(ls, rs, join_func);
-         mySetRealSource(rstr);
-      }
-   }
-}

src/DataStreamSuite/DataStream/Primitives/Literal/LiteralStream.cs

@@ -2,13 +2,9 @@ using System;
 using System.Collections;
 using System.Collections.Generic;
 
-using Amazon.Runtime.Internal.Util;
-
 using BCG;
 using BCG.Utilities;
 
-using bts;
-
 namespace BCG {
    public class LiteralStream : ViewStream {
 
@@ -26,34 +22,6 @@ namespace BCG {
          return rv;
       }
 
-      //literal("table1", "x:123, y:1.001, z:abc;x:123, y:1.001, z:abc")
-      public LiteralStream(string table_name, string rows) {
-         Func<string, int, IBlueRow> get_row = (json_row, id) => {
-            Dictionary<string, object> map = new Dictionary<string, object>();
-            string[] toks = json_row.Split(',', StringSplitOptions.TrimEntries);
-            //Logger.assert(col_vals.Length == 2);
-            foreach (string t in toks) {
-               var col_vals = t.Split(':', StringSplitOptions.TrimEntries);
-               string col = col_vals[0];
-               string val = col_vals[1];
-               map.Add(col, val);
-            }
-            map["id"] = id;
-            return new BlueRow(map);
-         };
-         string[] row_arr = rows.Split(';', StringSplitOptions.TrimEntries);
-         BlueTable table = null;
-         foreach (string json_string in row_arr) {
-            IBlueRow r = //BlueDataUtility.convert_json_to_row(json_string, table == null ? 0 : table.Count);
-            get_row(json_string, table == null ? 0 : table.Count);
-            table = table ?? new BlueTable(r.schema);
-            table.addRow(r);
-         }
-         ViewStream str = new BlueTableStream(table);
-         str = new SchemaGuesserStream(str);
-         mySetRealSource(str);
-      }
-
       public LiteralStream(string tableName, string schema, string value) {
          string[] columns = schema.Split(' ');
 
@@ -87,7 +55,7 @@ namespace BCG {
          mySetRealSource(new BlueRowCollectionStaticStream(t));
       }
 
-      // public LiteralStream(string schema, string value) : this("literal", schema, value) { }
+      public LiteralStream(string schema, string value) : this("literal", schema, value) { }
 
       // public LiteralStream(string value) {
 

src/DataStreamSuite/DataStream/Primitives/Map/GlobalMapStream.cs

@@ -1,72 +0,0 @@
-using BCG;
-
-using MongoDB.Driver.Core.WireProtocol.Messages;
-
-using System;
-using System.Collections.Generic;
-using System.Linq;
-
-// the first arg is all the rows, the second arg is the index of the row that needs mapping, and 
-// the func returns the mapped row (ie mapped version of the index row)
-using GlobalMap = System.Func<System.Collections.Generic.List<BCG.IBlueRow>, uint, BCG.IBlueRow>;
-
-namespace bts {
-   // In GlobalMap, each row is mapped with the full view of all rows, ie the map functions for a row
-   // can use the state fo all rows.
-   public class GlobalMapStream : ViewStream {
-      private ViewStream _base_stream;
-      private readonly string _sort_col_name, _group_by_col;
-      private GlobalMap _map_func;
-      public GlobalMapStream(ViewStream base_stream, string sort_col_name, string group_by_col, GlobalMap map_func) {
-         _base_stream = base_stream;
-         _sort_col_name = sort_col_name;
-         _group_by_col = group_by_col;
-         _map_func = map_func;
-      }
-
-      public GlobalMapStream(ViewStream base_stream, string sort_col_name, GlobalMap map_func) :
-      this(base_stream, sort_col_name, null, map_func) {
-      }
-
-      protected override void myOnFirstSubscriber() {
-         _base_stream.onChange += _handle_stream_update;
-      }
-
-      protected override void myOnLastSubscriber() {
-         _base_stream.onChange -= _handle_stream_update;
-      }
-
-      void _handle_stream_update(object dp_obj) {
-         DataPacket dp = dp_obj as DataPacket;
-         IDictionary<int, List<IBlueRow>> groups = !String.IsNullOrEmpty(_group_by_col) ?
-         BlueDataUtility.group_together(dp.newRows, new string[] { _group_by_col })
-         :
-         new Dictionary<int, List<IBlueRow>>() { { 0, dp.newRows.to_list() } };
-         BlueTable out_table = null;
-         foreach (var kv in groups) {
-            List<IBlueRow> group_rows = kv.Value;
-            group_rows.Sort((r1, r2) => BlueDataUtility.compare_rows(r2, r1, _sort_col_name));
-            uint i = 0;
-            foreach (IBlueRow r in group_rows) {
-               IBlueRow nr = _map_func(group_rows, i++);
-               out_table = out_table ?? new BlueTable(nr.schema);
-               out_table.addRow(nr);
-            }
-         }
-
-
-         // List<IBlueRow> rows = dp.newRows.to_ienum().ToList();
-         // rows.Sort((r1, r2) => BlueDataUtility.compare_rows(r1, r2, _sort_col_name));
-         // uint i = 0;
-
-         // BlueTable out_table = null;
-         // foreach (IBlueRow r in rows) {
-         //    IBlueRow nr = _map_func(rows, i++);
-         //    out_table = out_table ?? new BlueTable(nr.schema);
-         //    out_table.addRow(nr);
-         // }
-         DataPacket dp_out = BlueDataUtility.build_dp_inc(out_table, myLastValue as DataPacket);
-         myFireEvent(dp_out);
-      }
-   }
-}

src/DataStreamSuite/DataStream/Primitives/Map/MapStream.cs

@@ -1,6 +1,10 @@
+using bts;
+
 using Bts.DataStream;
+using Bts.DataStreamCore;
 
 using mbts.config;
+using mbts.utils;
 
 using System;
 
@@ -16,8 +20,8 @@ namespace BCG {
    [DataStreamUserRespresentation (false)]
    public class MapStream : MethodDrivenStream
    {
-      protected IStreamMapper myMapper;
-      protected IChangeableStreamMapper myChangeableMapper = null;
+      protected IStreamMapper? myMapper;
+      protected IChangeableStreamMapper? myChangeableMapper;
       private bool myUseNewMap = Configuration.get<DataStreamConfig>().UseNewMap;
 
       public class FuncWrapper : IStreamMapper {
@@ -72,7 +76,7 @@ namespace BCG {
 
 
       protected override void myHandleFirstCall() {
-
+         ArgumentNullException.ThrowIfNull(myMapper);
          IBlueTableSchema outputSchema = myMapper.getOutputSchema(myLastArrivedPacket.newRows.schema);
          myLastFiredPacket = new DataPacket();
          myLastFiredPacket.newRows = new BlueTable(outputSchema);
@@ -83,28 +87,34 @@ namespace BCG {
       }
 
       protected override void myHandleIncrementalAdd() {
-         //t.start ();
-         myLastFiredPacket.addedRows = new BlueTable(myLastFiredPacket.newRows.schema);
-         foreach (IBlueRow row in myLastArrivedPacket.addedRows) {
-            IBlueRow newRow = myMapper.map(row);
-            (myLastFiredPacket.newRows as BlueTable).addRow(newRow);
-            (myLastFiredPacket.addedRows as BlueTable).addRow(newRow);
+         if (myLastArrivedPacket.addedRows.Count == 0) {
+            Log.Gen.warn("MapStream.myHandleIncrementalAdd called with 0 addedRows");
+            return;
          }
-         //t.stop ();
-         //System.Diagnostics.Trace.WriteLine ("adding " + myLastFiredPacket.addedRows.Count.ToString () + " : " + t.lastIntervalTime);
-         //System.Diagnostics.Trace.WriteLine (" ");
+         ArgumentNullException.ThrowIfNull(myMapper);
+         var addedRows = new BlueTable(myLastFiredPacket.newRows.schema);
+         var newRows = ExceptionAssert.Cast(myLastFiredPacket.newRows).To<BlueTable>();
+         foreach (IBlueRow row in myLastArrivedPacket.addedRows) {
+            var newRow = myMapper.map(row);
+            newRows.addRow(newRow);
+            addedRows.addRow(newRow);
+         }
+         myLastFiredPacket.addedRows = addedRows;
       }
 
       protected override void myHandleIncrementalDelete() {
-         //t.start ();
-         myLastFiredPacket.deletedRows = new BlueTable(myLastFiredPacket.newRows.schema);
-         foreach (IBlueRow row in myLastArrivedPacket.deletedRows) {
-            IBlueRow delRow = (myLastFiredPacket.newRows as BlueTable)[(int)row["id"]];
-            (myLastFiredPacket.deletedRows as BlueTable).addRow(delRow);
-            (myLastFiredPacket.newRows as BlueTable).deleteRow(row);
+         if (myLastArrivedPacket.deletedRows.Count == 0) {
+            Log.Gen.warn("MapStream.myHandleIncrementalDelete called with 0 deletedRows");
+            return;
          }
-         //t.stop ();
-         //System.Diagnostics.Trace.WriteLine ("deleting " + myLastFiredPacket.deletedRows.Count.ToString () + " : " + t.lastIntervalTime);
+         var deletedRows = new BlueTable(myLastFiredPacket.newRows.schema);
+         var newRows = ExceptionAssert.Cast(myLastFiredPacket.newRows).To<BlueTable>();
+         foreach (IBlueRow row in myLastArrivedPacket.deletedRows) {
+            var delRow = newRows[row.Id()];
+            deletedRows.addRow(delRow);
+            newRows.deleteRow(row);
+         }
+         myLastFiredPacket.deletedRows = deletedRows;
       }
 
 

src/DataStreamSuite/DataStream/Primitives/Map/SuperMapStream.cs

@@ -1,11 +1,9 @@
+using Bts.DataStreamCore;
+
+using mbts.utils;
+
 using System;
 using System.Collections.Generic;
-using System.Text;
-
-using BCG.Utilities;
-using BCG.Utilities.General;
-
-using mbts;
 
 namespace BCG {
    /// <summary>
@@ -56,15 +54,15 @@ namespace BCG {
             return rows == null ? new IBlueRow[] { } : new List<IBlueRow>(rows).ToArray();
          }
 
-         public event MapperChangeDelegate onMapperChange;
-         private void _fake() { onMapperChange(); }
+         public event MapperChangeDelegate? onMapperChange;
       }
 
       //maps the id of the underlying stream to the
       //rows of the resulting stream
       private Dictionary<int, List<IBlueRow>> myMap = new Dictionary<int, List<IBlueRow>>();
       private int myID = 0;
-      private BlueTable myRows, myBaseRows;
+      private BlueTable? myRows;
+      private BlueTable? myBaseRows;
       private ViewStream myStream;
       private ISuperStreamMapper myMapper;
       public bool generate_ids = true;
@@ -92,49 +90,40 @@ namespace BCG {
          myRows = null;
          myBaseRows = null;
          myMap.Clear();
-         //myMapper.onMapperChange -= new MapperChangeDelegate(myHandleMapChange);
       }
 
       private void myHandleData(object newValue) {
-         DataPacket idp = newValue as DataPacket;
-         //HighPerformanceTimer timer = new HighPerformanceTimer();
-         //timer.start();
-         DataPacket dp = newValue as DataPacket;
-         myRows = myRows ?? new BlueTable(myMapper.getOutputSchema(dp.schema));
-         myBaseRows = myBaseRows ?? new BlueTable(dp.schema);
+         var dp = ExceptionAssert.Cast(newValue).To<DataPacket>();
+         myRows ??= new BlueTable(myMapper.getOutputSchema(dp.schema));
+         myBaseRows ??= new BlueTable(dp.schema);
          BlueDataUtility.absorbChanges(myBaseRows, dp.deletedRows, dp.addedRows);
-         BlueTable dels = new BlueTable(myRows.schema),
-                     adds = new BlueTable(myRows.schema);
+         var dels = new BlueTable(myRows.schema);
+         var adds = new BlueTable(myRows.schema);
 
          foreach (IBlueRow r in dp.deletedRows) {
-            List<IBlueRow> mappedRows = myMap[(int)r["id"]];
+            var mappedRows = myMap[r.Id()];
             foreach (IBlueRow mr in mappedRows) {
                dels.addRow(mr);
                myRows.deleteRow(mr);
             }
-            myMap.Remove((int)r["id"]);
+            myMap.Remove(r.Id());
          }
          foreach (IBlueRow r in dp.addedRows) {
             IBlueRow[] mappedRows = myMapper.map(r);
             foreach (IBlueRow mr in mappedRows) {
                IBlueRow idChangedMapRow = generate_ids ? BlueDataUtility.getNewReplacedRow(mr, myID++, "id") : mr;
-               if (!myMap.ContainsKey((int)r["id"])) {
-                  myMap.Add((int)r["id"], new List<IBlueRow>());
-               }
-               myMap[(int)r["id"]].Add(idChangedMapRow);
+               myMap.GetOrAdd(r.Id(), _ => new()).Add(idChangedMapRow);
                myRows.addRow(idChangedMapRow);
                adds.addRow(idChangedMapRow);
             }
          }
-         //timer.stop();
-         // if (timer.lastIntervalTime > 20)
-         //    {}//mbts.Logger.log(bts.log.Level.Info, bts.log.Subsystem.gen, String.Format("SuperMapper:myHandlerData for mapper {0} took {1} msecs", myMapper.GetType().ToString(), timer.lastIntervalTime));
          myFireEvent(new DataPacket(myRows, adds, dels));
       }
 
       private void myHandleMapChange() {
-         if (myBaseRows != null)
+         if (myBaseRows is not null) {
             myHandleData(new DataPacket(myBaseRows, new BlueTable(myBaseRows), new BlueTable(myBaseRows)));
+         }
       }
    }
 }

src/DataStreamSuite/DataStream/Primitives/Misc/PubSubTestStream.cs

@@ -8,33 +8,26 @@ using System;
 using System.IO;
 using System.Linq;
 
-namespace bts
-{
-    public class PubSubTestingStream : ViewStream, IWorker
-    {
-        public class PayloadMessage
-        {
+namespace bts {
+    public class PubSubTestingStream : ViewStream, IWorker {
+        public class PayloadMessage {
             public int msg_id = 0;
             public string guid;
             public Int64 travel_epoch_ms = 0;
             public byte[] payload = null;
-            public PayloadMessage(int msg_id_args, Int64 send_epoch_args, int payload_size, byte payload_byte)
-            {
+            public PayloadMessage(int msg_id_args, Int64 send_epoch_args, int payload_size, byte payload_byte) {
                 msg_id = msg_id_args;
                 travel_epoch_ms = send_epoch_args;
                 payload = new byte[payload_size];
-                for (int i = 0; i < payload_size; i++)
-                {
+                for (int i = 0; i < payload_size; i++) {
                     payload[i] = payload_byte;
                 }
             }
             private PayloadMessage() { }
             public PayloadMessage(int msg_id, int payload_size, byte payload_byte) :
-            this(msg_id, PubSubTestingStream.epoch_now(), payload_size, payload_byte)
-            { }
+            this(msg_id, PubSubTestingStream.epoch_now(), payload_size, payload_byte) { }
 
-            public byte[] serialize()
-            {
+            public byte[] serialize() {
                 var str = new MemoryStream();
                 BinaryWriter bw = new BinaryWriter(str);
                 bw.Write(msg_id);
@@ -47,8 +40,7 @@ namespace bts
                 return bytes;
             }
 
-            public static PayloadMessage deserialize(byte[] ser_bytes)
-            {
+            public static PayloadMessage deserialize(byte[] ser_bytes) {
                 var str = new MemoryStream(ser_bytes);
                 BinaryReader br = new BinaryReader(str);
                 int msg_id = br.ReadInt32();
@@ -56,8 +48,7 @@ namespace bts
                 Int64 epoch = br.ReadInt64();
                 int len = br.ReadInt32();
                 var payload = br.ReadBytes(len);
-                return new PayloadMessage()
-                {
+                return new PayloadMessage() {
                     msg_id = msg_id,
                     guid = guid,
                     travel_epoch_ms = epoch,
@@ -82,8 +73,7 @@ namespace bts
         private readonly int _payload_sz;
         private const byte _payload_byte = (byte)'A';
         private readonly string _tag_name;
-        public PubSubTestingStream(int publish_period_msecs, int max_pubs, int payload_size, string tag_name, ViewStreamDatabase vsd)
-        {
+        public PubSubTestingStream(int publish_period_msecs, int max_pubs, int payload_size, string tag_name, ViewStreamDatabase vsd) {
             _vsd = vsd;
             _payload_sz = payload_size;
             var schema = BlueDataUtility.form_schema("PubSubTest",
@@ -106,53 +96,43 @@ namespace bts
         public PubSubTestingStream(int publish_period_seconds, int max_pubs, int msg_size, string tag_name)
         : this(publish_period_seconds, max_pubs, msg_size, tag_name, ViewStreamDatabase.instance) { }
 
-        protected override void myOnFirstSubscriber()
-        {
+        protected override void myOnFirstSubscriber() {
             _vsd.addWorker(this);
             System.Threading.Thread th = new System.Threading.Thread(_subscribe_thread_func);
             th.IsBackground = true;
             th.Start();
         }
 
-        protected override void myOnLastSubscriber()
-        {
+        protected override void myOnLastSubscriber() {
             _vsd.removeWorker(this);
             _continue_sub_thread = false;
         }
 
-        void _log_updates(IEnumerable<PubSubServerMessage> updates)
-        {
+        void _log_updates(IEnumerable<PubSubServerMessage> updates) {
             var idx = 0;
-            foreach (var msg in updates)
-            {
+            foreach (var msg in updates) {
                 bts.Log.Gen.info($"PubSubTesStream._subscribe_thread_func idx: {idx} msg: {msg}");
                 idx++;
             }
         }
 
-        private void _subscribe_thread_func()
-        {
+        private void _subscribe_thread_func() {
             // subscription is done in another thread
             SubscriberCore sub_core = new SubscriberCore();
             sub_core.subscribe(_tag_name);
-            while (_continue_sub_thread)
-            {
+            while (_continue_sub_thread) {
                 // TODO recv_id
                 var updates = sub_core.get_updates(TimeSpan.FromMilliseconds(250));
                 if (updates == null || updates.Count() == 0)
                     continue;
                 _log_updates(updates);
                 _n_updates += updates.Count();
-                if (updates.Count() > 0)
-                {
+                if (updates.Count() > 0) {
                     var server_msg = updates.Last();
-                    if (server_msg.data != null && server_msg.data.Length > 0)
-                    {
+                    if (server_msg.data != null && server_msg.data.Length > 0) {
                         _last_recvd_msg = PayloadMessage.deserialize(server_msg.data);
                         _last_recvd_msg.travel_epoch_ms = epoch_now();
-                    }
-                    else
-                    {
+                    } else {
                         // why are we getting empty or null data ? (server is sending data regardless?)
                     }
                     _fire_dirty = true;
@@ -160,36 +140,29 @@ namespace bts
             }
         }
 
-        private void _publish_next_packet()
-        {
-            if (_last_sent_msg == null)
-            {
-                _last_sent_msg = new PayloadMessage(0, _payload_sz, _payload_byte)
-                {
+        private void _publish_next_packet() {
+            if (_last_sent_msg == null) {
+                _last_sent_msg = new PayloadMessage(0, _payload_sz, _payload_byte) {
                     guid = Guid.NewGuid().ToString()
                 };
-            }
-            else
-            {
+            } else {
                 _last_sent_msg.msg_id++;
                 _last_sent_msg.guid = Guid.NewGuid().ToString();
                 _last_sent_msg.travel_epoch_ms = epoch_now();
             }
             var ser_bytes = _last_sent_msg.serialize();
             _n_pubs++;
-            _pub_core.publish("Blob", _tag_name, ser_bytes);
+            _pub_core.publish(String.Empty, _tag_name, ser_bytes);
             _fire_dirty = true;
         }
 
-        private void _fire_dp()
-        {
+        private void _fire_dp() {
             object last_sent_id = _last_sent_msg == null ? (object)DBNull.Value : (object)_last_sent_msg.msg_id;
             object last_recv_id = _last_recvd_msg == null ? (object)DBNull.Value : (object)_last_recvd_msg.msg_id;
             object echo_time = DBNull.Value;
             object last_sent_guid = _last_sent_msg == null ? (object)DBNull.Value : (object)_last_sent_msg.guid;
             object last_recv_guid = _last_recvd_msg == null ? (object)DBNull.Value : (object)_last_recvd_msg.guid;
-            if (_last_sent_msg != null && _last_recvd_msg != null)
-            {
+            if (_last_sent_msg != null && _last_recvd_msg != null) {
                 echo_time = _last_recvd_msg.travel_epoch_ms - _last_sent_msg.travel_epoch_ms;
             }
             var match = last_sent_id.Equals(last_recv_id) && last_sent_guid.Equals(last_recv_guid);
@@ -201,25 +174,21 @@ namespace bts
             myFireEvent(dp);
         }
 
-        public void doWork()
-        {
+        public void doWork() {
             if (DateTime.Now - _last_publish_time > _publish_period)
-                if (_n_pubs <= _max_pubs)
-                {
+                if (_n_pubs <= _max_pubs) {
                     _publish_next_packet();
                     _last_publish_time = DateTime.Now;
                 }
             // publish 
             // TODO: send id
-            if (_fire_dirty)
-            {
+            if (_fire_dirty) {
                 _fire_dirty = false;
                 _fire_dp();
             }
         }
 
-        public static long epoch_now()
-        {
+        public static long epoch_now() {
             return DateTimeOffset.UtcNow.ToUnixTimeMilliseconds();
         }
 

src/DataStreamSuite/DataStream/Primitives/Misc/PubsubEchoTestStream.cs

@@ -8,23 +8,19 @@ using System.IO;
 using System.Linq;
 using System.Runtime.CompilerServices;
 using bts.api;
+using Bts.PubSub.Routing;
 
-namespace bts
-{
+namespace bts {
    [DataStreamUserRespresentation("pss_echo")]
-   public class PubsubEchoTestStream : ViewStream, IWorker
-   {
-      public class PayloadMessage
-      {
+   public class PubsubEchoTestStream : ViewStream, IWorker {
+      public class PayloadMessage {
          public readonly string msg_id;
          public readonly Int64 timestamp = 0;
          public readonly byte[] payload = null;
 
-         public static byte[] create_data(int payload_size, byte payload_byte)
-         {
+         public static byte[] create_data(int payload_size, byte payload_byte) {
             var rv = new byte[payload_size];
-            for (int i = 0; i < payload_size; i++)
-            {
+            for (int i = 0; i < payload_size; i++) {
                rv[i] = payload_byte;
             }
             return rv;
@@ -40,16 +36,14 @@ namespace bts
          //       payload[i] = payload_byte;
          //    }
          // }
-         public PayloadMessage(string guid_, Int64 timestamp_, byte[] payload_)
-         {
+         public PayloadMessage(string guid_, Int64 timestamp_, byte[] payload_) {
             msg_id = guid_;
             timestamp = timestamp_;
             payload = payload_;
          }
 
          private PayloadMessage() { }
-         public byte[] serialize()
-         {
+         public byte[] serialize() {
             var str = new MemoryStream();
             BinaryWriter bw = new BinaryWriter(str);
             bw.Write(msg_id);
@@ -61,8 +55,7 @@ namespace bts
             return bytes;
          }
 
-         public static PayloadMessage deserialize(byte[] ser_bytes)
-         {
+         public static PayloadMessage deserialize(byte[] ser_bytes) {
             var str = new MemoryStream(ser_bytes);
             BinaryReader br = new BinaryReader(str);
             string guid = br.ReadString();
@@ -88,17 +81,17 @@ namespace bts
       private readonly string _tag_name;
       private readonly int _payload_sz;
       private readonly IPubSubRouter _router;
-      private enum IOState
-      {
+      private enum IOState {
          SEND = 0,
          RECV = 1
       };
       IOState _state = IOState.SEND;
 
-      public PubsubEchoTestStream(string server_url, TimeSpan publish_period, int payload_size, string tag_name, ViewStreamDatabase vsd)
-      {
-         var route = new PubSubRoute(server_url, ".*");
-         _router = new PubSubRouter(route);
+      public PubsubEchoTestStream(string server_url, TimeSpan publish_period, int payload_size, string tag_name, ViewStreamDatabase vsd) {
+         _router = new PubSubRouter(new PubSubRouterConfig() {
+            PublisherRoutes = new() { new(server_url, ".*") },
+            SubscriberRoutes = new() { new(server_url, ".*") },
+         });
          _vsd = vsd;
          _payload_sz = payload_size;
          var schema = BlueDataUtility.form_schema("PubSubTest",
@@ -117,8 +110,7 @@ namespace bts
       public PubsubEchoTestStream(string server_url, int publish_period_msecs, int payload_size, string tag_name)
       : this(server_url, TimeSpan.FromMilliseconds(publish_period_msecs), payload_size, tag_name, ViewStreamDatabase.instance) { }
 
-      protected override void myOnFirstSubscriber()
-      {
+      protected override void myOnFirstSubscriber() {
          _pub_core = new PublisherCore(_router);
          _sub_core = new SubscriberCore(_router);
          _sub_core.subscribe(_tag_name);
@@ -126,14 +118,12 @@ namespace bts
 
       }
 
-      protected override void myOnLastSubscriber()
-      {
+      protected override void myOnLastSubscriber() {
          _vsd.removeWorker(this);
          _sub_core.unsubscribe(_tag_name);
       }
 
-      private void _fire_dp_after_send()
-      {
+      private void _fire_dp_after_send() {
          // as soon as the data is published, its reflected in the stream state
          Log.assert(_last_sent_msg != null);
          BlueRow r = new BlueRow(_table.schema, _table.Count, _last_sent_msg.msg_id, _last_sent_msg.timestamp, DBNull.Value, DBNull.Value,
@@ -143,8 +133,7 @@ namespace bts
          myFireEvent(dp_out);
       }
 
-      private void _fire_dp_after_recv()
-      {
+      private void _fire_dp_after_recv() {
          Log.assert(_last_recvd_msg != null);
          // update the last publish row
          int id = _table.Count - 1;
@@ -158,13 +147,11 @@ namespace bts
          myFireEvent(dp_out);
       }
 
-      private string _get_next_id()
-      {
+      private string _get_next_id() {
          return _table.Count.ToString();
       }
 
-      private void _publish_next_packet()
-      {
+      private void _publish_next_packet() {
          var bytes = PayloadMessage.create_data(_payload_sz, _payload_byte);
          _last_sent_msg = new PayloadMessage(_get_next_id(), epoch_now(), bytes);
          var ser_bytes = _last_sent_msg.serialize();
@@ -173,25 +160,19 @@ namespace bts
          _last_publish_time = DateTime.Now;
       }
 
-      public void doWork()
-      {
-         Func<IEnumerable<PubSubServerMessage>, PayloadMessage> find_packet = (updates) =>
-         {
-            foreach (var u in updates)
-            {
+      public void doWork() {
+         Func<IEnumerable<PubSubServerMessage>, PayloadMessage> find_packet = (updates) => {
+            foreach (var u in updates) {
                var recv_msg = PayloadMessage.deserialize(u.data);
-               if (recv_msg.msg_id == _last_sent_msg.msg_id)
-               {
+               if (recv_msg.msg_id == _last_sent_msg.msg_id) {
                   return recv_msg;
                }
             }
             return null;
          };
-         switch (_state)
-         {
+         switch (_state) {
             case IOState.SEND:
-               if (DateTime.Now - _last_publish_time >= _publish_period)
-               {
+               if (DateTime.Now - _last_publish_time >= _publish_period) {
                   _publish_next_packet();
                   _last_publish_time = DateTime.Now;
                   _fire_dp_after_send();
@@ -203,8 +184,7 @@ namespace bts
                var updates = _sub_core.get_updates(TimeSpan.Zero);
                updates = updates.Where(u => u.tag == _tag_name);
                var recv_msg = find_packet(updates);
-               if (recv_msg != null)
-               {
+               if (recv_msg != null) {
                   _last_recvd_msg = recv_msg;
                   _fire_dp_after_recv();
                   _state = IOState.SEND;
@@ -214,10 +194,9 @@ namespace bts
          }
       }
 
-      public static long epoch_now()
-      {
+      public static long epoch_now() {
          return DateTimeOffset.UtcNow.ToUnixTimeMilliseconds();
       }
 
    }
-}
+}

src/DataStreamSuite/DataStream/Primitives/Misc/TestStream.cs

@@ -95,7 +95,7 @@ namespace BCG {
             sub_msg.serialize_into_str(str);
             str.Position = 0;
             bts.SubscribeMsg sub_msg_ = new bts.SubscribeMsg();
-            sub_msg_.consruct_deserialize(str);
+            sub_msg_.construct_deserialize(str);
             bool res = sub_msg.Equals(sub_msg_);
             return new TestResult {
                 test_name = "SubscribeMsg",
@@ -117,7 +117,7 @@ namespace BCG {
             pub_msg.serialize_into_str(str);
             str.Position = 0;
             var pub_msg_ = new bts.PublishMsg();
-            pub_msg_.consruct_deserialize(str);
+            pub_msg_.construct_deserialize(str);
             bool res = pub_msg.Equals(pub_msg_);
             return new TestResult {
                 test_name = "PublishMsg",
@@ -135,7 +135,7 @@ namespace BCG {
             msg.serialize_into_str(str);
             str.Position = 0;
             var msg_ = new bts.PubSubServerMessage();
-            msg_.consruct_deserialize(str);
+            msg_.construct_deserialize(str);
             bool res = msg.Equals(msg_);
             return new TestResult {
                 test_name = "PubSubServerMessage",
@@ -156,7 +156,7 @@ namespace BCG {
             msg.serialize_into_str(str);
             str.Position = 0;
             var msg_ = new PubSubClientMessage();
-            msg_.consruct_deserialize(str);
+            msg_.construct_deserialize(str);
             var res = msg.Equals(msg_);
             return new TestResult {
                 test_name = "PubSubClientMessage",
@@ -617,14 +617,15 @@ namespace BCG {
 
 
         private static TestResult _run_single_tag_data_test_clean() {
+            // var tagHandlerFactory
             // create message
-            var test_name = MethodBase.GetCurrentMethod().Name;
+            var test_name = MethodBase.GetCurrentMethod()?.Name;
             var schema = create_schema_with_all_types();
             var tag = "DS_" + schema.tableName;
             BlueTable table = new BlueTable(schema);
             IStreamIOMsgSerializer serializer = new StreamIOSerializationController();
             IStreamMsgDeserilizer deser = new StreamIODeserializationController();
-            bts.SingleTagData st_data = new bts.SingleTagData(tag, DSTagManager.DataStreamTagType);
+            bts.SingleTagData st_data = new bts.SingleTagData(tag, DSTagManager.DataStreamTagType, new PubsubServerStreamTagHandler());
             // create with 100 rows
             int n_rows = 100;
             bool is_succ = true;

src/DataStreamSuite/DataStream/Primitives/Misc/TimeSeriesStream.cs

@@ -1,8 +1,9 @@
+using Google.Protobuf.WellKnownTypes;
+
 using System;
+using BCG.Utilities.General;
 
-using BCG;
-
-namespace bts.SilverPine {
+namespace BCG {
    // Keeps adding changing rows to the stream, hence preserving the history
    // and creating a time series, for example this can be used to make bbo updates
    // into a time series of prices/qty 
@@ -40,6 +41,7 @@ namespace bts.SilverPine {
          }
          _table.addRows(added_rows);
          myFireEvent(new DataPacket(_table, added_rows, new BlueTable(_table.schema)));
+
       }
    }
 

src/DataStreamSuite/DataStream/Primitives/MongoDB/BenzingaNewStream.cs

@@ -7,45 +7,26 @@ namespace bts {
    using System.Text.Json;
    using System.Text.Json.Nodes;
    [DataStreamUserRespresentation("benzingaNews")]
-   public class BenzingaNewsStream : ViewStream {
+   public class BenzingaNewStream : ViewStream {
       private IBlueTableSchema _schema = null;
       private int _id = 0;
-
-
-      private static ViewStream _pre_process_raw_kvstream(ViewStream mongo_db_str) {
-         ViewStream r = new KeyValueToRelationalStream(mongo_db_str, "doc_id", "key", "value");
-         r = new DateStandardizationStream(r, "created_at", "ISO8601");
-         return r;
-      }
-
-      public BenzingaNewsStream(string connection_string, string db_name, string collection_name, string poll_period, string ttl)
-      : this(new MongoDBStream(connection_string, db_name, collection_name, poll_period), ttl) {
-      }
-
-      // public BenzingaNewsStream()
-      // : this(new MongoDBStream(DataStreamPub connection_string, db_name, collection_name, poll_period), ttl) {
-      // }
-
-      // assumes it gets a raw mongodb_raw_kv_stream with key value pairs
-      public BenzingaNewsStream(ViewStream mongodb_raw_kv_stream, string ttl) {
-         ViewStream output_stream = _pre_process_raw_kvstream(mongodb_raw_kv_stream);
+      // constructor takes in the raw mongo db stream, and the time to live (ttl), "7d" for example
+      public BenzingaNewStream(ViewStream mongodb_raw_stream, string ttl) {
          // Hiding body (with project, columns can be hidden, or renamed)
-         output_stream = new ProjectStream(output_stream, "!body");
+         ViewStream output_stream = new ProjectStream(mongodb_raw_stream, "!body");
          // add column news_source = 'Benzinga', so we can append other news sources
          output_stream = new AddColumnsStream(output_stream, @"news_source : Benzinga");
          // now SuperMapStream is used to map one row into multiple, one per symbol,
          // (SuperMapStream takes 2 lambdas, one to get the enw schema and another to 
          // return an array of rows, per one row)
-         output_stream = new SuperMapStream(output_stream,
+         output_stream = new SuperMapStream(mongodb_raw_stream,
 
         (IBlueTableSchema s) => {
            // this lambda returns the new schema (ie with extra cols symbol, exchange, is_primary_exchange)
            if (_schema == null) {
               var out_schema = s;
               out_schema = BlueDataUtility.remove_columns(out_schema, "securities");
-              out_schema = BlueDataUtility.remove_columns(out_schema, "body");
-              out_schema = BlueDataUtility.remove_columns(out_schema, "tags");
-              out_schema = BlueDataUtility.add_column(out_schema, "doc_id_internal", typeof(string));
+              out_schema = BlueDataUtility.add_column(s, "doc_id_internal", typeof(string));
               out_schema = BlueDataUtility.add_column(out_schema, "symbol", typeof(string));
               out_schema = BlueDataUtility.add_column(out_schema, "exchange", typeof(string));
               out_schema = BlueDataUtility.add_column(out_schema, "is_primary_exchange", typeof(bool));
@@ -63,13 +44,11 @@ namespace bts {
            }
            foreach (JsonNode jnode in json_objs) {
               if (jnode is JsonObject json_object) {
-                 string symbol = jnode["symbol"] == null ? String.Empty : jnode["symbol"].ToString();
-                 string exchange = jnode["exchange"] == null ? String.Empty : jnode["exchange"].ToString();
                  MutableBlueRow nr = new MutableBlueRow(_schema, r);
                  nr.setValue("id", _id++);
                  nr.setValue("doc_id_internal", $"{r["doc_id"]}_{jnode["symbol"]}");
-                 nr.setValue("symbol", symbol);
-                 nr.setValue("exchange", exchange);
+                 nr.setValue("symbol", jnode["symbol"]);
+                 nr.setValue("exchange", jnode["exchange"]);
                  nr.setValue("is_primary_exchange", Boolean.Parse(jnode["primary"].ToString()));
                  expanded_rows.Add(nr);
               }

src/DataStreamSuite/DataStream/Primitives/MongoDB/MongoDBStream.cs

@@ -7,127 +7,27 @@ using MongoDB.Driver;
 using System;
 using System.Collections.Generic;
 using System.Diagnostics;
-using System.Threading.Tasks;
-using System.Threading;
 
 namespace bts {
-    // [DataStreamUserRespresentation("mongodb_sql")]
-    // public class MongoDBSQLStream : ViewStream, IWorker {
-    //     private ViewStreamDatabase _vsd;
-    //     private readonly string _connection_string, _db_name, _collection_name;
-    //     private MongoClient _client;
-    //     // make this a sorted table
-    //     private BlueTable _table;
-
-
-    //     // Access the database
-    //     private IMongoDatabase _database;
-    //     private IDictionary<string, int> _id_map = new Dictionary<string, int>();
-    //     // mongodb://192.168.1.80:27017
-
-    //     public MongoDBSQLStream(string connection_string, string db_name, string collection_name)
-    //     : this(connection_string, db_name, collection_name, ViewStreamDatabase.instance) {
-
-    //     }
-    //     public MongoDBSQLStream(string connection_string, string db_name, string collection_name, ViewStreamDatabase vsd) {
-    //         bts.Log.Gen.info($"MongoDBStream constructed connection_string: {connection_string} " +
-    //                                 $"db_name: {db_name}");
-    //         _connection_string = connection_string;
-    //         _db_name = db_name;
-    //         _collection_name = collection_name;
-    //         _vsd = vsd;
-    //         _client = new MongoClient(_connection_string);
-    //         _database = _client.GetDatabase(_db_name);
-    //     }
-
-    //     protected override void myOnFirstSubscriber() {
-    //         MasterWorker.getInstance(false).addWorker(this, TimeSpan.FromSeconds(10));
-    //     }
-
-    //     protected override void myOnLastSubscriber() {
-    //         MasterWorker.getInstance(false).removeWorker(this);
-    //     }
-
-    //     public void doWork() {
-    //         _read_mogodb();
-    //     }
-
-    //     private void _read_mogodb() {
-    //         var timer = Stopwatch.StartNew();
-    //         // Access the collection
-    //         IMongoCollection<BsonDocument> collection = _database.GetCollection<BsonDocument>(name: _collection_name);
-
-    //         // Fetch all documents from the collection
-    //         var documents = collection.Find(Builders<BsonDocument>.Filter.Empty).ToCursor();
-    //         _table = _table == null ? null : new BlueTable(_table.schema);
-    //         // Iterate through the documents and print values
-    //         foreach (var document in documents.ToEnumerable()) {
-    //             bts.Log.Gen.info($"document: {document} json: {document.ToJson()}");
-    //             if (_table == null) {
-    //                 _init_table(document);
-    //             }
-    //             _add_row(document);
-    //         }
-    //         timer.Stop();
-    //         Log.Gen.info($"MongoDBStream time to read {_collection_name} {timer.ElapsedMilliseconds}ms");
-    //         timer.Restart();
-    //         var dp = BlueDataUtility.build_dp_inc(_table, myLastValue == null ? null : myLastValue as DataPacket, true);
-    //         timer.Stop();
-    //         Log.Gen.info($"MongoDBStream time to compare tables :{timer.ElapsedMilliseconds}ms");
-    //         if (dp.addedRows.Count == 0 && dp.deletedRows.Count == 0)
-    //             return;
-    //         myFireEvent(dp);
-    //     }
-
-    //     private void _init_table(BsonDocument doc_row) {
-    //         bts.Log.assert(_table == null);
-    //         List<ColumnInfo> cis = new List<ColumnInfo>();
-    //         cis.Add((new ColumnInfo("id", 0, typeof(Int32))));
-    //         foreach (var n in doc_row.Names) {
-    //             cis.Add(new ColumnInfo(n, cis.Count, typeof(string)));
-    //         }
-    //         IBlueTableSchema schema = new BlueTableSchema(cis.ToArray(), _collection_name);
-    //         _table = new BlueTable(schema);
-    //     }
-
-    //     private void _add_row(BsonDocument doc) {
-    //         var key_val = doc["_id"];
-    //         var key = key_val.ToString();
-    //         int id;
-    //         if (!_id_map.TryGetValue(key, out id)) {
-    //             id = _id_map.Count;
-    //             _id_map.Add(key, id);
-    //         }
-    //         MutableBlueRow nr = new MutableBlueRow(_table.schema);
-    //         nr.setValue("id", id);
-    //         foreach (var n in doc.Names) {
-    //             var val = doc[n].ToString();
-    //             nr.setValue(n, val);
-    //         }
-    //         _table.addRow(nr);
-    //     }
-    // }
-
-    [DataStreamUserRespresentation("mongodb")]
-    public class MongoDBStream : ViewStream, IWorker {
+    [DataStreamUserRespresentation("mongodb_sql")]
+    public class MongoDBSQLStream : ViewStream, IWorker {
         private ViewStreamDatabase _vsd;
         private readonly string _connection_string, _db_name, _collection_name;
-        private readonly TimeSpan _poll_period;
         private MongoClient _client;
+        // make this a sorted table
         private BlueTable _table;
-        private SyncQueue<DataPacket> _table_queue = new SyncQueue<DataPacket>();
-        private BlockingQueue _work_queue = new BlockingQueue();
+
 
         // Access the database
         private IMongoDatabase _database;
         private IDictionary<string, int> _id_map = new Dictionary<string, int>();
         // mongodb://192.168.1.80:27017
 
-        public MongoDBStream(string connection_string, string db_name, string collection_name, string poll_period)
-        : this(connection_string, db_name, collection_name, TimeBasedFilter.parse_duration(poll_period), ViewStreamDatabase.instance) {
+        public MongoDBSQLStream(string connection_string, string db_name, string collection_name)
+        : this(connection_string, db_name, collection_name, ViewStreamDatabase.instance) {
 
         }
-        public MongoDBStream(string connection_string, string db_name, string collection_name, TimeSpan poll_period, ViewStreamDatabase vsd) {
+        public MongoDBSQLStream(string connection_string, string db_name, string collection_name, ViewStreamDatabase vsd) {
             bts.Log.Gen.info($"MongoDBStream constructed connection_string: {connection_string} " +
                                     $"db_name: {db_name}");
             _connection_string = connection_string;
@@ -135,30 +35,114 @@ namespace bts {
             _collection_name = collection_name;
             _vsd = vsd;
             _client = new MongoClient(_connection_string);
-            _poll_period = poll_period;
             _database = _client.GetDatabase(_db_name);
         }
 
         protected override void myOnFirstSubscriber() {
-            _vsd.addWorker(this);
-            //MasterWorker.getInstance(false).addWorker(this, _poll_period);
+            MasterWorker.getInstance(false).addWorker(this, TimeSpan.FromSeconds(10));
         }
 
         protected override void myOnLastSubscriber() {
-            _vsd.removeWorker(this);
-            //MasterWorker.getInstance(false).removeWorker(this);
+            MasterWorker.getInstance(false).removeWorker(this);
         }
 
-        private DateTime _last_poll_time = DateTime.MinValue;
         public void doWork() {
-            var now = DateTime.Now;
-            if (now - _last_poll_time > _poll_period) {
-                ThreadPool.QueueUserWorkItem(_ => _read_mogodb());
-                _last_poll_time = DateTime.Now;
+            _read_mogodb();
+        }
+
+        private void _read_mogodb() {
+            var timer = Stopwatch.StartNew();
+            // Access the collection
+            IMongoCollection<BsonDocument> collection = _database.GetCollection<BsonDocument>(name: _collection_name);
+
+            // Fetch all documents from the collection
+            var documents = collection.Find(Builders<BsonDocument>.Filter.Empty).ToCursor();
+            _table = _table == null ? null : new BlueTable(_table.schema);
+            // Iterate through the documents and print values
+            foreach (var document in documents.ToEnumerable()) {
+                bts.Log.Gen.info($"document: {document} json: {document.ToJson()}");
+                if (_table == null) {
+                    _init_table(document);
+                }
+                _add_row(document);
             }
-            var dp = _table_queue.dequeue(0);
-            if (dp != null)
-                myFireEvent(dp);
+            timer.Stop();
+            Log.Gen.info($"MongoDBStream time to read {_collection_name} {timer.ElapsedMilliseconds}ms");
+            timer.Restart();
+            var dp = BlueDataUtility.build_dp_inc(_table, myLastValue == null ? null : myLastValue as DataPacket, true);
+            timer.Stop();
+            Log.Gen.info($"MongoDBStream time to compare tables :{timer.ElapsedMilliseconds}ms");
+            if (dp.addedRows.Count == 0 && dp.deletedRows.Count == 0)
+                return;
+            myFireEvent(dp);
+        }
+
+        private void _init_table(BsonDocument doc_row) {
+            bts.Log.assert(_table == null);
+            List<ColumnInfo> cis = new List<ColumnInfo>();
+            cis.Add((new ColumnInfo("id", 0, typeof(Int32))));
+            foreach (var n in doc_row.Names) {
+                cis.Add(new ColumnInfo(n, cis.Count, typeof(string)));
+            }
+            IBlueTableSchema schema = new BlueTableSchema(cis.ToArray(), _collection_name);
+            _table = new BlueTable(schema);
+        }
+
+        private void _add_row(BsonDocument doc) {
+            var key_val = doc["_id"];
+            var key = key_val.ToString();
+            int id;
+            if (!_id_map.TryGetValue(key, out id)) {
+                id = _id_map.Count;
+                _id_map.Add(key, id);
+            }
+            MutableBlueRow nr = new MutableBlueRow(_table.schema);
+            nr.setValue("id", id);
+            foreach (var n in doc.Names) {
+                var val = doc[n].ToString();
+                nr.setValue(n, val);
+            }
+            _table.addRow(nr);
+        }
+    }
+
+    [DataStreamUserRespresentation("mongodb")]
+    public class MongoDBStream : ViewStream, IWorker {
+        private ViewStreamDatabase _vsd;
+        private readonly string _connection_string, _db_name, _collection_name;
+        private MongoClient _client;
+        private BlueTable _table;
+
+        // Access the database
+        private IMongoDatabase _database;
+        private IDictionary<string, int> _id_map = new Dictionary<string, int>();
+        // mongodb://192.168.1.80:27017
+
+        public MongoDBStream(string connection_string, string db_name, string collection_name)
+        : this(connection_string, db_name, collection_name, ViewStreamDatabase.instance) {
+
+        }
+        public MongoDBStream(string connection_string, string db_name, string collection_name, ViewStreamDatabase vsd) {
+            bts.Log.Gen.info($"MongoDBStream constructed connection_string: {connection_string} " +
+                                    $"db_name: {db_name}");
+            _connection_string = connection_string;
+            _db_name = db_name;
+            _collection_name = collection_name;
+            _vsd = vsd;
+            _client = new MongoClient(_connection_string);
+            _database = _client.GetDatabase(_db_name);
+        }
+
+        protected override void myOnFirstSubscriber() {
+            MasterWorker.getInstance(false).addWorker(this, TimeSpan.FromSeconds(10));
+        }
+
+        protected override void myOnLastSubscriber() {
+            MasterWorker.getInstance(false).removeWorker(this);
+        }
+
+        public void doWork() {
+            _read_mogodb();
         }
 
         private void _read_mogodb() {
@@ -182,11 +166,10 @@ namespace bts {
             timer.Restart();
             var dp = BlueDataUtility.build_dp_inc(_table, myLastValue == null ? null : myLastValue as DataPacket, true);
             timer.Stop();
-            Log.Gen.info($"MongoDBStream time to compare tables: {timer.ElapsedMilliseconds}ms"); ;
+            Log.Gen.info($"MongoDBStream time to compare tables: {timer.ElapsedMilliseconds}ms");;
             if (dp.addedRows.Count == 0 && dp.deletedRows.Count == 0)
                 return;
-            _table_queue.enqueue(dp);
-            //myFireEvent(dp);
+            myFireEvent(dp);
         }
 
         private void _init_table(BsonDocument doc_row) {
@@ -225,46 +208,46 @@ namespace bts {
         }
     }
 
-    // [DataStreamUserRespresentation("mongodb_flat")]
-    // public class MongoDBFlatStream : ViewStream {
-    //     public MongoDBFlatStream(string connection_string, string db_name, string collection_name, string timestamp_col, string ttl_duration)
-    //     :
-    //     this(ViewStreamDatabase.instance, connection_string, db_name, collection_name, timestamp_col, TimeBasedFilter.parse_duration(ttl_duration)) { }
+    [DataStreamUserRespresentation("mongodb_flat")]
+    public class MongoDBFlatStream : ViewStream {
+        public MongoDBFlatStream(string connection_string, string db_name, string collection_name, string timestamp_col, string ttl_duration)
+        :
+        this(ViewStreamDatabase.instance, connection_string, db_name, collection_name, timestamp_col, TimeBasedFilter.parse_duration(ttl_duration)) { }
 
-    //     public MongoDBFlatStream(ViewStreamDatabase vsd, string connection_string, string db_name, string collection_name, string timestamp_col, TimeSpan ttl_duration) {
-    //         ViewStream rs = new MongoDBStream(connection_string, db_name, collection_name, vsd);
-    //         rs = new KeyValueToRelationalStream(rs, "doc_id", "key", "val");
-    //         rs = new DateStandardizationStream(rs, timestamp_col, DateConversion.ISO8601);
-    //         //rs = new TimeBasedFilterStream(vsd, rs, timestamp_col, ttl_duration);
-    //         mySetRealSource(rs);
-    //     }
-    // }
+        public MongoDBFlatStream(ViewStreamDatabase vsd, string connection_string, string db_name, string collection_name, string timestamp_col, TimeSpan ttl_duration) {
+            ViewStream rs = new MongoDBStream(connection_string, db_name, collection_name, vsd);
+            rs = new KeyValueToRelationalStream(rs, "doc_id", "key", "val");
+            rs = new DateStandardizationStream(rs, timestamp_col, DateConversion.ISO8601);
+            //rs = new TimeBasedFilterStream(vsd, rs, timestamp_col, ttl_duration);
+            mySetRealSource(rs);
+        }
+    }
 
 
 
-    // [DataStreamUserRespresentation("newsreplay")]
-    // public class NewsReplayStream : ViewStream {
-    //     public NewsReplayStream(string connection_string, string db_name, string collection_name,
-    //                             int buffer_sz, string fire_period) :
-    //     this(ViewStreamDatabase.instance, connection_string, db_name, collection_name, buffer_sz, TimeBasedFilter.parse_duration(fire_period)) { }
+    [DataStreamUserRespresentation("newsreplay")]
+    public class NewsReplayStream : ViewStream {
+        public NewsReplayStream(string connection_string, string db_name, string collection_name,
+                                int buffer_sz, string fire_period) :
+        this(ViewStreamDatabase.instance, connection_string, db_name, collection_name, buffer_sz, TimeBasedFilter.parse_duration(fire_period)) { }
 
-    //     public NewsReplayStream(ViewStreamDatabase vsd, string connection_string, string db_name, string collection_name,
-    //                             int buffer_sz, TimeSpan fire_period) {
-    //         ViewStream r;
+        public NewsReplayStream(ViewStreamDatabase vsd, string connection_string, string db_name, string collection_name,
+                                int buffer_sz, TimeSpan fire_period) {
+            ViewStream r;
 
-    //         r = new MongoDBStream(connection_string, db_name, collection_name, vsd);
-    //         r = new KeyValueToRelationalStream(r,
-    //                                             "doc_id",
-    //                                             "key",
-    //                                             "value");
-    //         r = new DateStandardizationStream(r,
-    //                                            "created_at",
-    //                                            "ISO8601");
-    //         r = new SnapshotReplayStream(vsd, r, "created_at", buffer_sz, fire_period);
+            r = new MongoDBStream(connection_string, db_name, collection_name, vsd);
+            r = new KeyValueToRelationalStream(r,
+                                                "doc_id",
+                                                "key",
+                                                "value");
+            r = new DateStandardizationStream(r,
+                                               "created_at",
+                                               "ISO8601");
+            r = new SnapshotReplayStream(vsd, r, "created_at", buffer_sz, fire_period);
 
-    //         mySetRealSource(r);
-    //     }
-    // }
+            mySetRealSource(r);
+        }
+    }
 
 
 }

src/DataStreamSuite/DataStream/Primitives/Project/New/NewProjectStream.cs

@@ -6,7 +6,6 @@ namespace bts {
    using System;
    using System.Collections;
    using System.Collections.Generic;
-   using System.Linq;
 
    public class ProjectData {
       // schema from the base stream
@@ -137,24 +136,10 @@ namespace bts {
          HashSet<string> removed_cols = new HashSet<string>();
          Dictionary<string, string> mapped_to_orig_map = new Dictionary<string, string>();
          Dictionary<string, string> orig_to_mapped = new Dictionary<string, string>();
-         // by default every col will show up, so if we have the projection as (a:a1)
-         // it will convert a to a1, 
-         // examples: if a schema has cols a b c
-         // project(parent_str, "a:aa ") -> aa b c
-         // project(parent_str, "a:aa !c") -> aa b
-         // project(parent_str, "a:aa !") -> aa
-         // project(parent_str, "a:aa b = (aa*aa)!") -> aa b
-         bool hide_all = false;
          foreach (string p_tok in project_toks) {
             if (p_tok.StartsWith("!")) {
-               if (p_tok.Length == 1) {
-                  // if it is a "blanket "!"=> hide all cols except the one that
-                  // are stated explicitly
-                  hide_all = true;
-               } else {
-                  string removed_col = p_tok.Substring(1);
-                  removed_cols.Add(removed_col);
-               }
+               string removed_col = p_tok.Substring(1);
+               removed_cols.Add(removed_col);
             } else if (p_tok.Contains(":")) {
                string[] map_toks = p_tok.Split(":");
                string orig_col = map_toks[0];
@@ -162,22 +147,12 @@ namespace bts {
                mapped_to_orig_map.Add(mapped_col, orig_col);
                orig_to_mapped.Add(orig_col, mapped_col);
             } else {
-               if (input_schema.columnNames.Contains(p_tok)) {
-                  string orig_col = p_tok;
-                  string mapped_col = orig_col;
-                  orig_to_mapped.Add(orig_col, mapped_col);
-               }
+               Log.Gen.error($"NewProjectStream unexpecetd token: {p_tok}");
             }
          }
-
          // make the new schema
          List<ColumnInfo> new_cols = new List<ColumnInfo>();
          foreach (ColumnInfo ci in input_schema.columnInfo) {
-            if (ci.colName == "id") {
-               ColumnInfo nc = new ColumnInfo(ci.colName, new_cols.Count, ci.colType);
-               new_cols.Add(nc);
-               continue;
-            }
             if (removed_cols.Contains(ci.colName)) {
                continue;
             } else if (orig_to_mapped.ContainsKey(ci.colName)) {
@@ -185,13 +160,8 @@ namespace bts {
                ColumnInfo nc = new ColumnInfo(mapped_name, new_cols.Count, ci.colType);
                new_cols.Add(nc);
             } else {
-               if (hide_all)
-                  continue;
-               //if (ci.colName == "id" || removed_cols.Count > 0) {
-               // unless columns are  explicitely hidden, all columns show up
                ColumnInfo nc = new ColumnInfo(ci.colName, new_cols.Count, ci.colType);
                new_cols.Add(nc);
-               //}
             }
          }
          IBlueTableSchema new_schema = new BlueTableSchema(input_schema.tableName, new_cols.ToArray());

src/DataStreamSuite/DataStream/Primitives/SQL/SQLite/SQLiteHelper.cs

@@ -1,122 +0,0 @@
-using System;
-using System.Data;
-using System.Data.SQLite;
-using System.Diagnostics;
-using System.Collections;
-
-
-using BCG;
-using System.Linq;
-using System.Data.Entity;
-using System.Collections.Generic;
-using System.Data.SQLite;
-
-
-namespace bts.SilverPine {
-
-   public static class SQLiteHelper {
-      public static string location = "C:\\Users\\m_tah\\development\\data\\";
-      public static string db_name = "stream_data.db";
-
-      public static DataTable execute_select_query(string connection_string, string query) {
-         // Create a new DataTable to store the results
-         DataTable dataTable = new DataTable();
-
-         // Create and open a SQLite connection using the provided connection string
-         using (SQLiteConnection connection = new SQLiteConnection(connection_string)) {
-            try {
-               connection.Open();
-
-               // Create a SQLite command with the provided query
-               using (SQLiteCommand command = new SQLiteCommand(query, connection)) {
-                  // Use SQLiteDataAdapter to fill the DataTable
-                  using (SQLiteDataAdapter adapter = new SQLiteDataAdapter(command)) {
-                     adapter.Fill(dataTable);
-                  }
-               }
-            } catch (Exception ex) {
-               Console.WriteLine("An error occurred: " + ex.Message);
-            }
-         }
-         // Return the populated DataTable
-         return dataTable;
-      }
-
-      public static List<Dictionary<string, object>> ExecuteBulkRead(string connectionString, string query) {
-         var result = new List<Dictionary<string, object>>();
-
-         using (var connection = new SQLiteConnection(connectionString)) {
-            connection.Open();
-
-            // Begin a transaction to optimize read speed
-            using (var transaction = connection.BeginTransaction())
-            using (var command = new SQLiteCommand(query, connection, transaction))
-            using (var reader = command.ExecuteReader()) {
-               while (reader.Read()) {
-                  var row = new Dictionary<string, object>();
-
-                  for (int i = 0; i < reader.FieldCount; i++) {
-                     string columnName = reader.GetName(i);
-                     object columnValue = reader.IsDBNull(i) ? null : reader.GetValue(i);
-                     row[columnName] = columnValue;
-                  }
-
-                  result.Add(row);
-               }
-
-               // Commit the transaction
-               transaction.Commit();
-            }
-         }
-
-         return result;
-      }
-
-      public static List<Dictionary<string, object>> get_table(string connectionString, string query) {
-         var result = new List<Dictionary<string, object>>();
-
-         using (var connection = new SQLiteConnection(connectionString)) {
-            try {
-               connection.Open();
-
-               using (var command = new SQLiteCommand(query, connection))
-               using (var reader = command.ExecuteReader()) {
-                  while (reader.Read()) {
-                     var row = new Dictionary<string, object>();
-
-                     for (int i = 0; i < reader.FieldCount; i++) {
-                        string columnName = reader.GetName(i);
-                        object columnValue = reader.IsDBNull(i) ? null : reader.GetValue(i);
-                        row[columnName] = columnValue;
-                     }
-
-                     result.Add(row);
-                  }
-               }
-            } catch (Exception ex) {
-               Console.WriteLine("An error occurred: " + ex.Message);
-            }
-         }
-         return result;
-      }
-
-      public static IEnumerable<Dictionary<string, object>> get_table_2(string connectionString, string query) {
-         using (var connection = new SQLiteConnection(connectionString)) {
-            connection.Open();
-            using (var command = new SQLiteCommand(query, connection))
-            using (var reader = command.ExecuteReader()) {
-               while (reader.Read()) {
-                  var row = new Dictionary<string, object>();
-                  for (int i = 0; i < reader.FieldCount; i++) {
-                     string columnName = reader.GetName(i);
-                     object columnValue = reader.IsDBNull(i) ? null : reader.GetValue(i);
-                     row[columnName] = columnValue;
-                  }
-                  yield return row;
-               }
-            }
-         }
-      }
-   }
-}
-

src/DataStreamSuite/DataStream/Primitives/SQL/SQLite/SqliteStream.cs

@@ -1,73 +0,0 @@
-namespace bts.SilverPine {
-   using System;
-   using System.Data;
-   using System.Data.SQLite;
-   using System.Diagnostics;
-   using System.Collections;
-
-
-   using BCG;
-
-   [DataStreamUserRespresentation("SQLite")]
-   public class SQLiteStream : ViewStream {
-      private readonly string _connection_string, _query;
-      private Stopwatch _stopwatch = new Stopwatch();
-
-      private SQLiteStream(string connection_string, string query, bool noop) {
-         _connection_string = connection_string;
-         _query = query;
-      }
-
-      public SQLiteStream(string db_name, string query)
-      :
-      this($"Data Source={SQLiteHelper.location}{db_name};Version=3", query, true) { }
-
-      public SQLiteStream(string query)
-      :
-      this(SQLiteHelper.db_name, query) {
-      }
-
-
-      private void _read_from_sqlite_and_fire() {
-         _stopwatch.Start();
-         DataTable dt = SQLiteHelper.execute_select_query(_connection_string, _query);
-         _stopwatch.Stop();
-         Log.Gen.info($"SQLiteStream.ExecuteSelectQuery time to read db: {_stopwatch.ElapsedMilliseconds} msecs");
-         _stopwatch.Restart();
-         #region perf measures
-         // SQLiteHelper.get_table(_connection_string, _query);
-         // _stopwatch.Stop();
-         // Log.Gen.info($"SQLiteStream.get_table time to read db: {_stopwatch.ElapsedMilliseconds} msecs");
-         // _stopwatch.Restart();
-         // SQLiteHelper.ExecuteBulkRead(_connection_string, _query);
-         // _stopwatch.Stop();
-         // Log.Gen.info($"SQLiteStream.ExecuteBulkRead time to read db: {_stopwatch.ElapsedMilliseconds} msecs");
-         // _stopwatch.Restart();
-         // SQLiteHelper.get_table_2(_connection_string, _query);
-         // _stopwatch.Stop();
-         // Log.Gen.info($"SQLiteStream.get_table_2 time to read db: {_stopwatch.ElapsedMilliseconds} msecs");
-         // _stopwatch.Restart();
-         #endregion
-         IBlueRowCollection rows = BCG.BlueDataUtility.toIBlueRowCollection(dt);
-         //IBlueRowCollection rows = new DataTableWrapperTable(dt);
-         _stopwatch.Stop();
-         Log.Gen.info($"SQLiteStream._read_from_sqlite_and_fire time to load into IBlueRowCollection: {_stopwatch.ElapsedMilliseconds} msecs");
-         DataPacket dp = BlueDataUtility.build_dp_inc(rows, myLastValue as DataPacket);
-         myFireEvent(dp);
-      }
-
-      protected override void myOnFirstSubscriber() {
-         _read_from_sqlite_and_fire();
-      }
-
-      protected override void myOnLastSubscriber() {
-         // this value will be cleared by the base class
-         // but in this particular stream this behaviour
-         // is not usefull since any new subscribers 
-         // would force the query to be recreated
-         //myLastDP = myLastValue;
-      }
-
-   }
-
-}

src/DataStreamSuite/DataStream/Primitives/SideEffectStreams/CSVSaverStream.cs

@@ -3,15 +3,12 @@ using System.Collections.Generic;
 using System.Text;
 using System.IO;
 using BCG.Utilities;
-using System.Diagnostics;
-using Microsoft.Data.Analysis;
-using Parquet;
-using Parquet.Data;
 
-
-namespace BCG {
+namespace BCG
+{
    [DataStreamUserRespresentation("CSVSaver")]
-   public class CSVSaverStream : ViewStream, ISideEffectStream {
+   public class CSVSaverStream : ViewStream, ISideEffectStream
+   {
       private IBlueTableSchema myResultSchema = new BlueTableSchema("CSVResult", new ColumnInfo("id", 0, typeof(int)),
          new ColumnInfo("fileName", 1, typeof(string)),
          new ColumnInfo("nWrites", 2, typeof(int)),
@@ -23,9 +20,11 @@ namespace BCG {
       private int myNWrites = 0, myNMaxWrites = Int32.MaxValue;
 
       public CSVSaverStream(ViewStream vs, string fullyQualifiedFileName)
-         : this(vs, fullyQualifiedFileName, true, Int32.MaxValue, false) { }
+         : this(vs, fullyQualifiedFileName, true, Int32.MaxValue, false)
+      { }
 
-      public CSVSaverStream(ViewStream vs, string fullyQualifiedFileName, bool overwrite, int nMaxWrites, bool fireResultSnapshot) {
+      public CSVSaverStream(ViewStream vs, string fullyQualifiedFileName, bool overwrite, int nMaxWrites, bool fireResultSnapshot)
+      {
          myStream = vs;
          myFullyQualifiedFileName = fullyQualifiedFileName;
          myOverwrite = overwrite;
@@ -37,14 +36,18 @@ namespace BCG {
 
       protected override void myOnLastSubscriber() { myStream.onChange -= new ChangeDelegate(myHandleChanges); }
 
-      private void myHandleChanges(object newValue) {
+      private void myHandleChanges(object newValue)
+      {
          IBlueRowCollection totalState = (newValue as DataPacket).newRows;
-         if (totalState.Count > 0 && myNWrites < myNMaxWrites) {
+         if (totalState.Count > 0 && myNWrites < myNMaxWrites)
+         {
             myNWrites++;
             int i = 0;
             StreamWriter writer = new StreamWriter(new FileStream(myGetNewFileName(), FileMode.Create, FileAccess.Write));
-            foreach (IBlueRow r in totalState) {
-               if (i++ == 0) {
+            foreach (IBlueRow r in totalState)
+            {
+               if (i++ == 0)
+               {
                   string header = String.Empty;
                   foreach (string col in r.schema.columnNames)
                      if (col != "id")
@@ -56,17 +59,20 @@ namespace BCG {
                writer.Flush();
             }
             writer.Close();
-            if (myFireResultSnapshot) {
+            if (myFireResultSnapshot)
+            {
                IBlueRow nr = new BlueRow(myResultSchema, 0, myFullyQualifiedFileName, myNWrites, totalState.Count);
                BlueTable nRows = new BlueTable(myResultSchema);
                nRows.addRow(nr);
                myFireEvent(new DataPacket(nRows, nRows, new BlueTable(myLastValue == null ? new BlueTable(myResultSchema) : (myLastValue as DataPacket).newRows)));
-            } else
+            }
+            else
                myFireEvent(newValue);
          }
       }
 
-      private string myGetNewFileName() {
+      private string myGetNewFileName()
+      {
          string[] tags = myFullyQualifiedFileName.Split('.');
          if (myOverwrite)
             return tags[0] + ".csv";
@@ -74,11 +80,14 @@ namespace BCG {
             return tags[0] + "_" + myFormateDateTime(DateTime.Now) + ".csv";
       }
 
-      private static string myGetCSVRow(IBlueRow r) {
+      private static string myGetCSVRow(IBlueRow r)
+      {
          string rv = String.Empty;
          int i = 0;
-         foreach (string col in r.schema.columnNames) {
-            if (col != "id") {
+         foreach (string col in r.schema.columnNames)
+         {
+            if (col != "id")
+            {
                string colText = String.Empty;
                if (r[col] != null && r[col] != DBNull.Value)
                   colText = r[col].ToString();
@@ -90,7 +99,8 @@ namespace BCG {
          return rv;
       }
 
-      private string myFormateDateTime(DateTime dt) {
+      private string myFormateDateTime(DateTime dt)
+      {
          return
          string.Format("{0}-{1}-{2}-{3}-{4}-{5}-{6}",
                         dt.Year.ToString("0000"),
@@ -104,49 +114,52 @@ namespace BCG {
    }
 
    [DataStreamUserRespresentation("CSVReader")]
-   public class CSVReaderStream : ViewStream {
-      private readonly string _file_address;
-      Stopwatch _stopwatch = new Stopwatch();
-      public CSVReaderStream(string fileAddress) {
-         _file_address = fileAddress;
+   public class CSVReaderStream : ViewStream
+   {
+      private IBlueRowCollection myRows;
+
+      public CSVReaderStream(string fileAddress)
+      {
+         myRows = BlueDataUtility.createRowsFromCSVFileWithoutTypes(fileAddress);
       }
 
-      protected override void myOnFirstSubscriber() {
-         _stopwatch.Start();
-         DataFrame df = DataFrame.LoadCsv(_file_address);
-         _stopwatch.Stop();
-         bts.Log.Gen.info($"CSVReaderStream.myOnFirstSubscriber stream csv load col_count: {df.Columns.Count} rows, time: {_stopwatch.ElapsedMilliseconds} msecs");
-         _stopwatch.Start();
-         IBlueRowCollection rows = BlueDataUtility.createRowsFromCSVFileWithoutTypes(_file_address);
-         _stopwatch.Stop();
-         bts.Log.Gen.info($"CSVReaderStream.myOnFirstSubscriber stream csv load count: {rows.Count} rows, time: {_stopwatch.ElapsedMilliseconds} msecs");
-         DataPacket dp_out = BlueDataUtility.build_dp_inc(rows, myLastValue as DataPacket);
-         myFireEvent(dp_out);
+      protected override void myOnFirstSubscriber()
+      {
+         myFireEvent(new DataPacket(myRows, myRows, new BlueTable(myRows.schema)));
       }
    }
 
-   [DataStreamUserRespresentation("CSVReaderStreamMapper")]
-   public class CSVReaderStreamMapperStream : ViewStream {
-      public CSVReaderStreamMapperStream(ViewStream stream, string output_schema) {
+   [DataStreamUserRespresentation ("CSVReaderStreamMapper")]
+   public class CSVReaderStreamMapperStream : ViewStream
+   {
+      public CSVReaderStreamMapperStream(ViewStream stream, string output_schema)
+      {
          mySetRealSource(new MapStream(stream, new CSVReaderStreamMapper(output_schema)));
       }
    }
 
-   public class CSVReaderStreamMapper : IStreamMapper {
+   public class CSVReaderStreamMapper : IStreamMapper
+   {
       private IBlueTableSchema mySchema;
       private readonly string[] myOutputSchemaStringArray;
 
-      public CSVReaderStreamMapper(string outputSchema) {
+      public CSVReaderStreamMapper(string outputSchema)
+      {
          myOutputSchemaStringArray = outputSchema.Split(' ');
       }
 
-      public IBlueTableSchema getOutputSchema(IBlueTableSchema inputSchema) {
-         if (mySchema == null) {
+      public IBlueTableSchema getOutputSchema(IBlueTableSchema inputSchema)
+      {
+         if (mySchema == null)
+         {
             ColumnInfo[] ci = inputSchema.columnInfo;
-            foreach (string col in myOutputSchemaStringArray) {
+            foreach (string col in myOutputSchemaStringArray)
+            {
                string colName = col.Substring(0, col.IndexOf('\\')).ToLower();
-               for (int i = 0; i < ci.Length; i++) {
-                  if (ci[i].colName.ToLower().Equals(colName)) {
+               for (int i = 0; i < ci.Length; i++)
+               {
+                  if (ci[i].colName.ToLower().Equals(colName))
+                  {
                      Type t = myGetType(col.Substring(col.IndexOf('\\') + 1));
                      ci[i] = new ColumnInfo(ci[i].colName, i, t);
                      break;
@@ -158,52 +171,74 @@ namespace BCG {
          return mySchema;
       }
 
-      public IBlueRow map(IBlueRow row) {
+      public IBlueRow map(IBlueRow row)
+      {
          object[] vals = new object[row.schema.columnInfo.Length];
          row.Values.CopyTo(vals, 0);
-         for (int i = 0; i < mySchema.columnInfo.Length; i++) {
+         for (int i = 0; i < mySchema.columnInfo.Length; i++)
+         {
             Type t = mySchema.columnInfo[i].colType;
-            try {
+            try
+            {
                vals[i] = System.Convert.ChangeType(vals[i], mySchema.columnInfo[i].colType, System.Globalization.CultureInfo.InvariantCulture);
-            } catch {
-               if (t.Equals(typeof(Int16))) {
+            }
+            catch
+            {
+               if (t.Equals(typeof(Int16)))
+               {
                   Int16 result;
-                  if (Int16.TryParse((string)vals[i], out result)) {
+                  if (Int16.TryParse((string)vals[i], out result))
+                  {
                      vals[i] = result;
                      continue;
                   }
                   double r2;
-                  if (Double.TryParse((string)vals[i], out r2)) {
-                     try {
+                  if (Double.TryParse((string)vals[i], out r2))
+                  {
+                     try
+                     {
                         vals[i] = (Int16)r2;
                         continue;
-                     } catch { }
+                     }
+                     catch { }
                   }
-               } else if (t.Equals(typeof(Int32))) {
+               }
+               else if (t.Equals(typeof(Int32)))
+               {
                   Int32 result;
-                  if (Int32.TryParse((string)vals[i], out result)) {
+                  if (Int32.TryParse((string)vals[i], out result))
+                  {
                      vals[i] = result;
                      continue;
                   }
                   double r2;
-                  if (Double.TryParse((string)vals[i], out r2)) {
-                     try {
+                  if (Double.TryParse((string)vals[i], out r2))
+                  {
+                     try
+                     {
                         vals[i] = (Int32)r2;
                         continue;
-                     } catch { }
+                     }
+                     catch { }
                   }
-               } else if (t.Equals(typeof(Int64))) {
+               }
+               else if (t.Equals(typeof(Int64)))
+               {
                   Int64 result;
-                  if (Int64.TryParse((string)vals[i], out result)) {
+                  if (Int64.TryParse((string)vals[i], out result))
+                  {
                      vals[i] = result;
                      continue;
                   }
                   double r2;
-                  if (Double.TryParse((string)vals[i], out r2)) {
-                     try {
+                  if (Double.TryParse((string)vals[i], out r2))
+                  {
+                     try
+                     {
                         vals[i] = (Int64)r2;
                         continue;
-                     } catch { }
+                     }
+                     catch { }
                   }
                }
                vals[i] = DBNull.Value;
@@ -212,7 +247,8 @@ namespace BCG {
          return new BlueRow(mySchema, vals);
       }
 
-      private Type myGetType(string type) {
+      private Type myGetType(string type)
+      {
          type = type.ToLower();
          if (type.Contains("int"))
             return typeof(int);