feat: Add server side csharp sdk

For Ophil

.devcontainer/devcontainer.json

@@ -22,6 +22,7 @@
       "extensions": [
         "eamodio.gitlens",
         "llvm-vs-code-extensions.vscode-clangd",
+        "ms-python.python",
         "ms-vscode.cmake-tools",
         "ms-vscode.cpptools",
         "xaver.clang-format"

.env

@@ -46,4 +46,11 @@ SPARKD__SIM_FIX_PORT=8002
 SPARKD__SHM_SIZE=4gb
 SPARKD__PYTHON_REQUIREMENTS=requirements.txt
 
+### PubSubServer ###
+PUBSUB__CONFIG_DIR=src/DataStreamSuite/PubSubServer/Configurations/Development
+PUBSUB__SOURCE_DIR=build/Release/bin/net8.0/linux-x64/publish
+
+### MdPublisher ###
+MD_PUBLISHER__CONFIG_DIR=src/DataStreamSuite/DataStreamPublisher/Configurations/ReplayMarketData
+MD_PUBLISHER__SOURCE_DIR=build/Release/bin/net8.0/linux-x64/publish
 # TODO: add config for other services (pubsub, md-publisher, webapp, api-server, etc.)

.gitignore

@@ -55,6 +55,8 @@ trace_data/
 .cache/
 /src/Spark.Admin.WebApp/tsconfig.*.g.json
 /src/Spark.Admin.WebApp/src/bin/
+/src/**/*appsettings.Local.json
+/src/**/*/Configurations/Local/
 **/CMakeDeps.cmake
 **/CMakeDeps_win.cmake
 .devcontainer/

.vscode/launch.json

@@ -19,15 +19,19 @@
       "type": "coreclr",
       "request": "launch",
       "program": "",
+      "preLaunchTask": "Build Api Server (Debug)",
       "windows": {
-        "preLaunchTask": "Build Api Server (Debug)",
         "program": "spark_api_server.exe",
-        "args": [
-          "run",
-          "--sparkd",
-          "http://127.0.0.1:8000/config"
-        ],
-        "cwd": "build/Debug/bin/net7.0",
+      },
+      "linux": {
+        "program": "spark_api_server",
+      },
+      "args": [
+        "run",
+      ],
+      "cwd": "build/Debug/bin/net8.0",
+      "env": {
+        "DOTNET_ENVIRONMENT": "${input:apiServerEnvironment}",
       },
       "stopAtEntry": false,
       "console": "integratedTerminal"
@@ -39,43 +43,20 @@
       "program": "",
       "windows": {
         "program": "spark_api_server.exe",
-        "args": [
-          "run",
-          "--sparkd",
-          "http://127.0.0.1:8000/config"
-        ],
-        "cwd": "build/Debug/bin/net7.0",
+      },
+      "linux": {
+        "program": "spark_api_server",
+      },
+      "args": [
+        "run",
+      ],
+      "cwd": "build/Debug/bin/net8.0",
+      "env": {
+        "DOTNET_ENVIRONMENT": "${input:apiServerEnvironment}",
       },
       "stopAtEntry": false,
       "console": "integratedTerminal"
     },
-    {
-      "name": "Bts.DataStreamWebApp.Server",
-      "type": "coreclr",
-      "request": "launch",
-      "preLaunchTask": "Build DataStream WebApp (Debug)",
-      "program": "${workspaceFolder}/build/Debug/bin/net8.0/Bts.DataStreamWebApp.Server.dll",
-      "cwd": "${workspaceFolder}/build/Debug/bin/net8.0",
-      "env": {
-        "ASPNETCORE_HTTP_PORT": "8080",
-        "ASPNETCORE_HTTPS_PORT": "8443",
-        "ASPNETCORE_ENVIRONMENT": "${input:dataStreamWebAppServerEnvironment}",
-        "ASPNETCORE_HOSTINGSTARTUPASSEMBLIES": "Microsoft.AspNetCore.SpaProxy",
-      }
-    },
-    {
-      "name": "Bts.DataStreamWebApp.Server (Skip Build)",
-      "type": "coreclr",
-      "request": "launch",
-      "program": "${workspaceFolder}/build/Debug/bin/net8.0/Bts.DataStreamWebApp.Server.dll",
-      "cwd": "${workspaceFolder}/build/Debug/bin/net8.0",
-      "env": {
-        "ASPNETCORE_HTTP_PORT": "8080",
-        "ASPNETCORE_HTTPS_PORT": "8443",
-        "ASPNETCORE_ENVIRONMENT": "${input:dataStreamWebAppServerEnvironment}",
-        "ASPNETCORE_HOSTINGSTARTUPASSEMBLIES": "Microsoft.AspNetCore.SpaProxy",
-      }
-    },
     {
       "name": "DataStreamPublisher",
       "type": "coreclr",
@@ -165,12 +146,25 @@
       "program": "",
       "windows": {
         "program": "DataStreamQueryAnalyzer.exe",
-        "args": [
-          "--config",
-          "${workspaceFolder}/src/DataStreamSuite/DataStreamQueryAnalyzer/Configurations/DataStreamQueryAnalyzer.json"
-        ],
         "cwd": "build/Debug/bin/net8.0-windows",
         "env": {
+          "DOTNET_ENVIRONMENT": "${input:dataStreamQueryAnalyzerEnvironment}",
+          "PYTHON_EXE": "${workspaceFolder}/.venv/Scripts/python.exe"
+        },
+      },
+      "stopAtEntry": false,
+      "console": "integratedTerminal",
+    },
+    {
+      "name": "DataStreamQueryAnalyzer (Skip Build)",
+      "type": "coreclr",
+      "request": "launch",
+      "program": "",
+      "windows": {
+        "program": "DataStreamQueryAnalyzer.exe",
+        "cwd": "build/Debug/bin/net8.0-windows",
+        "env": {
+          "DOTNET_ENVIRONMENT": "${input:dataStreamQueryAnalyzerEnvironment}",
           "PYTHON_EXE": "${workspaceFolder}/.venv/Scripts/python.exe"
         },
       },
@@ -189,6 +183,10 @@
       "windows": {
         "program": "PubSubServer.exe",
       },
+      "args": [
+        "--config",
+        "${workspaceFolder}/src/DataStreamSuite/PubSubServer/Configurations/${input:pubSubServerConfig}.json"
+      ],
       "cwd": "build/Debug/bin/net8.0",
       "env": {
         "DOTNET_ENVIRONMENT": "${input:pubSubServerEnvironment}",
@@ -207,7 +205,7 @@
       "windows": {
         "program": "PubSubServer.exe",
       },
-      "cwd": "build/Debug/bin/net7.0",
+      "cwd": "build/Debug/bin/net8.0",
       "env": {
         "DOTNET_ENVIRONMENT": "${input:pubSubServerEnvironment}",
       },
@@ -223,6 +221,9 @@
         "preLaunchTask": "Build Spark (Debug)",
         "program": "Spark.exe",
         "cwd": "build/Debug/bin/net8.0-windows",
+        "env": {
+          "DOTNET_ENVIRONMENT": "${input:sparkEnvironment}",
+        }
       },
       "stopAtEntry": false,
       "console": "integratedTerminal"
@@ -235,6 +236,9 @@
       "windows": {
         "program": "Spark.exe",
         "cwd": "build/Debug/bin/net8.0-windows",
+        "env": {
+          "DOTNET_ENVIRONMENT": "${input:sparkEnvironment}",
+        }
       },
       "stopAtEntry": false,
       "console": "integratedTerminal"
@@ -348,41 +352,78 @@
   ],
   "inputs": [
     {
-      "id": "dataStreamPublisherEnvironment",
+      "id": "apiServerEnvironment",
       "type": "pickString",
-      "description": "Publisher Configuration",
+      "description": "Api Server Configuration",
       "options": [
         "Development",
-        "MarketData",
-        "Production",
-        "ProductionAws",
-        "Staging",
-      ],
-      "default": "MarketDataPublisher",
-    },
-    {
-      "id": "pubSubServerEnvironment",
-      "type": "pickString",
-      "description": "PubSub Configuration",
-      "options": [
-        "Development",
-        "EchoServer",
-        "Production",
-        "ProductionAws",
+        "Local",
         "Staging",
       ],
       "default": "Development",
     },
     {
-      "id": "dataStreamWebAppServerEnvironment",
+      "id": "dataStreamPublisherConfig",
       "type": "pickString",
-      "description": "DataStream WebApp Environment",
+      "description": "Publisher Configuration",
+      "options": [
+        "MarketDataPublisher",
+        "SimpleEchoClient",
+        "SimpleTestDataPublisher",
+        "TestDataPublisher",
+      ],
+      "default": "MarketDataPublisher",
+    },
+    {
+      "id": "dataStreamPublisherEnvironment",
+      "type": "pickString",
+      "description": "DataStreamPublisher Configuration",
+      "options": [
+        "Local",
+        "ReplayMarketData",
+      ],
+      "default": "Local",
+    },
+    {
+      "id": "pubSubServerConfig",
+      "type": "pickString",
+      "description": "PubSub Configuration",
+      "options": [
+        "PubSubServer",
+        "SimpleEchoClient",
+      ],
+      "default": "PubSubServer",
+    },
+    {
+      "id": "pubSubServerEnvironment",
+      "type": "pickString",
+      "description": "Api Server Configuration",
       "options": [
         "Development",
-        "Production",
-        "ProductionAws",
+        "Local",
         "Staging",
-      ]
+      ],
+      "default": "Development",
+    },
+    {
+      "id": "dataStreamQueryAnalyzerEnvironment",
+      "type": "pickString",
+      "description": "DataStreamQueryAnalyzer Configuration",
+      "options": [
+        "Development",
+        "Local",
+      ],
+      "default": "Development",
+    },
+    {
+      "id": "sparkEnvironment",
+      "type": "pickString",
+      "description": "Spark Configuration",
+      "options": [
+        "Development",
+        "Local",
+      ],
+      "default": "Development",
     },
     {
       "id": "sparkConfigServer",
@@ -400,15 +441,5 @@
       "type": "promptString",
       "description": "Test to run",
     },
-    {
-      "id": "isSparkApiEnabled",
-      "type": "pickString",
-      "description": "Enable Spark Api (connect to sparkd)",
-      "options": [
-        "True",
-        "False",
-      ],
-      "default": "True",
-    }
   ]
 }

.vscode/python.env

@@ -1 +1 @@
-PYTHONPATH=/workspaces/spark/build/sparkd-debug-clang/src/lib_python_interop;/workspaces/spark/src/lib_python_interop/python_interop_plugins
+PYTHONPATH=/workspaces/spark/build/sparkd-debug-clang/src/lib_python_interop;/workspaces/spark/src/lib_python_interop/python_interop_plugins;/workspaces/spark/tools/py

.vscode/tasks.json

@@ -172,66 +172,6 @@
         "color": "terminal.ansiCyan"
       },
     },
-    {
-      "label": "Build DataStream WebApp",
-      "group": "build",
-      "type": "process",
-      "command": "",
-      "linux": {
-        "command": "tools/build_data_stream_web_app.sh",
-      },
-      "windows": {
-        "command": "tools/build_data_stream_web_app.cmd",
-      },
-      "args": [
-        "--configuration",
-        "${input:buildConfiguration}",
-      ],
-      "icon": {
-        "id": "terminal",
-        "color": "terminal.ansiCyan"
-      },
-    },
-    {
-      "label": "Build DataStream WebApp (Debug)",
-      "group": "build",
-      "type": "process",
-      "command": "",
-      "linux": {
-        "command": "tools/build_data_stream_web_app.sh",
-      },
-      "windows": {
-        "command": "tools/build_data_stream_web_app.cmd",
-      },
-      "args": [
-        "--configuration",
-        "Debug",
-      ],
-      "icon": {
-        "id": "terminal",
-        "color": "terminal.ansiCyan"
-      },
-    },
-    {
-      "label": "Build DataStream WebApp (Release)",
-      "group": "build",
-      "type": "process",
-      "command": "",
-      "linux": {
-        "command": "tools/build_data_stream_web_app.sh",
-      },
-      "windows": {
-        "command": "tools/build_data_stream_web_app.cmd",
-      },
-      "args": [
-        "--configuration",
-        "Release",
-      ],
-      "icon": {
-        "id": "terminal",
-        "color": "terminal.ansiCyan"
-      },
-    },
     {
       "label": "Build Spark",
       "group": "build",
@@ -414,22 +354,6 @@
         "color": "terminal.ansiCyan"
       },
     },
-    {
-      "label": "Publish DataStreamWebApp",
-      "group": "build",
-      "type": "process",
-      "command": "",
-      "linux": {
-        "command": "tools/publish_data_stream_web_app.sh",
-      },
-      "windows": {
-        "command": "tools/publish_data_stream_web_app.cmd",
-      },
-      "icon": {
-        "id": "terminal",
-        "color": "terminal.ansiCyan"
-      },
-    },
     {
       "label": "Edit DataStream Config",
       "type": "process",
@@ -478,6 +402,129 @@
         "id": "wrench"
       },
     },
+    {
+      "label": "Run Api Server",
+      "dependsOn": "Build Api Server",
+      "type": "process",
+      "command": "",
+      "windows": {
+        "command": "spark_api_server.exe",
+      },
+      "linux": {
+        "command": "spark_api_server",
+      },
+      "args": [
+        "run",
+      ],
+      "options": {
+        "cwd": "${workspaceFolder}/build/${input:buildConfiguration}/bin/net8.0",
+        "env": {
+          "DOTNET_ENVIRONMENT": "${input:apiServerEnvironment}",
+        }
+      },
+      "icon": {
+        "id": "run",
+        "color": "terminal.ansiGreen",
+      },
+    },
+    {
+      "label": "Run Api Server (Debug)",
+      "dependsOn": "Build Api Server (Debug)",
+      "type": "process",
+      "command": "",
+      "windows": {
+        "command": "spark_api_server.exe",
+      },
+      "linux": {
+        "command": "spark_api_server",
+      },
+      "args": [
+        "run",
+      ],
+      "options": {
+        "cwd": "${workspaceFolder}/build/Debug/bin/net8.0",
+        "env": {
+          "DOTNET_ENVIRONMENT": "${input:apiServerEnvironment}",
+        }
+      },
+      "icon": {
+        "id": "run",
+        "color": "terminal.ansiGreen",
+      },
+    },
+    {
+      "label": "Run Api Server (Release)",
+      "dependsOn": "Build Api Server (Release)",
+      "type": "process",
+      "command": "",
+      "windows": {
+        "command": "spark_api_server.exe",
+      },
+      "linux": {
+        "command": "spark_api_server",
+      },
+      "args": [
+        "run",
+      ],
+      "options": {
+        "cwd": "${workspaceFolder}/build/Release/bin/net8.0",
+        "env": {
+          "DOTNET_ENVIRONMENT": "${input:apiServerEnvironment}",
+        }
+      },
+      "icon": {
+        "id": "run",
+        "color": "terminal.ansiGreen",
+      },
+    },
+    {
+      "label": "Run Api Server (Debug | Skip Build)",
+      "type": "process",
+      "command": "",
+      "windows": {
+        "command": "spark_api_server.exe",
+      },
+      "linux": {
+        "command": "spark_api_server",
+      },
+      "args": [
+        "run",
+      ],
+      "options": {
+        "cwd": "${workspaceFolder}/build/Debug/bin/net8.0",
+        "env": {
+          "DOTNET_ENVIRONMENT": "${input:apiServerEnvironment}",
+        }
+      },
+      "icon": {
+        "id": "run",
+        "color": "terminal.ansiGreen",
+      },
+    },
+    {
+      "label": "Run Api Server (Release | Skip Build)",
+      "type": "process",
+      "command": "",
+      "windows": {
+        "command": "spark_api_server.exe",
+      },
+      "linux": {
+        "command": "spark_api_server",
+      },
+      "args": [
+        "run",
+      ],
+      "options": {
+        "cwd": "${workspaceFolder}/build/Release/bin/net8.0",
+        "env": {
+          "DOTNET_ENVIRONMENT": "${input:apiServerEnvironment}",
+        }
+      },
+      "icon": {
+        "id": "run",
+        "color": "terminal.ansiGreen",
+      },
+    },
     {
       "label": "Run DataStreamQueryAnalyzer",
       "dependsOn": "Build DataStreamQueryAnalyzer",
@@ -549,12 +596,11 @@
       "windows": {
         "command": "DataStreamPublisher.exe",
       },
-      "args": [
-        "--config",
-        "${workspaceFolder}/src/DataStreamSuite/DataStreamPublisher/Configurations/${input:dataStreamPublisherConfig}.json",
-      ],
       "options": {
-        "cwd": "${workspaceFolder}/build/${input:buildConfiguration}/bin/net8.0"
+        "cwd": "${workspaceFolder}/build/${input:buildConfiguration}/bin/net8.0",
+        "env": {
+          "DOTNET_ENVIRONMENT": "${input:dataStreamPublisherEnvironment}",
+        },
       },
       "icon": {
         "id": "run",
@@ -572,12 +618,11 @@
       "windows": {
         "command": "DataStreamPublisher.exe",
       },
-      "args": [
-        "--config",
-        "${workspaceFolder}/src/DataStreamSuite/DataStreamPublisher/Configurations/${input:dataStreamPublisherConfig}.json",
-      ],
       "options": {
-        "cwd": "${workspaceFolder}/build/Debug/bin/net8.0"
+        "cwd": "${workspaceFolder}/build/Debug/bin/net8.0",
+        "env": {
+          "DOTNET_ENVIRONMENT": "${input:dataStreamPublisherEnvironment}",
+        },
       },
       "icon": {
         "id": "run",
@@ -595,95 +640,11 @@
       "windows": {
         "command": "DataStreamPublisher.exe",
       },
-      "args": [
-        "--config",
-        "${workspaceFolder}/src/DataStreamSuite/DataStreamPublisher/Configurations/${input:dataStreamPublisherConfig}.json",
-      ],
       "options": {
-        "cwd": "${workspaceFolder}/build/Release/bin/net8.0"
-      },
-      "icon": {
-        "id": "run",
-        "color": "terminal.ansiGreen",
-      },
-    },
-    {
-      "label": "Run DataStream WebApp (Debug)",
-      "dependsOn": "Build DataStream WebApp (Debug)",
-      "command": "",
-      "linux": {
-        "command": "Bts.DataStreamWebApp.Server",
-      },
-      "windows": {
-        "command": "Bts.DataStreamWebApp.Server.exe",
-      },
-      "args": [
-        "--config",
-        "${workspaceFolder}/src/DataStreamSuite/Bts.DataStreamWebApp.Server/Configurations/DevServer.json",
-      ],
-      "options": {
-        "cwd": "${workspaceFolder}/build/Debug/bin/net7.0",
+        "cwd": "${workspaceFolder}/build/Release/bin/net8.0",
         "env": {
-          "ASPNETCORE_HTTP_PORT": "8080",
-          "ASPNETCORE_HTTPS_PORT": "8443",
-          "ASPNETCORE_ENVIRONMENT": "Development",
-          "ASPNETCORE_HOSTINGSTARTUPASSEMBLIES": "Microsoft.AspNetCore.SpaProxy",
-        }
-      },
-      "icon": {
-        "id": "run",
-        "color": "terminal.ansiGreen",
-      },
-    },
-    {
-      "label": "Run DataStream WebApp (Release)",
-      "dependsOn": "Build DataStream WebApp (Release)",
-      "command": "",
-      "linux": {
-        "command": "Bts.DataStreamWebApp.Server",
-      },
-      "windows": {
-        "command": "Bts.DataStreamWebApp.Server.exe",
-      },
-      "args": [
-        "--config",
-        "${workspaceFolder}/src/DataStreamSuite/Bts.DataStreamWebApp.Server/Configurations/DevServer.json",
-      ],
-      "options": {
-        "cwd": "${workspaceFolder}/build/Release/bin/net7.0",
-        "env": {
-          "ASPNETCORE_HTTP_PORT": "8080",
-          "ASPNETCORE_HTTPS_PORT": "8443",
-          "ASPNETCORE_ENVIRONMENT": "Production",
-          "ASPNETCORE_HOSTINGSTARTUPASSEMBLIES": "Microsoft.AspNetCore.SpaProxy",
-        }
-      },
-      "icon": {
-        "id": "run",
-        "color": "terminal.ansiGreen",
-      },
-    },
-    {
-      "label": "Run DataStream WebApp (Release | Skip Build)",
-      "command": "",
-      "linux": {
-        "command": "Bts.DataStreamWebApp.Server",
-      },
-      "windows": {
-        "command": "Bts.DataStreamWebApp.Server.exe",
-      },
-      "args": [
-        "--config",
-        "${workspaceFolder}/src/DataStreamSuite/Bts.DataStreamWebApp.Server/Configurations/DevServer.json",
-      ],
-      "options": {
-        "cwd": "${workspaceFolder}/build/Release/bin/net7.0",
-        "env": {
-          "ASPNETCORE_HTTP_PORT": "8080",
-          "ASPNETCORE_HTTPS_PORT": "8443",
-          "ASPNETCORE_ENVIRONMENT": "Production",
-          "ASPNETCORE_HOSTINGSTARTUPASSEMBLIES": "Microsoft.AspNetCore.SpaProxy",
-        }
+          "DOTNET_ENVIRONMENT": "${input:dataStreamPublisherEnvironment}",
+        },
       },
       "icon": {
         "id": "run",
@@ -701,12 +662,11 @@
       "windows": {
         "command": "PubSubServer.exe",
       },
-      "args": [
-        "--config",
-        "${workspaceFolder}/src/DataStreamSuite/PubSubServer/Configurations/${input:pubSubServerConfig}.json",
-      ],
       "options": {
-        "cwd": "${workspaceFolder}/build/${input:buildConfiguration}/bin/net8.0"
+        "cwd": "${workspaceFolder}/build/${input:buildConfiguration}/bin/net8.0",
+        "env": {
+          "DOTNET_ENVIRONMENT": "${input:pubSubServerEnvironment}",
+        },
       },
       "icon": {
         "id": "run",
@@ -724,12 +684,11 @@
       "windows": {
         "command": "PubSubServer.exe",
       },
-      "args": [
-        "--config",
-        "${workspaceFolder}/src/DataStreamSuite/PubSubServer/Configurations/${input:pubSubServerConfig}.json",
-      ],
       "options": {
-        "cwd": "${workspaceFolder}/build/Debug/bin/net8.0"
+        "cwd": "${workspaceFolder}/build/Debug/bin/net8.0",
+        "env": {
+          "DOTNET_ENVIRONMENT": "${input:pubSubServerEnvironment}",
+        },
       },
       "icon": {
         "id": "run",
@@ -746,12 +705,11 @@
       "windows": {
         "command": "PubSubServer.exe",
       },
-      "args": [
-        "--config",
-        "${workspaceFolder}/src/DataStreamSuite/PubSubServer/Configurations/${input:pubSubServerConfig}.json",
-      ],
       "options": {
-        "cwd": "${workspaceFolder}/build/Debug/bin/net8.0"
+        "cwd": "${workspaceFolder}/build/Debug/bin/net8.0",
+        "env": {
+          "DOTNET_ENVIRONMENT": "${input:pubSubServerEnvironment}",
+        },
       },
       "icon": {
         "id": "run",
@@ -769,12 +727,11 @@
       "windows": {
         "command": "PubSubServer.exe",
       },
-      "args": [
-        "--config",
-        "${workspaceFolder}/src/DataStreamSuite/PubSubServer/Configurations/${input:pubSubServerConfig}.json",
-      ],
       "options": {
-        "cwd": "${workspaceFolder}/build/Release/bin/net8.0"
+        "cwd": "${workspaceFolder}/build/Release/bin/net8.0",
+        "env": {
+          "DOTNET_ENVIRONMENT": "${input:pubSubServerEnvironment}",
+        },
       },
       "icon": {
         "id": "run",
@@ -791,12 +748,11 @@
       "windows": {
         "command": "PubSubServer.exe",
       },
-      "args": [
-        "--config",
-        "${workspaceFolder}/src/DataStreamSuite/PubSubServer/Configurations/${input:pubSubServerConfig}.json",
-      ],
       "options": {
-        "cwd": "${workspaceFolder}/build/Release/bin/net8.0"
+        "cwd": "${workspaceFolder}/build/Release/bin/net8.0",
+        "env": {
+          "DOTNET_ENVIRONMENT": "${input:pubSubServerEnvironment}",
+        },
       },
       "icon": {
         "id": "run",
@@ -1062,6 +1018,17 @@
     },
   ],
   "inputs": [
+    {
+      "id": "apiServerEnvironment",
+      "type": "pickString",
+      "description": "Api Server Configuration",
+      "options": [
+        "Development",
+        "Local",
+        "Staging",
+      ],
+      "default": "Development",
+    },
     {
       "id": "buildConfiguration",
       "type": "pickString",
@@ -1095,26 +1062,31 @@
       "default": "sparkd-debug-clang",
     },
     {
-      "id": "dataStreamPublisherConfig",
+      "id": "dataStreamPublisherEnvironment",
       "type": "pickString",
       "description": "Publisher Configuration",
       "options": [
-        "MarketDataPublisher",
-        "SimpleEchoClient",
-        "SimpleTestDataPublisher",
-        "TestDataPublisher",
+        "Development",
+        "MarketData",
+        "Production",
+        "ProductionAws",
+        "Staging",
       ],
       "default": "MarketDataPublisher",
     },
     {
-      "id": "pubSubServerConfig",
+      "id": "pubSubServerEnvironment",
       "type": "pickString",
       "description": "PubSub Configuration",
       "options": [
-        "PubSubServer",
-        "SimpleEchoClient",
+        "Development",
+        "EchoServer",
+        "Local",
+        "Production",
+        "ProductionAws",
+        "Staging",
       ],
-      "default": "PubSubServer",
+      "default": "Development",
     },
     {
       "id": "sparkConfigQuery",

CMakePresets.json

@@ -180,7 +180,7 @@
           "value": "ON"
         },
         "CONFIG_DEFAULTS_PATH": "spark/config/profiles/default/*.conf",
-        "DOTNET_BINARY_DIR": "${sourceDir}/build/Spark.Api.Server/$<CONFIG>/bin/net8.0",
+        "DOTNET_BINARY_DIR": "${sourceDir}/build/$<CONFIG>/bin/net8.0",
         "EXPORT_MD_CONSUMER": {
           "type": "BOOL",
           "value": "ON"
@@ -252,7 +252,7 @@
         "CMAKE_CXX_FLAGS_DEBUG": "-g -fPIC -Wall -pedantic -Wextra -Werror -Wno-missing-braces -Wno-ignored-qualifiers --std=c++20 -fdiagnostics-color -DDEBUG",
         "CMAKE_CXX_FLAGS_RELEASE": "-fPIC -Wall -pedantic -Wextra -Werror -Wno-missing-braces -Wno-ignored-qualifiers --std=c++20 -fdiagnostics-color -fno-omit-frame-pointer -DNDEBUG -O3",
         "CONFIG_DEFAULTS_PATH": "spark/config/profiles/default/*.conf;spark/config/profiles/data_stream/*.conf",
-        "DOTNET_BINARY_DIR": "${sourceDir}/build/$<CONFIG>/bin/net7.0",
+        "DOTNET_BINARY_DIR": "${sourceDir}/build/$<CONFIG>/bin/net8.0",
         "EXPORT_MD_CONSUMER": {
           "type": "BOOL",
           "value": "ON"

Spark.Api.Server.sln

@@ -42,6 +42,10 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Bts.ColumnExpressions", "sr
 EndProject
 Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "DataStream", "src\DataStreamSuite\DataStream\DataStream.csproj", "{39AEA075-6457-46C0-8821-1158BE6EFA57}"
 EndProject
+Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "sdk", "sdk", "{56ACFCD1-60C7-4E2C-B90D-B1F95ECE33A1}"
+EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Bts.Spark.Sdk", "src\sdk\Bts.Spark.Sdk\Bts.Spark.Sdk.csproj", "{ADA9E802-8EFA-4E72-8E9A-6AA1E4181184}"
+EndProject
 Global
 	GlobalSection(SolutionConfigurationPlatforms) = preSolution
 		Debug|x64 = Debug|x64
@@ -194,6 +198,14 @@ Global
 		{6BD3F6D8-B6FC-44E8-B8FE-06352F638B9B}.Release|x64.Build.0 = Release|Any CPU
 		{6BD3F6D8-B6FC-44E8-B8FE-06352F638B9B}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
 		{6BD3F6D8-B6FC-44E8-B8FE-06352F638B9B}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
+		{ADA9E802-8EFA-4E72-8E9A-6AA1E4181184}.Debug|x64.ActiveCfg = Debug|Any CPU
+		{ADA9E802-8EFA-4E72-8E9A-6AA1E4181184}.Debug|x64.Build.0 = Debug|Any CPU
+		{ADA9E802-8EFA-4E72-8E9A-6AA1E4181184}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
+		{ADA9E802-8EFA-4E72-8E9A-6AA1E4181184}.MinSizeRel|x64.Build.0 = Debug|Any CPU
+		{ADA9E802-8EFA-4E72-8E9A-6AA1E4181184}.Release|x64.ActiveCfg = Release|Any CPU
+		{ADA9E802-8EFA-4E72-8E9A-6AA1E4181184}.Release|x64.Build.0 = Release|Any CPU
+		{ADA9E802-8EFA-4E72-8E9A-6AA1E4181184}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
+		{ADA9E802-8EFA-4E72-8E9A-6AA1E4181184}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
 	EndGlobalSection
 	GlobalSection(SolutionProperties) = preSolution
 		HideSolutionNode = FALSE
@@ -218,5 +230,7 @@ Global
 		{39AEA075-6457-46C0-8821-1158BE6EFA57} = {C09ED521-D2DF-40BC-A76F-EB3CB870B2F6}
 		{9F6D7DE8-185B-4C1D-B809-3314D092DF28} = {C09ED521-D2DF-40BC-A76F-EB3CB870B2F6}
 		{6BD3F6D8-B6FC-44E8-B8FE-06352F638B9B} = {C09ED521-D2DF-40BC-A76F-EB3CB870B2F6}
+		{56ACFCD1-60C7-4E2C-B90D-B1F95ECE33A1} = {267F81EF-A8AB-49FD-9F56-D7EC3C1F7C7C}
+		{ADA9E802-8EFA-4E72-8E9A-6AA1E4181184} = {56ACFCD1-60C7-4E2C-B90D-B1F95ECE33A1}
 	EndGlobalSection
 EndGlobal

compose.yaml

@@ -1,7 +1,7 @@
 services:
   sparkd:
     volumes:
-      - ${SPARKD__DATA_DIR:-sparkd_data}:/home/bts/sparkdata/data
+      - ${SPARKD__DATA_DIR:-sparkd_data}:/home/bts/.local/share/bts/sparkd/data
     build:
       dockerfile: src/app_sparkd/Dockerfile
       args:
@@ -12,19 +12,19 @@ services:
     shm_size: "${SPARKD__SHM_SIZE:-512mb}"
     command: >
       sh -c "
-        echo bts | sudo -S chown -R bts:bts /home/bts/sparkdata/data
-        if [ -e "/home/bts/sparkdata/data/logging.sparkd/log" ]; then
-          mv "/home/bts/sparkdata/data/logging.sparkd/log" "/home/bts/sparkdata/data/logging.sparkd/log_$(date '+%Y%m%d-%H%M')"
+        echo bts | sudo -S chown -R bts:bts /home/bts/.local/share/bts/sparkd/data
+        if [ -e "/home/bts/.local/share/bts/sparkd/data/logging.sparkd/log" ]; then
+          mv "/home/bts/.local/share/bts/sparkd/data/logging.sparkd/log" "/home/bts/.local/share/bts/sparkd/data/logging.sparkd/log_$(date '+%Y%m%d-%H%M')"
         fi
-        if [ -e "/home/bts/sparkdata/conf/.env.local" ]; then
-          echo 'Sourcing /home/bts/sparkdata/conf/.env.local'
+        if [ -e "/home/bts/.local/share/bts/sparkd/conf/.env.local" ]; then
+          echo 'Sourcing /home/bts/.local/share/bts/sparkd/conf/.env.local'
           set -a
-          . /home/bts/sparkdata/conf/.env.local
+          . /home/bts/.local/share/bts/sparkd/conf/.env.local
           set +a
         fi
-        if [ -d /home/bts/sparkdata/conf/plugins ]; then
-          echo 'Copying /home/bts/sparkdata/conf/plugins to runtime dir (/home/bts/bin)'
-          cp -r /home/bts/sparkdata/conf/plugins /home/bts/bin/
+        if [ -d /home/bts/.local/share/bts/sparkd/conf/plugins ]; then
+          echo 'Copying /home/bts/.local/share/bts/sparkd/conf/plugins to runtime dir (/home/bts/bin)'
+          cp -r /home/bts/.local/share/bts/sparkd/conf/plugins /home/bts/bin/
         fi
         ./sparkd run"
     env_file:
@@ -41,8 +41,8 @@ services:
 
   sparkd-symbols:
     volumes:
-      - ${SPARKD__DATA_DIR:-sparkd_data}:/home/bts/sparkdata/data
-      - ${SPARKD__RUNTIME_CONF_DIR:-sparkd_conf}:/home/bts/sparkdata/conf
+      - ${SPARKD__DATA_DIR:-sparkd_data}:/home/bts/.local/share/bts/sparkd/data
+      - ${SPARKD__RUNTIME_CONF_DIR:-sparkd_conf}:/home/bts/.local/share/bts/sparkd/conf
     build:
       dockerfile: src/app_sparkd/Dockerfile
       args:
@@ -53,21 +53,21 @@ services:
     shm_size: "${SPARKD__SHM_SIZE:-512mb}"
     command: >
       sh -c "
-        echo bts | sudo -S chown -R bts:bts /home/bts/sparkdata
-        if [ -e "/home/bts/sparkdata/data/logging.sparkd/log" ]; then
-          mv "/home/bts/sparkdata/data/logging.sparkd/log" "/home/bts/sparkdata/data/logging.sparkd/log_$(date '+%Y%m%d-%H%M')"
+        echo bts | sudo -S chown -R bts:bts /home/bts/.local/share/bts/sparkd
+        if [ -e "/home/bts/.local/share/bts/sparkd/data/logging.sparkd/log" ]; then
+          mv "/home/bts/.local/share/bts/sparkd/data/logging.sparkd/log" "/home/bts/.local/share/bts/sparkd/data/logging.sparkd/log_$(date '+%Y%m%d-%H%M')"
         fi
-        ls -alh /home/bts/sparkdata/conf/
-        cat /home/bts/sparkdata/conf/.env.local
-        if [ -e "/home/bts/sparkdata/conf/.env.local" ]; then
-          echo 'Sourcing /home/bts/sparkdata/conf/.env.local'
+        ls -alh /home/bts/.local/share/bts/sparkd/conf/
+        cat /home/bts/.local/share/bts/sparkd/conf/.env.local
+        if [ -e "/home/bts/.local/share/bts/sparkd/conf/.env.local" ]; then
+          echo 'Sourcing /home/bts/.local/share/bts/sparkd/conf/.env.local'
           set -a
-          . /home/bts/sparkdata/conf/.env.local
+          . /home/bts/.local/share/bts/sparkd/conf/.env.local
           set +a
         fi
-        if [ -d /home/bts/sparkdata/conf/plugins ]; then
-          echo 'Copying /home/bts/sparkdata/conf/plugins to runtime dir (/home/bts/bin)'
-          cp -r /home/bts/sparkdata/conf/plugins /home/bts/bin/
+        if [ -d /home/bts/.local/share/bts/sparkd/conf/plugins ]; then
+          echo 'Copying /home/bts/.local/share/bts/sparkd/conf/plugins to runtime dir (/home/bts/bin)'
+          cp -r /home/bts/.local/share/bts/sparkd/conf/plugins /home/bts/bin/
         fi
         ./sparkd run"
     env_file:
@@ -91,10 +91,10 @@ services:
       - type: volume
         read_only: true
         source: sparkd_data
-        target: /home/bts/sparkdata/data/logging.sparkd
+        target: /home/bts/.local/share/bts/sparkd/data/logging.sparkd
         volume:
-          subpath: logging.sparkd
-    command: ["tail", "-f", "/home/bts/sparkdata/data/logging.sparkd/log"]
+          subpath: logging
+    command: ["tail", "-f", "/home/bts/.local/share/bts/sparkd/data/logging.sparkd/log"]
 
   sparkd-ping-test:
     depends_on:
@@ -110,43 +110,69 @@ services:
     command: [ "ping", "sparkd" ]
 
   pubsub:
-    environment:
-      - DOTNET_ENVIRONMENT=Staging
+    volumes:
+      - ${PUBSUB__DATA_DIR:-pubsub_data}:/home/bts/.local/share/bts/PubSubServer/data
     build:
       dockerfile: src/DataStreamSuite/PubSubServer/Dockerfile
       args:
-        - SOURCE_DIR=build/Release/bin/net8.0/linux-x64/publish
-        - CONF_FILE=src/DataStreamSuite/PubSubServer/spark.conf
+        - SOURCE_DIR=${PUBSUB__SOURCE_DIR}
+        - CONFIG_DIR=${PUBSUB__CONFIG_DIR}
     ports:
       - 22019:22019
-    command: [ "./PubSubServer" ]
+    command: >
+      sh -c "
+        echo bts | sudo -S chown -R bts:bts /home/bts/.local/share/bts/PubSubServer
+        if [ -e "/home/bts/.local/share/bts/PubSubServer/data/logging.PubSubServer/log" ]; then
+          mv "/home/bts/.local/share/bts/PubSubServer/data/logging.PubSubServer/log" "/home/bts/.local/share/bts/PubSubServer/data/logging.PubSubServer/log_$(date '+%Y%m%d-%H%M')"
+        fi
+        if [ -e "/home/bts/.local/share/bts/PubSubServer/conf/.env" ]; then
+          echo 'Sourcing /home/bts/.local/share/bts/PubSubServer/conf/.env'
+          set -a
+          . /home/bts/.local/share/bts/PubSubServer/conf/.env
+          set +a
+        fi
+        if [ -e "/home/bts/.local/share/bts/PubSubServer/conf/.env.local" ]; then
+          echo 'Sourcing /home/bts/.local/share/bts/PubSubServer/conf/.env.local'
+          set -a
+          . /home/bts/.local/share/bts/PubSubServer/conf/.env.local
+          set +a
+        fi
+        ./PubSubServer"
+    env_file:
+      - .env
+      - ${PUBSUB__CONFIG_DIR}/.env
 
   md-publisher:
+    volumes:
+      - ${MD_PUBLISHER__DATA_DIR:-md_publisher_data}:/home/bts/.local/share/bts/DataStreamPublisher/data
+      - historic_md:/home/bts/.local/share/bts/databento
     build:
       dockerfile: src/DataStreamSuite/DataStreamPublisher/Dockerfile
       args:
-        - SOURCE_DIR=build/Release/bin/net8.0/linux-x64/publish
-        - CONF_FILE=src/DataStreamSuite/DataStreamPublisher/spark.conf
-    depends_on:
-      - pubsub
-      - sparkd
-    command: [ "dockerize", "-wait", "tcp://pubsub:22019", "-wait", "tcp://sparkd:8000", "-timeout", "10s", "./DataStreamPublisher" ]
-    environment:
-      - DOTNET_ENVIRONMENT=MarketData
-
-  webapp:
-    ports:
-      - 8080:8080
-      - 8443:8443
-    build:
-      dockerfile: src/DataStreamSuite/Bts.DataStreamWebApp.Server/Dockerfile
-      args:
-        - SOURCE_DIR=build/Release/bin/net8.0/linux-x64/publish
-        - WEB_ROOT=build/Bts.DataStreamWebApp.Client/production/dist
-        - CONF_FILE=src/DataStreamSuite/Bts.DataStreamWebApp.Server/spark.conf
-    command: [ "./Bts.DataStreamWebApp.Server" ]
-    environment:
-      - ASPNETCORE_ENVIRONMENT=Staging
+        - SOURCE_DIR=${MD_PUBLISHER__SOURCE_DIR}
+        - CONFIG_DIR=${MD_PUBLISHER__CONFIG_DIR}
+    command: >
+      sh -c "
+        echo bts | sudo -S chown -R bts:bts /home/bts/.local/share/bts/DataStreamPublisher
+        if [ -e "/home/bts/.local/share/bts/DataStreamPublisher/data/logging.DataStreamPublisher/log" ]; then
+          mv "/home/bts/.local/share/bts/DataStreamPublisher/data/logging.DataStreamPublisher/log" "/home/bts/.local/share/bts/DataStreamPublisher/data/logging.DataStreamPublisher/log_$(date '+%Y%m%d-%H%M')"
+        fi
+        if [ -e "/home/bts/.local/share/bts/DataStreamPublisher/conf/.env" ]; then
+          echo 'Sourcing /home/bts/.local/share/bts/DataStreamPublisher/conf/.env'
+          set -a
+          . /home/bts/.local/share/bts/DataStreamPublisher/conf/.env
+          set +a
+        fi
+        if [ -e "/home/bts/.local/share/bts/DataStreamPublisher/conf/.env.local" ]; then
+          echo 'Sourcing /home/bts/.local/share/bts/DataStreamPublisher/conf/.env.local'
+          set -a
+          . /home/bts/.local/share/bts/DataStreamPublisher/conf/.env.local
+          set +a
+        fi
+      dockerize -wait tcp://pubsub:22019 -timeout 10s ./DataStreamPublisher"
+    env_file:
+      - .env
+      - ${MD_PUBLISHER__CONFIG_DIR}/.env
 
   api-server:
     depends_on:
@@ -163,3 +189,9 @@ services:
 volumes:
   sparkd_data:
     name: "sparkd_${SPARKD__CONFIG_PORT:-8000}"
+  pubsub_data:
+    name: "pubsub_22019"
+  md_publisher_data:
+    name: "md_publisher_22019"
+  historic_md:
+    name: "historic_md"

docker/ubuntu-22.04/dotnet-7/Dockerfile

@@ -1,39 +0,0 @@
-FROM ubuntu:22.04
-
-ARG DEBIAN_FRONTEND=noninteractive
-
-RUN apt-get update \
-  && apt-get install -y pkg-config \
-  && apt-get install -y locales \
-  && apt-get install -y ca-certificates \
-  && apt-get install -y libssh2-1-dev \
-  && apt-get install -y software-properties-common \
-  && apt-get install -y sudo
-
-# GCC
-ARG gcc_major=13
-RUN add-apt-repository ppa:ubuntu-toolchain-r/test \
-  && apt-get update \
-  && apt-get install -y g++-$gcc_major
-
-RUN apt-get update \
-  && apt-get install -y dotnet7
-
-RUN locale-gen en_US.UTF-8
-
-ARG docker_user="bts"
-ARG docker_uid="1000"
-ARG docker_pass="bts"
-
-RUN \
-  groupadd -g "$docker_uid" "$docker_user" \
-  && useradd \
-  -m \
-  -u "$docker_uid" \
-  -g "$docker_uid" \
-  -G sudo \
-  --password "$(openssl passwd -1 "$docker_pass")" \
-  "$docker_user"
-
-USER bts
-WORKDIR /home/bts

global.json

@@ -1,6 +0,0 @@
-{
-  "sdk": {
-    "version": "6.0.200",
-    "rollForward": "latestMajor"
-  }
-}

spark_client.sln

@@ -1,143 +1,164 @@
-Microsoft Visual Studio Solution File, Format Version 12.00
-# Visual Studio Version 17
-VisualStudioVersion = 17.3.32922.545
-MinimumVisualStudioVersion = 15.0.26124.0
-Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "spark_api", "src\client\spark_api\spark_api.csproj", "{DC9A8975-8211-4FC9-A9BA-C8B9D26AD481}"
-EndProject
-Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Spark", "src\client\spark\Spark.csproj", "{9A4D4D07-1AB6-4B2B-B111-A4F34D6C267E}"
-EndProject
-Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Spark.Console", "src\client\Spark.Console\Spark.Console.csproj", "{387566C6-C6F6-412C-A30C-9855BBCB4816}"
-EndProject
-Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "spark_core", "src\client\spark_core\spark_core.csproj", "{6E449EB6-20AC-4FA6-B8D9-10F4BB7E1FEE}"
-EndProject
-Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "spark_sdk", "src\client\spark_sdk\spark_sdk.csproj", "{6963C314-83C4-4667-B0D0-C2F7AE208C29}"
-EndProject
-Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "spark_test", "src\client\spark_test\spark_test.csproj", "{ECD78CC5-138F-4809-A764-2358C2AC2E82}"
-EndProject
-Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "spark_websocket", "src\client\spark_websocket\spark_websocket.csproj", "{3201359D-C071-4089-9EDE-B04D759F1508}"
-EndProject
-Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "spark_api_server", "src\client\spark_api_server\spark_api_server.csproj", "{2B8C5D01-FF78-4BFA-B332-5E430586DAF9}"
-EndProject
-Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "DataStreamCore", "src\DataStreamSuite\DataStreamCore\DataStreamCore.csproj", "{4E4F491F-C688-43C4-88DA-AC92BC5FA1B8}"
-EndProject
-Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "DataStream", "src\DataStreamSuite\DataStream\DataStream.csproj", "{2AC54ADC-3246-427E-9CA9-FEA2DBCB9432}"
-EndProject
-Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "SharedStreams", "src\DataStreamSuite\SharedStreams\SharedStreams.csproj", "{4E96E5CC-7791-40B3-A8A1-C34FBB736B3B}"
-EndProject
-Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "PubSub", "src\DataStreamSuite\PubSub\PubSub.csproj", "{51C40BA9-F226-4A35-939D-1ED6F5FF7299}"
-EndProject
-Global
-	GlobalSection(SolutionConfigurationPlatforms) = preSolution
-		Debug|x64 = Debug|x64
-		MinSizeRel|x64 = MinSizeRel|x64
-		Release|x64 = Release|x64
-		RelWithDebInfo|x64 = RelWithDebInfo|x64
-	EndGlobalSection
-	GlobalSection(ProjectConfigurationPlatforms) = postSolution
-		{DC9A8975-8211-4FC9-A9BA-C8B9D26AD481}.Debug|x64.ActiveCfg = Debug|x64
-		{DC9A8975-8211-4FC9-A9BA-C8B9D26AD481}.Debug|x64.Build.0 = Debug|x64
-		{DC9A8975-8211-4FC9-A9BA-C8B9D26AD481}.MinSizeRel|x64.ActiveCfg = Release|x64
-		{DC9A8975-8211-4FC9-A9BA-C8B9D26AD481}.MinSizeRel|x64.Build.0 = Release|x64
-		{DC9A8975-8211-4FC9-A9BA-C8B9D26AD481}.Release|x64.ActiveCfg = Release|x64
-		{DC9A8975-8211-4FC9-A9BA-C8B9D26AD481}.Release|x64.Build.0 = Release|x64
-		{DC9A8975-8211-4FC9-A9BA-C8B9D26AD481}.RelWithDebInfo|x64.ActiveCfg = Release|x64
-		{DC9A8975-8211-4FC9-A9BA-C8B9D26AD481}.RelWithDebInfo|x64.Build.0 = Release|x64
-		{9A4D4D07-1AB6-4B2B-B111-A4F34D6C267E}.Debug|x64.ActiveCfg = Debug|x64
-		{9A4D4D07-1AB6-4B2B-B111-A4F34D6C267E}.Debug|x64.Build.0 = Debug|x64
-		{9A4D4D07-1AB6-4B2B-B111-A4F34D6C267E}.MinSizeRel|x64.ActiveCfg = Release|x64
-		{9A4D4D07-1AB6-4B2B-B111-A4F34D6C267E}.MinSizeRel|x64.Build.0 = Release|x64
-		{9A4D4D07-1AB6-4B2B-B111-A4F34D6C267E}.Release|x64.ActiveCfg = Release|x64
-		{9A4D4D07-1AB6-4B2B-B111-A4F34D6C267E}.Release|x64.Build.0 = Release|x64
-		{9A4D4D07-1AB6-4B2B-B111-A4F34D6C267E}.RelWithDebInfo|x64.ActiveCfg = Release|x64
-		{9A4D4D07-1AB6-4B2B-B111-A4F34D6C267E}.RelWithDebInfo|x64.Build.0 = Release|x64
-		{387566C6-C6F6-412C-A30C-9855BBCB4816}.Debug|x64.ActiveCfg = Debug|x64
-		{387566C6-C6F6-412C-A30C-9855BBCB4816}.Debug|x64.Build.0 = Debug|x64
-		{387566C6-C6F6-412C-A30C-9855BBCB4816}.MinSizeRel|x64.ActiveCfg = Debug|x64
-		{387566C6-C6F6-412C-A30C-9855BBCB4816}.MinSizeRel|x64.Build.0 = Debug|x64
-		{387566C6-C6F6-412C-A30C-9855BBCB4816}.Release|x64.ActiveCfg = Release|x64
-		{387566C6-C6F6-412C-A30C-9855BBCB4816}.Release|x64.Build.0 = Release|x64
-		{387566C6-C6F6-412C-A30C-9855BBCB4816}.RelWithDebInfo|x64.ActiveCfg = Release|x64
-		{387566C6-C6F6-412C-A30C-9855BBCB4816}.RelWithDebInfo|x64.Build.0 = Release|x64
-		{6E449EB6-20AC-4FA6-B8D9-10F4BB7E1FEE}.Debug|x64.ActiveCfg = Debug|x64
-		{6E449EB6-20AC-4FA6-B8D9-10F4BB7E1FEE}.Debug|x64.Build.0 = Debug|x64
-		{6E449EB6-20AC-4FA6-B8D9-10F4BB7E1FEE}.MinSizeRel|x64.ActiveCfg = Release|x64
-		{6E449EB6-20AC-4FA6-B8D9-10F4BB7E1FEE}.MinSizeRel|x64.Build.0 = Release|x64
-		{6E449EB6-20AC-4FA6-B8D9-10F4BB7E1FEE}.Release|x64.ActiveCfg = Release|x64
-		{6E449EB6-20AC-4FA6-B8D9-10F4BB7E1FEE}.Release|x64.Build.0 = Release|x64
-		{6E449EB6-20AC-4FA6-B8D9-10F4BB7E1FEE}.RelWithDebInfo|x64.ActiveCfg = Release|x64
-		{6E449EB6-20AC-4FA6-B8D9-10F4BB7E1FEE}.RelWithDebInfo|x64.Build.0 = Release|x64
-		{6963C314-83C4-4667-B0D0-C2F7AE208C29}.Debug|x64.ActiveCfg = Debug|x64
-		{6963C314-83C4-4667-B0D0-C2F7AE208C29}.Debug|x64.Build.0 = Debug|x64
-		{6963C314-83C4-4667-B0D0-C2F7AE208C29}.MinSizeRel|x64.ActiveCfg = Release|x64
-		{6963C314-83C4-4667-B0D0-C2F7AE208C29}.MinSizeRel|x64.Build.0 = Release|x64
-		{6963C314-83C4-4667-B0D0-C2F7AE208C29}.Release|x64.ActiveCfg = Release|x64
-		{6963C314-83C4-4667-B0D0-C2F7AE208C29}.Release|x64.Build.0 = Release|x64
-		{6963C314-83C4-4667-B0D0-C2F7AE208C29}.RelWithDebInfo|x64.ActiveCfg = Release|x64
-		{6963C314-83C4-4667-B0D0-C2F7AE208C29}.RelWithDebInfo|x64.Build.0 = Release|x64
-		{ECD78CC5-138F-4809-A764-2358C2AC2E82}.Debug|x64.ActiveCfg = Debug|x64
-		{ECD78CC5-138F-4809-A764-2358C2AC2E82}.Debug|x64.Build.0 = Debug|x64
-		{ECD78CC5-138F-4809-A764-2358C2AC2E82}.MinSizeRel|x64.ActiveCfg = Debug|x64
-		{ECD78CC5-138F-4809-A764-2358C2AC2E82}.MinSizeRel|x64.Build.0 = Debug|x64
-		{ECD78CC5-138F-4809-A764-2358C2AC2E82}.Release|x64.ActiveCfg = Release|x64
-		{ECD78CC5-138F-4809-A764-2358C2AC2E82}.Release|x64.Build.0 = Release|x64
-		{ECD78CC5-138F-4809-A764-2358C2AC2E82}.RelWithDebInfo|x64.ActiveCfg = Release|x64
-		{ECD78CC5-138F-4809-A764-2358C2AC2E82}.RelWithDebInfo|x64.Build.0 = Release|x64
-		{3201359D-C071-4089-9EDE-B04D759F1508}.Debug|x64.ActiveCfg = Debug|x64
-		{3201359D-C071-4089-9EDE-B04D759F1508}.Debug|x64.Build.0 = Debug|x64
-		{3201359D-C071-4089-9EDE-B04D759F1508}.MinSizeRel|x64.ActiveCfg = Debug|x64
-		{3201359D-C071-4089-9EDE-B04D759F1508}.MinSizeRel|x64.Build.0 = Debug|x64
-		{3201359D-C071-4089-9EDE-B04D759F1508}.Release|x64.ActiveCfg = Release|x64
-		{3201359D-C071-4089-9EDE-B04D759F1508}.Release|x64.Build.0 = Release|x64
-		{3201359D-C071-4089-9EDE-B04D759F1508}.RelWithDebInfo|x64.ActiveCfg = Release|x64
-		{3201359D-C071-4089-9EDE-B04D759F1508}.RelWithDebInfo|x64.Build.0 = Release|x64
-		{2B8C5D01-FF78-4BFA-B332-5E430586DAF9}.Debug|x64.ActiveCfg = Debug|x64
-		{2B8C5D01-FF78-4BFA-B332-5E430586DAF9}.Debug|x64.Build.0 = Debug|x64
-		{2B8C5D01-FF78-4BFA-B332-5E430586DAF9}.MinSizeRel|x64.ActiveCfg = Debug|x64
-		{2B8C5D01-FF78-4BFA-B332-5E430586DAF9}.MinSizeRel|x64.Build.0 = Debug|x64
-		{2B8C5D01-FF78-4BFA-B332-5E430586DAF9}.Release|x64.ActiveCfg = Release|x64
-		{2B8C5D01-FF78-4BFA-B332-5E430586DAF9}.Release|x64.Build.0 = Release|x64
-		{2B8C5D01-FF78-4BFA-B332-5E430586DAF9}.RelWithDebInfo|x64.ActiveCfg = Release|x64
-		{2B8C5D01-FF78-4BFA-B332-5E430586DAF9}.RelWithDebInfo|x64.Build.0 = Release|x64
-		{4E4F491F-C688-43C4-88DA-AC92BC5FA1B8}.Debug|x64.ActiveCfg = Debug|Any CPU
-		{4E4F491F-C688-43C4-88DA-AC92BC5FA1B8}.Debug|x64.Build.0 = Debug|Any CPU
-		{4E4F491F-C688-43C4-88DA-AC92BC5FA1B8}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
-		{4E4F491F-C688-43C4-88DA-AC92BC5FA1B8}.MinSizeRel|x64.Build.0 = Debug|Any CPU
-		{4E4F491F-C688-43C4-88DA-AC92BC5FA1B8}.Release|x64.ActiveCfg = Release|Any CPU
-		{4E4F491F-C688-43C4-88DA-AC92BC5FA1B8}.Release|x64.Build.0 = Release|Any CPU
-		{4E4F491F-C688-43C4-88DA-AC92BC5FA1B8}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
-		{4E4F491F-C688-43C4-88DA-AC92BC5FA1B8}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
-		{2AC54ADC-3246-427E-9CA9-FEA2DBCB9432}.Debug|x64.ActiveCfg = Debug|Any CPU
-		{2AC54ADC-3246-427E-9CA9-FEA2DBCB9432}.Debug|x64.Build.0 = Debug|Any CPU
-		{2AC54ADC-3246-427E-9CA9-FEA2DBCB9432}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
-		{2AC54ADC-3246-427E-9CA9-FEA2DBCB9432}.MinSizeRel|x64.Build.0 = Debug|Any CPU
-		{2AC54ADC-3246-427E-9CA9-FEA2DBCB9432}.Release|x64.ActiveCfg = Release|Any CPU
-		{2AC54ADC-3246-427E-9CA9-FEA2DBCB9432}.Release|x64.Build.0 = Release|Any CPU
-		{2AC54ADC-3246-427E-9CA9-FEA2DBCB9432}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
-		{2AC54ADC-3246-427E-9CA9-FEA2DBCB9432}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
-		{4E96E5CC-7791-40B3-A8A1-C34FBB736B3B}.Debug|x64.ActiveCfg = Debug|Any CPU
-		{4E96E5CC-7791-40B3-A8A1-C34FBB736B3B}.Debug|x64.Build.0 = Debug|Any CPU
-		{4E96E5CC-7791-40B3-A8A1-C34FBB736B3B}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
-		{4E96E5CC-7791-40B3-A8A1-C34FBB736B3B}.MinSizeRel|x64.Build.0 = Debug|Any CPU
-		{4E96E5CC-7791-40B3-A8A1-C34FBB736B3B}.Release|x64.ActiveCfg = Release|Any CPU
-		{4E96E5CC-7791-40B3-A8A1-C34FBB736B3B}.Release|x64.Build.0 = Release|Any CPU
-		{4E96E5CC-7791-40B3-A8A1-C34FBB736B3B}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
-		{4E96E5CC-7791-40B3-A8A1-C34FBB736B3B}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
-		{51C40BA9-F226-4A35-939D-1ED6F5FF7299}.Debug|x64.ActiveCfg = Debug|Any CPU
-		{51C40BA9-F226-4A35-939D-1ED6F5FF7299}.Debug|x64.Build.0 = Debug|Any CPU
-		{51C40BA9-F226-4A35-939D-1ED6F5FF7299}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
-		{51C40BA9-F226-4A35-939D-1ED6F5FF7299}.MinSizeRel|x64.Build.0 = Debug|Any CPU
-		{51C40BA9-F226-4A35-939D-1ED6F5FF7299}.Release|x64.ActiveCfg = Release|Any CPU
-		{51C40BA9-F226-4A35-939D-1ED6F5FF7299}.Release|x64.Build.0 = Release|Any CPU
-		{51C40BA9-F226-4A35-939D-1ED6F5FF7299}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
-		{51C40BA9-F226-4A35-939D-1ED6F5FF7299}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
-	EndGlobalSection
-	GlobalSection(SolutionProperties) = preSolution
-		HideSolutionNode = FALSE
-	EndGlobalSection
-	GlobalSection(ExtensibilityGlobals) = postSolution
-		SolutionGuid = {422D1620-0166-4C26-9A81-CA8A2EB1AB58}
-	EndGlobalSection
-	GlobalSection(Performance) = preSolution
-		HasPerformanceSessions = true
-	EndGlobalSection
-EndGlobal
+Microsoft Visual Studio Solution File, Format Version 12.00
+# Visual Studio Version 17
+VisualStudioVersion = 17.3.32922.545
+MinimumVisualStudioVersion = 15.0.26124.0
+Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "spark_api", "src\client\spark_api\spark_api.csproj", "{DC9A8975-8211-4FC9-A9BA-C8B9D26AD481}"
+EndProject
+Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Spark", "src\client\spark\Spark.csproj", "{9A4D4D07-1AB6-4B2B-B111-A4F34D6C267E}"
+EndProject
+Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Spark.Console", "src\client\Spark.Console\Spark.Console.csproj", "{387566C6-C6F6-412C-A30C-9855BBCB4816}"
+EndProject
+Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "spark_core", "src\client\spark_core\spark_core.csproj", "{6E449EB6-20AC-4FA6-B8D9-10F4BB7E1FEE}"
+EndProject
+Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "spark_sdk", "src\client\spark_sdk\spark_sdk.csproj", "{6963C314-83C4-4667-B0D0-C2F7AE208C29}"
+EndProject
+Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "spark_test", "src\client\spark_test\spark_test.csproj", "{ECD78CC5-138F-4809-A764-2358C2AC2E82}"
+EndProject
+Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "spark_websocket", "src\client\spark_websocket\spark_websocket.csproj", "{3201359D-C071-4089-9EDE-B04D759F1508}"
+EndProject
+Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "spark_api_server", "src\client\spark_api_server\spark_api_server.csproj", "{2B8C5D01-FF78-4BFA-B332-5E430586DAF9}"
+EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "DataStreamCore", "src\DataStreamSuite\DataStreamCore\DataStreamCore.csproj", "{4E4F491F-C688-43C4-88DA-AC92BC5FA1B8}"
+EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "DataStream", "src\DataStreamSuite\DataStream\DataStream.csproj", "{2AC54ADC-3246-427E-9CA9-FEA2DBCB9432}"
+EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "SharedStreams", "src\DataStreamSuite\SharedStreams\SharedStreams.csproj", "{4E96E5CC-7791-40B3-A8A1-C34FBB736B3B}"
+EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "PubSub", "src\DataStreamSuite\PubSub\PubSub.csproj", "{51C40BA9-F226-4A35-939D-1ED6F5FF7299}"
+EndProject
+Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "src", "src", "{4564E677-343A-46CB-AF02-AFC4E38BF94B}"
+EndProject
+Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "client", "client", "{56A8A70A-DA03-4CED-A144-EC10F1379FD8}"
+EndProject
+Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "DataStreamSuite", "DataStreamSuite", "{915EF238-CA0D-4E73-AF50-88CB3F29C7AD}"
+EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Bts.Extensions.Hosting", "src\DataStreamSuite\Bts.Extensions.Hosting\Bts.Extensions.Hosting.csproj", "{4E0E51AA-7B59-4808-B7CC-9EAEDF397460}"
+EndProject
+Global
+	GlobalSection(SolutionConfigurationPlatforms) = preSolution
+		Debug|x64 = Debug|x64
+		MinSizeRel|x64 = MinSizeRel|x64
+		Release|x64 = Release|x64
+		RelWithDebInfo|x64 = RelWithDebInfo|x64
+	EndGlobalSection
+	GlobalSection(ProjectConfigurationPlatforms) = postSolution
+		{DC9A8975-8211-4FC9-A9BA-C8B9D26AD481}.Debug|x64.ActiveCfg = Debug|x64
+		{DC9A8975-8211-4FC9-A9BA-C8B9D26AD481}.Debug|x64.Build.0 = Debug|x64
+		{DC9A8975-8211-4FC9-A9BA-C8B9D26AD481}.MinSizeRel|x64.ActiveCfg = Release|x64
+		{DC9A8975-8211-4FC9-A9BA-C8B9D26AD481}.MinSizeRel|x64.Build.0 = Release|x64
+		{DC9A8975-8211-4FC9-A9BA-C8B9D26AD481}.Release|x64.ActiveCfg = Release|x64
+		{DC9A8975-8211-4FC9-A9BA-C8B9D26AD481}.Release|x64.Build.0 = Release|x64
+		{DC9A8975-8211-4FC9-A9BA-C8B9D26AD481}.RelWithDebInfo|x64.ActiveCfg = Release|x64
+		{DC9A8975-8211-4FC9-A9BA-C8B9D26AD481}.RelWithDebInfo|x64.Build.0 = Release|x64
+		{9A4D4D07-1AB6-4B2B-B111-A4F34D6C267E}.Debug|x64.ActiveCfg = Debug|x64
+		{9A4D4D07-1AB6-4B2B-B111-A4F34D6C267E}.Debug|x64.Build.0 = Debug|x64
+		{9A4D4D07-1AB6-4B2B-B111-A4F34D6C267E}.MinSizeRel|x64.ActiveCfg = Release|x64
+		{9A4D4D07-1AB6-4B2B-B111-A4F34D6C267E}.MinSizeRel|x64.Build.0 = Release|x64
+		{9A4D4D07-1AB6-4B2B-B111-A4F34D6C267E}.Release|x64.ActiveCfg = Release|x64
+		{9A4D4D07-1AB6-4B2B-B111-A4F34D6C267E}.Release|x64.Build.0 = Release|x64
+		{9A4D4D07-1AB6-4B2B-B111-A4F34D6C267E}.RelWithDebInfo|x64.ActiveCfg = Release|x64
+		{9A4D4D07-1AB6-4B2B-B111-A4F34D6C267E}.RelWithDebInfo|x64.Build.0 = Release|x64
+		{387566C6-C6F6-412C-A30C-9855BBCB4816}.Debug|x64.ActiveCfg = Debug|x64
+		{387566C6-C6F6-412C-A30C-9855BBCB4816}.Debug|x64.Build.0 = Debug|x64
+		{387566C6-C6F6-412C-A30C-9855BBCB4816}.MinSizeRel|x64.ActiveCfg = Debug|x64
+		{387566C6-C6F6-412C-A30C-9855BBCB4816}.MinSizeRel|x64.Build.0 = Debug|x64
+		{387566C6-C6F6-412C-A30C-9855BBCB4816}.Release|x64.ActiveCfg = Release|x64
+		{387566C6-C6F6-412C-A30C-9855BBCB4816}.Release|x64.Build.0 = Release|x64
+		{387566C6-C6F6-412C-A30C-9855BBCB4816}.RelWithDebInfo|x64.ActiveCfg = Release|x64
+		{387566C6-C6F6-412C-A30C-9855BBCB4816}.RelWithDebInfo|x64.Build.0 = Release|x64
+		{6E449EB6-20AC-4FA6-B8D9-10F4BB7E1FEE}.Debug|x64.ActiveCfg = Debug|x64
+		{6E449EB6-20AC-4FA6-B8D9-10F4BB7E1FEE}.Debug|x64.Build.0 = Debug|x64
+		{6E449EB6-20AC-4FA6-B8D9-10F4BB7E1FEE}.MinSizeRel|x64.ActiveCfg = Release|x64
+		{6E449EB6-20AC-4FA6-B8D9-10F4BB7E1FEE}.MinSizeRel|x64.Build.0 = Release|x64
+		{6E449EB6-20AC-4FA6-B8D9-10F4BB7E1FEE}.Release|x64.ActiveCfg = Release|x64
+		{6E449EB6-20AC-4FA6-B8D9-10F4BB7E1FEE}.Release|x64.Build.0 = Release|x64
+		{6E449EB6-20AC-4FA6-B8D9-10F4BB7E1FEE}.RelWithDebInfo|x64.ActiveCfg = Release|x64
+		{6E449EB6-20AC-4FA6-B8D9-10F4BB7E1FEE}.RelWithDebInfo|x64.Build.0 = Release|x64
+		{6963C314-83C4-4667-B0D0-C2F7AE208C29}.Debug|x64.ActiveCfg = Debug|x64
+		{6963C314-83C4-4667-B0D0-C2F7AE208C29}.Debug|x64.Build.0 = Debug|x64
+		{6963C314-83C4-4667-B0D0-C2F7AE208C29}.MinSizeRel|x64.ActiveCfg = Release|x64
+		{6963C314-83C4-4667-B0D0-C2F7AE208C29}.MinSizeRel|x64.Build.0 = Release|x64
+		{6963C314-83C4-4667-B0D0-C2F7AE208C29}.Release|x64.ActiveCfg = Release|x64
+		{6963C314-83C4-4667-B0D0-C2F7AE208C29}.Release|x64.Build.0 = Release|x64
+		{6963C314-83C4-4667-B0D0-C2F7AE208C29}.RelWithDebInfo|x64.ActiveCfg = Release|x64
+		{6963C314-83C4-4667-B0D0-C2F7AE208C29}.RelWithDebInfo|x64.Build.0 = Release|x64
+		{ECD78CC5-138F-4809-A764-2358C2AC2E82}.Debug|x64.ActiveCfg = Debug|x64
+		{ECD78CC5-138F-4809-A764-2358C2AC2E82}.Debug|x64.Build.0 = Debug|x64
+		{ECD78CC5-138F-4809-A764-2358C2AC2E82}.MinSizeRel|x64.ActiveCfg = Debug|x64
+		{ECD78CC5-138F-4809-A764-2358C2AC2E82}.MinSizeRel|x64.Build.0 = Debug|x64
+		{ECD78CC5-138F-4809-A764-2358C2AC2E82}.Release|x64.ActiveCfg = Release|x64
+		{ECD78CC5-138F-4809-A764-2358C2AC2E82}.Release|x64.Build.0 = Release|x64
+		{ECD78CC5-138F-4809-A764-2358C2AC2E82}.RelWithDebInfo|x64.ActiveCfg = Release|x64
+		{ECD78CC5-138F-4809-A764-2358C2AC2E82}.RelWithDebInfo|x64.Build.0 = Release|x64
+		{3201359D-C071-4089-9EDE-B04D759F1508}.Debug|x64.ActiveCfg = Debug|x64
+		{3201359D-C071-4089-9EDE-B04D759F1508}.Debug|x64.Build.0 = Debug|x64
+		{3201359D-C071-4089-9EDE-B04D759F1508}.MinSizeRel|x64.ActiveCfg = Debug|x64
+		{3201359D-C071-4089-9EDE-B04D759F1508}.MinSizeRel|x64.Build.0 = Debug|x64
+		{3201359D-C071-4089-9EDE-B04D759F1508}.Release|x64.ActiveCfg = Release|x64
+		{3201359D-C071-4089-9EDE-B04D759F1508}.Release|x64.Build.0 = Release|x64
+		{3201359D-C071-4089-9EDE-B04D759F1508}.RelWithDebInfo|x64.ActiveCfg = Release|x64
+		{3201359D-C071-4089-9EDE-B04D759F1508}.RelWithDebInfo|x64.Build.0 = Release|x64
+		{2B8C5D01-FF78-4BFA-B332-5E430586DAF9}.Debug|x64.ActiveCfg = Debug|x64
+		{2B8C5D01-FF78-4BFA-B332-5E430586DAF9}.Debug|x64.Build.0 = Debug|x64
+		{2B8C5D01-FF78-4BFA-B332-5E430586DAF9}.MinSizeRel|x64.ActiveCfg = Debug|x64
+		{2B8C5D01-FF78-4BFA-B332-5E430586DAF9}.MinSizeRel|x64.Build.0 = Debug|x64
+		{2B8C5D01-FF78-4BFA-B332-5E430586DAF9}.Release|x64.ActiveCfg = Release|x64
+		{2B8C5D01-FF78-4BFA-B332-5E430586DAF9}.Release|x64.Build.0 = Release|x64
+		{2B8C5D01-FF78-4BFA-B332-5E430586DAF9}.RelWithDebInfo|x64.ActiveCfg = Release|x64
+		{2B8C5D01-FF78-4BFA-B332-5E430586DAF9}.RelWithDebInfo|x64.Build.0 = Release|x64
+		{4E4F491F-C688-43C4-88DA-AC92BC5FA1B8}.Debug|x64.ActiveCfg = Debug|Any CPU
+		{4E4F491F-C688-43C4-88DA-AC92BC5FA1B8}.Debug|x64.Build.0 = Debug|Any CPU
+		{4E4F491F-C688-43C4-88DA-AC92BC5FA1B8}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
+		{4E4F491F-C688-43C4-88DA-AC92BC5FA1B8}.MinSizeRel|x64.Build.0 = Debug|Any CPU
+		{4E4F491F-C688-43C4-88DA-AC92BC5FA1B8}.Release|x64.ActiveCfg = Release|Any CPU
+		{4E4F491F-C688-43C4-88DA-AC92BC5FA1B8}.Release|x64.Build.0 = Release|Any CPU
+		{4E4F491F-C688-43C4-88DA-AC92BC5FA1B8}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
+		{4E4F491F-C688-43C4-88DA-AC92BC5FA1B8}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
+		{2AC54ADC-3246-427E-9CA9-FEA2DBCB9432}.Debug|x64.ActiveCfg = Debug|Any CPU
+		{2AC54ADC-3246-427E-9CA9-FEA2DBCB9432}.Debug|x64.Build.0 = Debug|Any CPU
+		{2AC54ADC-3246-427E-9CA9-FEA2DBCB9432}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
+		{2AC54ADC-3246-427E-9CA9-FEA2DBCB9432}.MinSizeRel|x64.Build.0 = Debug|Any CPU
+		{2AC54ADC-3246-427E-9CA9-FEA2DBCB9432}.Release|x64.ActiveCfg = Release|Any CPU
+		{2AC54ADC-3246-427E-9CA9-FEA2DBCB9432}.Release|x64.Build.0 = Release|Any CPU
+		{2AC54ADC-3246-427E-9CA9-FEA2DBCB9432}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
+		{2AC54ADC-3246-427E-9CA9-FEA2DBCB9432}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
+		{4E96E5CC-7791-40B3-A8A1-C34FBB736B3B}.Debug|x64.ActiveCfg = Debug|Any CPU
+		{4E96E5CC-7791-40B3-A8A1-C34FBB736B3B}.Debug|x64.Build.0 = Debug|Any CPU
+		{4E96E5CC-7791-40B3-A8A1-C34FBB736B3B}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
+		{4E96E5CC-7791-40B3-A8A1-C34FBB736B3B}.MinSizeRel|x64.Build.0 = Debug|Any CPU
+		{4E96E5CC-7791-40B3-A8A1-C34FBB736B3B}.Release|x64.ActiveCfg = Release|Any CPU
+		{4E96E5CC-7791-40B3-A8A1-C34FBB736B3B}.Release|x64.Build.0 = Release|Any CPU
+		{4E96E5CC-7791-40B3-A8A1-C34FBB736B3B}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
+		{4E96E5CC-7791-40B3-A8A1-C34FBB736B3B}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
+		{51C40BA9-F226-4A35-939D-1ED6F5FF7299}.Debug|x64.ActiveCfg = Debug|Any CPU
+		{51C40BA9-F226-4A35-939D-1ED6F5FF7299}.Debug|x64.Build.0 = Debug|Any CPU
+		{51C40BA9-F226-4A35-939D-1ED6F5FF7299}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
+		{51C40BA9-F226-4A35-939D-1ED6F5FF7299}.MinSizeRel|x64.Build.0 = Debug|Any CPU
+		{51C40BA9-F226-4A35-939D-1ED6F5FF7299}.Release|x64.ActiveCfg = Release|Any CPU
+		{51C40BA9-F226-4A35-939D-1ED6F5FF7299}.Release|x64.Build.0 = Release|Any CPU
+		{51C40BA9-F226-4A35-939D-1ED6F5FF7299}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
+		{51C40BA9-F226-4A35-939D-1ED6F5FF7299}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
+		{4E0E51AA-7B59-4808-B7CC-9EAEDF397460}.Debug|x64.ActiveCfg = Debug|Any CPU
+		{4E0E51AA-7B59-4808-B7CC-9EAEDF397460}.Debug|x64.Build.0 = Debug|Any CPU
+		{4E0E51AA-7B59-4808-B7CC-9EAEDF397460}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
+		{4E0E51AA-7B59-4808-B7CC-9EAEDF397460}.MinSizeRel|x64.Build.0 = Debug|Any CPU
+		{4E0E51AA-7B59-4808-B7CC-9EAEDF397460}.Release|x64.ActiveCfg = Release|Any CPU
+		{4E0E51AA-7B59-4808-B7CC-9EAEDF397460}.Release|x64.Build.0 = Release|Any CPU
+		{4E0E51AA-7B59-4808-B7CC-9EAEDF397460}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
+		{4E0E51AA-7B59-4808-B7CC-9EAEDF397460}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
+	EndGlobalSection
+	GlobalSection(SolutionProperties) = preSolution
+		HideSolutionNode = FALSE
+	EndGlobalSection
+	GlobalSection(ExtensibilityGlobals) = postSolution
+		SolutionGuid = {422D1620-0166-4C26-9A81-CA8A2EB1AB58}
+	EndGlobalSection
+	GlobalSection(Performance) = preSolution
+		HasPerformanceSessions = true
+	EndGlobalSection
+	GlobalSection(NestedProjects) = preSolution
+		{56A8A70A-DA03-4CED-A144-EC10F1379FD8} = {4564E677-343A-46CB-AF02-AFC4E38BF94B}
+		{915EF238-CA0D-4E73-AF50-88CB3F29C7AD} = {4564E677-343A-46CB-AF02-AFC4E38BF94B}
+		{4E0E51AA-7B59-4808-B7CC-9EAEDF397460} = {915EF238-CA0D-4E73-AF50-88CB3F29C7AD}
+	EndGlobalSection
+EndGlobal

src/DataStreamSuite/Bts.DataStreamWebApp.Client/.env.development

@@ -1 +0,0 @@
-VITE_DATASTREAM_WEBSOCKET_URL=wss://localhost:8443/ws

src/DataStreamSuite/Bts.DataStreamWebApp.Client/.env.production

@@ -1 +0,0 @@
-VITE_DATASTREAM_WEBSOCKET_URL=wss://localhost:8443/ws

src/DataStreamSuite/Bts.DataStreamWebApp.Client/.eslintrc.cjs

@@ -1,20 +0,0 @@
-module.exports = {
-  root: true,
-  env: { browser: true, es2020: true },
-  extends: [
-    'eslint:recommended',
-    'plugin:react/recommended',
-    'plugin:react/jsx-runtime',
-    'plugin:react-hooks/recommended',
-  ],
-  ignorePatterns: ['dist', '.eslintrc.cjs'],
-  parserOptions: { ecmaVersion: 'latest', sourceType: 'module' },
-  settings: { react: { version: '18.3.1' } },
-  plugins: ['react-refresh'],
-  rules: {
-    'react-refresh/only-export-components': [
-      'warn',
-      { allowConstantExport: true },
-    ],
-  },
-}

src/DataStreamSuite/Bts.DataStreamWebApp.Client/.gitignore

@@ -1,24 +0,0 @@
-# Logs
-logs
-*.log
-npm-debug.log*
-yarn-debug.log*
-yarn-error.log*
-pnpm-debug.log*
-lerna-debug.log*
-
-node_modules
-dist
-dist-ssr
-*.local
-
-# Editor directories and files
-.vscode/*
-!.vscode/extensions.json
-.idea
-.DS_Store
-*.suo
-*.ntvs*
-*.njsproj
-*.sln
-*.sw?

src/DataStreamSuite/Bts.DataStreamWebApp.Client/Bts.DataStreamWebApp.Client.esproj

@@ -1,11 +0,0 @@
-<Project Sdk="Microsoft.VisualStudio.JavaScript.Sdk/0.5.128-alpha">
-  <PropertyGroup>
-    <StartupCommand>npm run dev</StartupCommand>
-    <JavaScriptTestRoot>src\</JavaScriptTestRoot>
-    <JavaScriptTestFramework>Jest</JavaScriptTestFramework>
-    <!-- Allows the build (or compile) script located on package.json to run on Build -->
-    <ShouldRunBuildScript>true</ShouldRunBuildScript>
-    <!-- Folder where production build objects will be placed -->
-    <PublishAssetsDirectory>$(MSBuildProjectDirectory)\distill</PublishAssetsDirectory>
-  </PropertyGroup>
-</Project>

src/DataStreamSuite/Bts.DataStreamWebApp.Client/README.md

@@ -1,8 +0,0 @@
-# React + Vite
-
-This template provides a minimal setup to get React working in Vite with HMR and some ESLint rules.
-
-Currently, two official plugins are available:
-
-- [@vitejs/plugin-react](https://github.com/vitejs/vite-plugin-react/blob/main/packages/plugin-react/README.md) uses [Babel](https://babeljs.io/) for Fast Refresh
-- [@vitejs/plugin-react-swc](https://github.com/vitejs/vite-plugin-react-swc) uses [SWC](https://swc.rs/) for Fast Refresh

src/DataStreamSuite/Bts.DataStreamWebApp.Client/index.html

@@ -1,13 +0,0 @@
-<!doctype html>
-<html lang="en">
-  <head>
-    <meta charset="UTF-8" />
-    <link rel="icon" type="image/ico" href="/logo.ico" />
-    <meta name="viewport" content="width=device-width, initial-scale=1.0" />
-    <title>BTS Cloud Streams</title>
-  </head>
-  <body>
-    <div id="root"></div>
-    <script type="module" src="/src/main.jsx"></script>
-  </body>
-</html>

src/DataStreamSuite/Bts.DataStreamWebApp.Client/nuget.config

@@ -1,10 +0,0 @@
-<?xml version="1.0" encoding="utf-8"?>
-<configuration>
-  <packageSources>
-    <clear />
-    <add key="nuget.org" value="https://api.nuget.org/v3/index.json" />
-  </packageSources>
-  <disabledPackageSources>
-    <clear />
-  </disabledPackageSources>
-</configuration>

src/DataStreamSuite/Bts.DataStreamWebApp.Client/package.json

@@ -1,43 +0,0 @@
-{
-  "name": "bts.datastreamwebapp.client",
-  "private": true,
-  "version": "0.0.0",
-  "type": "module",
-  "scripts": {
-    "dev": "vite",
-    "build": "vite build",
-    "lint": "eslint . --ext js,jsx --report-unused-disable-directives --max-warnings 0",
-    "preview": "vite preview"
-  },
-  "dependencies": {
-    "@angular/material": "^18.0.4",
-    "@emotion/react": "^11.11.4",
-    "@emotion/styled": "^11.11.5",
-    "@fontsource/noto-sans": "^5.0.22",
-    "@material/data-table": "^15.0.0-canary.7f224ddd4.0",
-    "@mui/icons-material": "^5.15.20",
-    "@mui/material": "^5.15.20",
-    "@mui/x-data-grid": "^7.7.1",
-    "bulma": "^1.0.1",
-    "classnames": "^2.5.1",
-    "react": "^18.3.1",
-    "react-data-grid": "^7.0.0-beta.44",
-    "react-data-table-component": "^7.6.2",
-    "react-dom": "^18.3.1",
-    "react-json-view-lite": "^1.4.0",
-    "react-pro-sidebar": "^1.1.0",
-    "react-use-websocket": "^4.8.1",
-    "rpc-websockets": "^7.11.2"
-  },
-  "devDependencies": {
-    "@types/react": "^18.3.3",
-    "@types/react-dom": "^18.3.0",
-    "@vitejs/plugin-react": "^4.3.1",
-    "eslint": "^8.57.0",
-    "eslint-plugin-react": "^7.34.2",
-    "eslint-plugin-react-hooks": "^4.6.2",
-    "eslint-plugin-react-refresh": "^0.4.7",
-    "react-mui-sidebar": "^1.3.5",
-    "vite": "^5.2.13"
-  }
-}

src/DataStreamSuite/Bts.DataStreamWebApp.Client/public/vite.svg

@@ -1 +0,0 @@
-<svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" class="iconify iconify--logos" width="31.88" height="32" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 257"><defs><linearGradient id="IconifyId1813088fe1fbc01fb466" x1="-.828%" x2="57.636%" y1="7.652%" y2="78.411%"><stop offset="0%" stop-color="#41D1FF"></stop><stop offset="100%" stop-color="#BD34FE"></stop></linearGradient><linearGradient id="IconifyId1813088fe1fbc01fb467" x1="43.376%" x2="50.316%" y1="2.242%" y2="89.03%"><stop offset="0%" stop-color="#FFEA83"></stop><stop offset="8.333%" stop-color="#FFDD35"></stop><stop offset="100%" stop-color="#FFA800"></stop></linearGradient></defs><path fill="url(#IconifyId1813088fe1fbc01fb466)" d="M255.153 37.938L134.897 252.976c-2.483 4.44-8.862 4.466-11.382.048L.875 37.958c-2.746-4.814 1.371-10.646 6.827-9.67l120.385 21.517a6.537 6.537 0 0 0 2.322-.004l117.867-21.483c5.438-.991 9.574 4.796 6.877 9.62Z"></path><path fill="url(#IconifyId1813088fe1fbc01fb467)" d="M185.432.063L96.44 17.501a3.268 3.268 0 0 0-2.634 3.014l-5.474 92.456a3.268 3.268 0 0 0 3.997 3.378l24.777-5.718c2.318-.535 4.413 1.507 3.936 3.838l-7.361 36.047c-.495 2.426 1.782 4.5 4.151 3.78l15.304-4.649c2.372-.72 4.652 1.36 4.15 3.788l-11.698 56.621c-.732 3.542 3.979 5.473 5.943 2.437l1.313-2.028l72.516-144.72c1.215-2.423-.88-5.186-3.54-4.672l-25.505 4.922c-2.396.462-4.435-1.77-3.759-4.114l16.646-57.705c.677-2.35-1.37-4.583-3.769-4.113Z"></path></svg>

src/DataStreamSuite/Bts.DataStreamWebApp.Client/src/AnimatedSideBar.jsx

@@ -1,40 +0,0 @@
-import { Sidebar, Menu, MenuItem, Submenu, Logo } from "react-mui-sidebar";
-import './App.css';
-
-const AnimatedSideBar = () => {
-  return (
-    <Sidebar className={"sidebar-container"} width={"270px"}>
-      <Logo img="https://adminmart.com/wp-content/uploads/2024/03/logo-admin-mart-news.png">
-        AdminMart
-      </Logo>
-      <Menu subHeading="HOME">
-        <MenuItem link="/" badge="true">
-          Modern
-        </MenuItem>
-        <MenuItem>eCommerce</MenuItem>
-        <MenuItem>Analytical</MenuItem>
-      </Menu>
-      <Menu subHeading="APPS">
-        <MenuItem>Chat</MenuItem>
-        <MenuItem>Calendar</MenuItem>
-      </Menu>
-      <Menu subHeading="OTHERS">
-        <Submenu title="Menu Level">
-          <MenuItem>Post</MenuItem>
-          <MenuItem>Details</MenuItem>
-          <Submenu title="Level 2">
-            <MenuItem>new</MenuItem>
-            <MenuItem>Hello</MenuItem>
-          </Submenu>
-        </Submenu>
-
-        <MenuItem>Chip</MenuItem>
-        <MenuItem target="_blank" link="google.com">
-          External Link
-        </MenuItem>
-      </Menu>
-    </Sidebar>
-  );
-};
-
-export default AnimatedSideBar;

src/DataStreamSuite/Bts.DataStreamWebApp.Client/src/App.css

@@ -1,101 +0,0 @@
-@use "@material/checkbox";
-@use "@material/icon-button";
-@use "@material/data-table/data-table";
-
-.root-div {
-  display: flex;
-  flex-direction: row;
-  align-items: flex-start;
-}
-
-.main-content-div {
-  display: flex;
-  flex-direction: row;
-}
-
-.title-div {
-  width: 100%;
-}
-
-.container {
-  display: flex;
-}
-
-.container-column {
-  display: flex;
-  flex-direction: column;
-}
-
-.container-row {
-  display: flex;
-  flex-direction: row;
-  align-items: stretch;
-}
-
-box.query-input-form {
-   /* p: 2, display: "flex", flexFlow: "column", alignItems: "stretch"  */
-  display: flex;
-  flex-flow: column;
-  align-items: stretch;
-}
-
-/* li.ps-menuitem-root:hover {
-  background-color: #ff9922;  
-} */
-
-.container-row-centered {
-  display: flex;
-  flex-direction: row;
-  justify-content: center;
-  align-items: center;
-}
-
-h1#main-header {
-  padding: 4px 8px 4px 8px;
-  text-align: center;
-}
-
-td:hover {
-  background-color: cadetblue;
-}
-
-td#cell-decimal {
-  text-align: right;
-}
-
-td#cell-integer {
-  text-align: right;
-}
-
-td#cell-string {
-  text-align: center;
-}
-
-td#cell-date-time {
-  text-align: center;
-}
-
-/* Sidebar.css */
-/* .sidebar-container {
-  display: flex;
-  flex-direction: column;
-}
-
-.sidebar {
-  position: relative;
-  top: 0;
-  left: 0;
-  height: 100vh;
-  background-color: #333;
-  color: white;
-  overflow: hidden;
-  transition: transform 0.3s ease;
-}
-
-.sidebar.closed {
-  width: 0;
-}
-
-.sidebar.open {
-  width: 250px;
-} */

src/DataStreamSuite/Bts.DataStreamWebApp.Client/src/App.jsx

@@ -1,173 +0,0 @@
-import { ThemeProvider, CssBaseline, GlobalStyles } from '@mui/material';
-import QueryStatusView from './QueryStatusView';
-import useMediaQuery from '@mui/material/useMediaQuery';
-import QueryControl from './QueryControl';
-import DataTable from './DataTable';
-import { useState, useEffect, useMemo } from 'react';
-import useWebSocket, { ReadyState } from 'react-use-websocket';
-import Sidebar from './SideBar';
-import getTheme from './Theme';
-
-import '@fontsource/noto-sans';
-import './App.css';
-import './SideBar.css';
-
-const webSocketUrl = import.meta.env.VITE_DATASTREAM_WEBSOCKET_URL;
-// Always use the same subscription id (1), because we only show 1 table at a time.
-// The server will automatically dispose the previous subscription and start a new one
-// if we're currently subscribed to another query.
-const subscriptionId = "1";
-
-var rpcReqIdCounter = 1;
-
-function subscribeReq(query) {
-  return {
-    jsonrpc: "2.0",
-    id: rpcReqIdCounter++,
-    method: "subscribe",
-    params: {
-      subscription_id: subscriptionId,
-      query: query
-    }
-  };
-}
-
-function unsubscribeReq() {
-  return {
-    jsonrpc: "2.0",
-    id: rpcReqIdCounter++,
-    method: "unsubscribe",
-    params: {
-      subscription_id: subscriptionId,
-    }
-  };
-}
-
-const App = () => {
-  const prefersDarkMode = useMediaQuery('(prefers-color-scheme: dark)');
-  const theme = getTheme({ mode: prefersDarkMode ? 'dark' : 'light' });
-  const globalStyles = {
-    ':root': {
-      '--primary-color': theme.palette.primary.main,
-      '--primary-dark-color': theme.palette.primary.dark,
-      '--secondary-color': theme.palette.secondary.main,
-      '--secondary-dark-color': theme.palette.secondary.dark,
-      '--background-color': theme.palette.background.default,
-    },
-  };
-  const [blueTable, setBlueTable] = useState({ columns: null, rows: null });
-  const [queryError, setQueryError] = useState(null);
-  const [queryState, setQueryState] = useState("Unsubscribed");
-  const queryStateDetails = useMemo(() => {
-    if (queryState == "Unsubscribed") {
-      return "Enter query above, then click the play button to subscribe";
-    }
-    if (queryState == "Subscribed") {
-      return null;
-    }
-  }, [queryState]);
-  const [sidebarIsOpen, setSidebarIsOpen] = useState(false);
-  const [cannedQuery, setCannedQuery] = useState(null);
-
-  const onCannedQueryClicked = (cannedQuery) => {
-    console.info(`Executing canned query: ${JSON.stringify(cannedQuery)}`);
-    const queryText = cannedQuery.query.join("\n");
-    setCannedQuery({ queryText: queryText, clickTime: Date.now() });
-    subscribe(queryText);
-  };
-
-  const { sendJsonMessage, getWebSocket, readyState } = useWebSocket(
-    webSocketUrl,
-    {
-      onOpen: (e) => console.info(`WebSocket open. ${JSON.stringify(e)}`),
-      onClose: (e) => console.info(`WebSocket closed. ${JSON.stringify(e)}`),
-      onError: (e) => console.error(`WebSocket error: ${JSON.stringify(e)}`),
-      shouldReconnect: () => true,
-    });
-
-  useEffect(() => {
-    console.info(`WebSocket readyState: ${readyState}`)
-    if (readyState != ReadyState.OPEN) {
-      return;
-    }
-    const ws = getWebSocket();
-    ws.onmessage = e => {
-      console.info(`Received message: ${e.data}`);
-      const msg = JSON.parse(e.data);
-      setQueryError(msg.error);
-      if (msg.error != null) {
-        setQueryState("Error");
-      } else if (msg.id != null) {
-        setQueryState(msg.result.status);
-      } else {
-        console.assert(msg.params.subscription_id == subscriptionId);
-        console.info(`Received rows: ${msg.params.table.all}`);
-        setBlueTable({ columns: msg.params.table.columns, rows: msg.params.table.all });
-      }
-    };
-  }, [getWebSocket, readyState]);
-
-  const subscribe = (query) => {
-    const req = subscribeReq(query);
-    console.info(`Subscribing: ${JSON.stringify(subscribeReq(query))}`);
-    sendJsonMessage(req);
-  };
-
-  const unsubscribe = () => {
-    const req = unsubscribeReq();
-    console.info(`Unsubscribing: ${JSON.stringify(req)}`);
-    sendJsonMessage(req);
-  };
-
-  const onOpened = () => {
-    setSidebarIsOpen(true);
-  };
-
-  const onClosed = () => {
-    setSidebarIsOpen(false);
-  };
-
-  const sidebarWidth = sidebarIsOpen ? '250px' : '65px';
-  return (
-    <ThemeProvider theme={theme}>
-      <CssBaseline />
-      <GlobalStyles styles={globalStyles} />
-      <div className='root-div'>
-        <div style={{
-          display: 'flex',
-          flexDirection: 'column',
-          alignItems: 'flex-start',
-          flexWrap: 'nowrap',
-          width: sidebarWidth,
-          transition: 'width 0.3s ease',
-        }}>
-          <Sidebar onSelection={onCannedQueryClicked} onOpened={onOpened} onClosed={onClosed} />
-        </div>
-        <div style={{
-          display: 'flex',
-          flexDirection: 'column',
-          left: sidebarWidth,
-          width: '100%',
-          transition: 'width 0.3s ease',
-        }}>
-          <div style={{ width: '100%', display: 'flex', flexDirection: 'column', alignItems: 'flex-start', margin: theme.spacing(2, 0, 0, 0) }}>
-            <div style={{ width: '90%', rightMargin: '100px' }}>
-              <QueryControl
-                queryState={queryState}
-                queryText={cannedQuery?.queryText}
-                queryTextResetTime={cannedQuery?.clickTime}
-                subscribe={subscribe}
-                unsubscribe={unsubscribe} />
-            </div>
-            <div style={{ width: '90%', rightMargin: '100px' }}>
-              <QueryStatusView queryState={queryState} details={queryStateDetails} error={queryError} />
-              <DataTable columns={blueTable?.columns ?? []} rows={blueTable?.rows ?? []} />
-            </div>
-          </div>
-        </div>
-      </div>
-    </ThemeProvider>
-  );
-};
-
-export default App;

src/DataStreamSuite/Bts.DataStreamWebApp.Client/src/DataGrid.css

@@ -1,53 +0,0 @@
-@use "@material/checkbox";
-@use "@material/icon-button";
-@use "@material/data-table/data-table";
-
-.data-grid-container {
-  overflow: auto;
-  white-space: nowrap;
-}
-
-
-.data-grid {
-  max-height: 1000px;
-}
-
-div[role=columnheader] {
-  border-top: 0px;
-  border-bottom: 0px;
-  background-color: var(--primary-dark-color);
-}
-
-/* div[role=columnheader] {
-  border-top: 0px;
-  border-bottom: 0px;
-  background-color: var(--primary-dark-color);
-}
-
-.rdg-row {
-  border: 0px;
-  border-top: 0px;
-  border-bottom: 0px;
-  background-color: rgba(var(--primary-dark-color), 0.8);
-  color: var(--primary-color);
-}
-
-.rdg-row-odd {
-  border: 0px;
-  border-top: 0px;
-  border-bottom: 0px;
-  background-color: var(--primary-dark-color);
-  color: var(--primary-color);
-}
-
-.rdg-cell {
-  border-bottom: 0px;
-}
-
-.rdg-cell.even {
-  border-bottom: 5px;
-}
-
-.rdg-cell.odd {
-  border-bottom: 5px;
-} */

src/DataStreamSuite/Bts.DataStreamWebApp.Client/src/DataTable.css

@@ -1,32 +0,0 @@
-@use "@material/checkbox";
-@use "@material/icon-button";
-@use "@material/data-table/data-table";
-
-th {
-  font-size: large;  
-  padding: 15px;
-}
-
-td {
-  padding: 15px;
-}
-
-td:hover {
-  background-color: cadetblue;
-}
-
-td#cell-decimal {
-  text-align: right;
-}
-
-td#cell-integer {
-  text-align: right;
-}
-
-td#cell-string {
-  text-align: center;
-}
-
-td#cell-date-time {
-  text-align: center;
-}

src/DataStreamSuite/Bts.DataStreamWebApp.Client/src/DataTable.jsx

@@ -1,74 +0,0 @@
-import PropTypes from 'prop-types';
-import { useTheme } from '@mui/material/styles';
-import { DataGrid } from '@mui/x-data-grid';
-import './DataGrid.css'
-import { useMemo, useState } from 'react';
-
-export default function DataTable({ columns, rows }) {
-  const theme = useTheme();
-  const [columnWidths, setColumnWidths] = useState({});
-
-  const handleColumnResize = (params) => {
-    const newWidth = params.width;
-    setColumnWidths((prevWidths) => ({
-      ...prevWidths,
-      [params.colDef.field]: params.width,
-    }));
-  };
-
-  const getGridColDef = useMemo(() => {
-    return col => {
-      return {
-        field: col.name,
-        headerName: col.name,
-        width: columnWidths[col.name],
-      };
-    }
-  }, [columnWidths]);
-
-  return (
-    <DataGrid
-      className='data-grid-container'
-      autosizeOnMount={false}
-      columns={columns?.map(getGridColDef) ?? []}
-      onColumnResize={handleColumnResize}
-      rows={rows ?? []}
-      sx={{
-        margin: theme.spacing(2, 0, 0, 0),
-        borderColor: 'primary.main',
-        backgroundColor: 'primary.dark',
-        color: 'text.primary',
-        '& .MuiDataGrid-columnHeader': {
-          color: 'primary.main',
-          fontWeight: 700,
-          backgroundColor: 'primary.dark'
-        },
-        '& .MuiDataGrid-filler': {
-          color: 'primary.main',
-          fontWeight: 700,
-          backgroundColor: 'primary.dark'
-        },
-        '& .MuiDataGrid-cell': {
-          color: 'text.primary',
-        },
-        '& .MuiDataGrid-cell:hover': {
-          color: 'primary.main',
-        },
-        '& .MuiDataGrid-row': {
-          '--DataGrid-rowBorderColor': 'secondary.dark'
-        }
-      }}
-    />
-  );
-}
-
-DataTable.propTypes = {
-  columns: PropTypes.arrayOf(
-    PropTypes.shape({
-      name: PropTypes.string.isRequired,
-      type: PropTypes.string.isRequired,
-    })
-  ).isRequired,
-  rows: PropTypes.arrayOf(PropTypes.array).isRequired,
-  apiRef: PropTypes.object,
-};

src/DataStreamSuite/Bts.DataStreamWebApp.Client/src/QueryControl.jsx

@@ -1,119 +0,0 @@
-import PropTypes from 'prop-types';
-import DataTable from './DataTable';
-import { useRef, useState, useEffect } from 'react';
-import { useGridApiRef } from '@mui/x-data-grid';
-import Box from '@mui/material/Box';
-import Button from '@mui/material/Button';
-import Grid from '@mui/material/Grid';
-import Paper from '@mui/material/Paper';
-import PlayArrowIcon from '@mui/icons-material/PlayArrow';
-import StopIcon from '@mui/icons-material/Stop';
-import TextField from '@mui/material/TextField';
-import Tooltip from '@mui/material/Tooltip';
-
-const QueryControl = (props) => {
-  const { queryState, queryText, queryTextResetTime, subscribe, unsubscribe } = props;
-  // const theme = useTheme();
-  let textInput = useRef();
-  const [queryTextChangeTime, setQueryTextChangeTime] = useState(null);
-  useEffect(() => {
-    if (queryText == null || textInput.current == null) {
-      return;
-    }
-    // Init text field (unless a non-empty string is already present, or the user has modified
-    // the query - we don't want to overwrite their changes).
-    if (queryTextChangeTime == null || queryTextResetTime > queryTextChangeTime) {
-      console.info(`Initializing query text: ${queryText}`)
-      textInput.current.value = queryText;
-    }
-  }, [queryText, queryTextChangeTime, queryTextResetTime]);
-  const onClearClick = () => {
-    textInput.current.value = '';
-  };
-
-  const onRunClick = () => {
-    subscribe(textInput.current.value);
-  };
-
-  const onStopClick = () => {
-    unsubscribe();
-  };
-
-  const onKeyDown = (e) => {
-    // Detect keyboard shortcuts.
-    //   Ctrl+Enter : Run
-    if (e.ctrlKey && e.key === 'Enter') {
-      onRunClick();
-    }
-  };
-
-  const onChange = (e) => {
-    // So we know not to overwrite the changes with the canned query on re-render
-    setQueryTextChangeTime(Date.now());
-  };
-
-  return (
-    <Box sx={{ display: "flex", flexFlow: "column" }}>
-      <Paper elevation={3} sx={{ color: 'primary.main' }}>
-        <Box sx={{ p: 2 }}>
-          <TextField
-            fullWidth
-            multiline
-            rows={10}
-            variant="outlined"
-            onChange={onChange}
-            onKeyDown={onKeyDown}
-            inputRef={textInput}
-            placeholder="Enter Query..."
-          />
-        </Box>
-        <Box sx={{ p: 2, borderTop: 1 }}>
-          <Grid container spacing={2} justifyContent="flex-start">
-            <Grid item xs>
-              <Tooltip title="Clear Query Text">
-                <Button onClick={onClearClick} sx={{ padding: '11px', color: 'text.dark' }}>
-                  Clear
-                </Button>
-              </Tooltip>
-            </Grid>
-            <Grid item>
-              <Tooltip title="Run (Ctrl+Enter)">
-                <Button onClick={onRunClick} sx={{
-                  color: `queryStartButton.${queryState}.color`,
-                  bgcolor: `queryStartButton.${queryState}.backgroundColor`,
-                  padding: '11px'
-                }}>
-                  <PlayArrowIcon />
-                </Button>
-              </Tooltip>
-            </Grid>
-            <Grid item>
-              <Tooltip title="Stop">
-                <Button onClick={onStopClick} sx={{
-                  color: `queryStopButton.${queryState}.color`,
-                  bgcolor: `queryStopButton.${queryState}.backgroundColor`,
-                  padding: '11px'
-                }}>
-                  <StopIcon />
-                </Button>
-              </Tooltip>
-            </Grid>
-          </Grid>
-        </Box>
-      </Paper>
-    </Box>
-  );
-};
-
-QueryControl.propTypes = {
-  columns: PropTypes.array,
-  rows: PropTypes.array,
-  isRunning: PropTypes.bool,
-  queryText: PropTypes.string,
-  queryTextResetTime: PropTypes.number,
-  queryState: PropTypes.string,
-  subscribe: PropTypes.func,
-  unsubscribe: PropTypes.func,
-};
-
-export default QueryControl;

src/DataStreamSuite/Bts.DataStreamWebApp.Client/src/QueryStatusView.jsx

@@ -1,73 +0,0 @@
-import { useState } from 'react';
-import PropTypes from 'prop-types';
-import { Box, Typography, Collapse } from '@mui/material';
-import Alert from '@mui/material/Alert';
-import AlertTitle from '@mui/material/AlertTitle';
-import { useTheme } from '@mui/material/styles';
-import { alpha } from '@mui/system';
-
-const QueryStatusView = ({ queryState, details, error }) => {
-  const theme = useTheme();
-  const [open, setOpen] = useState(false);
-
-  let header = queryState;
-  let headerContent = null;
-  let detailElements = [];
-  let severity = queryState == "Subscribed" ? "success" : "info";
-  let cursor = details == null ? "default" : "pointer";
-  if (error != null) {
-    severity = "error";
-    cursor = "pointer";
-    header = `Error Code: ${error.code}`;
-    headerContent = error.message;
-    detailElements.push(<Typography variant="body1">{error.data.error_message}</Typography>);
-    detailElements.push(<Typography variant="subtitle1">Stack Trace:</Typography>);
-    (error.data.stack_trace ?? []).forEach(line => detailElements.push(
-      <Typography variant="body2" component="pre" sx={{ whiteSpace: 'pre-wrap' }}>
-        {line}
-      </Typography>));
-  } else if (details != null) {
-    detailElements.push(<Typography variant="body1">{details}</Typography>);
-  }
-  return (
-    <Box sx={{ margin: theme.spacing(2, 0, 0, 0) }}>
-      <Alert
-        severity={severity}
-        onClick={() => setOpen(!open)}
-        sx={{
-          margin: theme.spacing(2, 0, 0, 0),
-          cursor: cursor,
-        }}>
-        <AlertTitle>{header}</AlertTitle>
-        {headerContent}
-      </Alert>
-      {detailElements.length == 0
-        ? null
-        : <Collapse in={open}>
-          <Box
-            sx={{
-              backgroundColor: alpha(theme.palette[severity].light, 0.75),
-              color: theme.palette[severity].contrastText,
-              padding: theme.spacing(2),
-              marginTop: theme.spacing(0),
-              borderRadius: theme.shape.borderRadius,
-            }}
-          >
-            {detailElements}
-          </Box>
-        </Collapse>}
-    </Box>
-  );
-};
-
-QueryStatusView.propTypes = {
-  queryState: PropTypes.string,
-  details: PropTypes.string,
-  error: PropTypes.shape({
-    code: PropTypes.number,
-    message: PropTypes.string,
-    data: PropTypes.object,
-  })
-};
-
-export default QueryStatusView;

src/DataStreamSuite/Bts.DataStreamWebApp.Client/src/ReactDataGrid.jsx

@@ -1,27 +0,0 @@
-import 'react-data-grid/lib/styles.css';
-import PropTypes from 'prop-types';
-import DataGrid from 'react-data-grid';
-import './DataGrid.css';
-
-function getGridColDef(col) {
-  return {
-    key: col.name,
-    name: col.name,
-  };
-}
-
-export default function ReactDataGrid({columns, rows}) {
-  return (
-    <DataGrid className={"data-grid"} rows={rows} columns={columns.map(getGridColDef)} />
-  )
-}
-
-ReactDataGrid.propTypes = {
-  columns: PropTypes.arrayOf(
-    PropTypes.shape({
-      name: PropTypes.string.isRequired,
-      type: PropTypes.string.isRequired,
-    })
-  ).isRequired,
-  rows: PropTypes.arrayOf(PropTypes.array).isRequired,
-};

src/DataStreamSuite/Bts.DataStreamWebApp.Client/src/SideBar.css

@@ -1,98 +0,0 @@
-/* Sidebar.css */
-.app-container {
-  display: flex;
-  width: 100%;
-  overflow: hidden;
-  flex-direction: row;
-  justify-content: flex-start;
-  align-items: flex-start;
-}
-
-.sidebar {
-  display: flex;
-  flex-direction: row;
-  justify-content: flex-start;
-  align-items: flex-start;
-  background-color: transparent;
-  color: white;
-  overflow: hidden;
-  transition: width 0.3s ease;
-}
-
-.sidebar.open {
-  width: 100%;
-}
-
-.sidebar.closed {
-  width: 0%;
-}
-
-.sidebar-toggle-button {
-  border: 0px;
-  border-radius: 50%;
-  box-shadow: rgba(9, 30, 66, 0.08) 0px 0px 0px 1px, rgba(9, 30, 66, 0.08) 0px 2px 4px 1px;
-  color: var(--primary-color);
-  cursor: pointer;
-  height: 24px;
-  opacity: 1;
-  outline: 0px;
-  padding: 0px;
-  background-color: transparent;
-  transition: background-color 100ms linear 0s, color 100ms linear 0s, opacity 300ms cubic-bezier(0.2, 0, 0, 1) 0s, transform 300ms cubic-bezier(0.2, 0, 0, 1) 0s;
-  width: 20%;
-}
-
-.sidebar-toggle-button:hover {
-  transition: background-color 100ms linear 0s, color 100ms linear 0s, opacity 300ms cubic-bezier(0.2, 0, 0, 1) 0s, transform 300ms cubic-bezier(0.2, 0, 0, 1) 0s;
-}
-
-.main-content.open {
-  width: 165px;
-  transition: width 0.3s ease;
-  background-color: transparent;
-}
-
-.main-content.closed {
-  width: 0%;
-  /* min-width: 25px; */
-  transition: width 0.3s ease;
-  background-color: transparent;
-}
-
-.sidebar-menu-item {
-  width: 100%;
-  box-sizing: border-box;
-}
-
-.sidebar-menu-item.open {
-  width: 100%;
-  /* justify-content: flex-start; */
-  transition: width 0.3s ease;
-}
-
-.sidebar-menu-item.closed {
-  width: 0%;
-  min-width: 0px;
-  /* justify-content: flex-start; */
-  transition: width 0.3s ease;
-}
-
-.sidebar-menu-item-content.open {
-  width: 100%;
-  /* justify-content: flex-start; */
-  transition: width 0.3s ease;
-}
-
-.sidebar-menu-item-content.closed {
-  width: 0px;
-  /* justify-content: flex-start; */
-  transition: width 0.3s ease;
-}
-
-/* Ensure span can shrink */
-.button-text {
-  display: block;    /* Make span a block element */
-  width: 100%;       /* Ensure it takes the full width of its parent */
-  overflow: hidden;  /* Hide any overflow content */
-  white-space: nowrap; /* Prevent text wrapping */
-}

src/DataStreamSuite/Bts.DataStreamWebApp.Client/src/SideBar.jsx

@@ -1,109 +0,0 @@
-import { useState, useEffect } from 'react';
-import Button from '@mui/material/Button';
-import { useTheme } from '@mui/material/styles';
-import PropTypes from 'prop-types';
-import Tooltip from '@mui/material/Tooltip';
-import ChevronLeftRounded from "@mui/icons-material/ChevronLeftRounded"
-import ChevronRightRounded from '@mui/icons-material/ChevronRightRounded';
-import FileCopyIcon from '@mui/icons-material/FileCopy';
-import './SideBar.css';
-
-const Sidebar = ({ onSelection, onOpened, onClosed }) => {
-  const theme = useTheme();
-  const [isOpen, setIsOpen] = useState(false);
-  const [cannedQueries, setCannedQueries] = useState([]);
-  const Icon = isOpen ? ChevronLeftRounded : ChevronRightRounded;
-
-  const toggleIsOpen = () => {
-    if (isOpen) {
-      setIsOpen(false);
-      if (onClosed != null) {
-        onClosed();
-      }
-    } else {
-      setIsOpen(true);
-      if (onOpened != null) {
-        onOpened();
-      }
-    }
-  };
-
-  const onMenuItemClick = (event, cannedQuery) => {
-    console.info(`menu item clicked: ${cannedQuery.name}`);
-    onSelection(cannedQuery);
-  };
-
-  // const onMenuItemClick = async (event, cannedQuery) => {
-  //   try {
-  //     await navigator.clipboard.writeText(cannedQuery.query.join("\n"));
-  //     alert(`Copied ${cannedQuery.name} to clipboard`);
-  //   } catch (err) {
-  //     console.error('Failed to copy query text: ', err);
-  //   }
-  // };
-
-  useEffect(() => {
-    const fetchCannedQueries = async () => {
-      try {
-        const response = await fetch('api/cannedqueries');
-        const data = await response.json();
-        setCannedQueries(data);
-      } catch (error) {
-        console.error('Error fetching WebSocket config:', error);
-      }
-    }
-    fetchCannedQueries();
-  }, []);
-
-  const mainContentClass = `main-content ${isOpen ? 'open' : 'closed'}`;
-  const menuItemClass = `sidebar-menu-item ${isOpen ? 'open' : 'closed'}`;
-  const menuItemContentClass = `sidebar-menu-item-content ${isOpen ? 'open' : 'closed'}`;
-  const toggleButtonTitle = `${isOpen ? 'Hide' : 'Show'} Common Queries`;
-
-  return (
-    <div
-      className={"sidebar"}
-      style={{
-        display: 'flex',
-        flexDirection: 'row',
-        justifyContent: 'flexStart',
-        alignItems: 'flexStart',
-        margin: theme.spacing(2, 0, 0, 0)
-      }}
-    >
-      <div className={mainContentClass}>
-        {cannedQueries.map(cq =>
-          <Button key={cq.name} className={menuItemClass} onClick={(event) => onMenuItemClick(event, cq)}>
-            <div className={menuItemContentClass}>
-              <span className='button-text'>{cq.name}</span>
-            </div>
-          </Button>)
-        }
-        {/* {cannedQueries.map(cq =>
-          <Button
-            variant="contained"
-            color="primary"
-            startIcon={<FileCopyIcon />}
-            onClick={onMenuItemClick}
-          >
-            <div className={menuItemContentClass}>
-              <span className='button-text'>{cq.name}</span>
-            </div>
-          </Button>)} */}
-      </div>
-      <Tooltip title={toggleButtonTitle}>
-        <Button onClick={toggleIsOpen}>
-          <Icon className="w-5 h-5" />
-        </Button>
-      </Tooltip>
-    </div>
-  );
-};
-
-export default Sidebar;
-
-Sidebar.propTypes = {
-  onSelection: PropTypes.func,
-  onOpened: PropTypes.func,
-  onClosed: PropTypes.func,
-};

src/DataStreamSuite/Bts.DataStreamWebApp.Client/src/Theme.js

@@ -1,112 +0,0 @@
-import { createTheme } from '@mui/material/styles';
-import PropTypes from 'prop-types';
-import '@fontsource/noto-sans';
-import './App.css';
-
-const getLightPalette = () => {
-  return {
-    mode: 'light',
-    primary: {
-      light: '#6573c3',
-      main: '#3f51b5',
-      dark: '#2c387e',
-      contrastText: '#ffffff',
-    },
-    secondary: {
-      light: '#f73378',
-      main: '#f50057',
-      dark: '#ab003c',
-      contrastText: '#ffffff',
-    },
-    text: {
-      primary: '#000000c4',
-      secondary: '#00000086',
-      disabled: '#0000005b',
-      hint: '#0000005b',
-    }
-  };
-};
-
-const getDarkPalette = () => {
-  return {
-    mode: 'dark',
-    primary: {
-      light: '#bcc5ff',
-      main: '#33baf9',
-      dark: '#202334',
-      contrastText: '#78a2ac',
-    },
-    secondary: {
-      light: '#9fd0b8',
-      main: '#5eb289',
-      dark: '#ffffff12',
-      contrastText: '#c8d4f7',
-    },
-    action: {
-      hoverOpacity: 0.08
-    },
-    text: {
-      primary: '#eaf8f8',
-      dark: '#eaf8f8a0'
-    },
-    queryStartButton: {
-      Subscribed: {
-        color: '#33baf9',
-        backgroundColor: '#ffffff40',
-      },
-      Unsubscribed: {
-        color: '#ffffff40',
-        backgroundColor: '#ffffff00',
-      }
-    },
-    queryStopButton: {
-      Subscribed: {
-        color: '#ff4444a0',
-        backgroundColor: '#ffffff40',
-      },
-      Unsubscribed: {
-        color: '#ff444450',
-        backgroundColor: '#ffffff00',
-      }
-    },
-    error: {
-      light: '#e57373',
-      main: '#f44336',
-      dark: '#d32f2f',
-      contrastText: '#fff',
-    },
-    warning: {
-      light: '#ffb74d',
-      main: '#ff9800',
-      dark: '#f57c00',
-      contrastText: '#000',
-    },
-    info: {
-      light: '#64b5f6',
-      main: '#2196f3',
-      dark: '#303030',
-      contrastText: '#fff',
-    },
-    success: {
-      light: '#81c784',
-      main: '#4caf50',
-      dark: '#388e3c',
-      contrastText: '#000',
-    },
-  };
-};
-
-const getTheme = ({ mode }) => {
-  return createTheme({
-    typography: {
-      fontFamily: '"Helvetica"'
-    },
-    palette: mode == 'dark' ? getDarkPalette() : getLightPalette()
-  });
-};
-
-export default getTheme;
-
-getTheme.propTypes = {
-  mode: PropTypes.string.isRequired,
-};

src/DataStreamSuite/Bts.DataStreamWebApp.Client/src/assets/react.svg

@@ -1 +0,0 @@
-<svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" class="iconify iconify--logos" width="35.93" height="32" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 228"><path fill="#00D8FF" d="M210.483 73.824a171.49 171.49 0 0 0-8.24-2.597c.465-1.9.893-3.777 1.273-5.621c6.238-30.281 2.16-54.676-11.769-62.708c-13.355-7.7-35.196.329-57.254 19.526a171.23 171.23 0 0 0-6.375 5.848a155.866 155.866 0 0 0-4.241-3.917C100.759 3.829 77.587-4.822 63.673 3.233C50.33 10.957 46.379 33.89 51.995 62.588a170.974 170.974 0 0 0 1.892 8.48c-3.28.932-6.445 1.924-9.474 2.98C17.309 83.498 0 98.307 0 113.668c0 15.865 18.582 31.778 46.812 41.427a145.52 145.52 0 0 0 6.921 2.165a167.467 167.467 0 0 0-2.01 9.138c-5.354 28.2-1.173 50.591 12.134 58.266c13.744 7.926 36.812-.22 59.273-19.855a145.567 145.567 0 0 0 5.342-4.923a168.064 168.064 0 0 0 6.92 6.314c21.758 18.722 43.246 26.282 56.54 18.586c13.731-7.949 18.194-32.003 12.4-61.268a145.016 145.016 0 0 0-1.535-6.842c1.62-.48 3.21-.974 4.76-1.488c29.348-9.723 48.443-25.443 48.443-41.52c0-15.417-17.868-30.326-45.517-39.844Zm-6.365 70.984c-1.4.463-2.836.91-4.3 1.345c-3.24-10.257-7.612-21.163-12.963-32.432c5.106-11 9.31-21.767 12.459-31.957c2.619.758 5.16 1.557 7.61 2.4c23.69 8.156 38.14 20.213 38.14 29.504c0 9.896-15.606 22.743-40.946 31.14Zm-10.514 20.834c2.562 12.94 2.927 24.64 1.23 33.787c-1.524 8.219-4.59 13.698-8.382 15.893c-8.067 4.67-25.32-1.4-43.927-17.412a156.726 156.726 0 0 1-6.437-5.87c7.214-7.889 14.423-17.06 21.459-27.246c12.376-1.098 24.068-2.894 34.671-5.345a134.17 134.17 0 0 1 1.386 6.193ZM87.276 214.515c-7.882 2.783-14.16 2.863-17.955.675c-8.075-4.657-11.432-22.636-6.853-46.752a156.923 156.923 0 0 1 1.869-8.499c10.486 2.32 22.093 3.988 34.498 4.994c7.084 9.967 14.501 19.128 21.976 27.15a134.668 134.668 0 0 1-4.877 4.492c-9.933 8.682-19.886 14.842-28.658 17.94ZM50.35 144.747c-12.483-4.267-22.792-9.812-29.858-15.863c-6.35-5.437-9.555-10.836-9.555-15.216c0-9.322 13.897-21.212 37.076-29.293c2.813-.98 5.757-1.905 8.812-2.773c3.204 10.42 7.406 21.315 12.477 32.332c-5.137 11.18-9.399 22.249-12.634 32.792a134.718 134.718 0 0 1-6.318-1.979Zm12.378-84.26c-4.811-24.587-1.616-43.134 6.425-47.789c8.564-4.958 27.502 2.111 47.463 19.835a144.318 144.318 0 0 1 3.841 3.545c-7.438 7.987-14.787 17.08-21.808 26.988c-12.04 1.116-23.565 2.908-34.161 5.309a160.342 160.342 0 0 1-1.76-7.887Zm110.427 27.268a347.8 347.8 0 0 0-7.785-12.803c8.168 1.033 15.994 2.404 23.343 4.08c-2.206 7.072-4.956 14.465-8.193 22.045a381.151 381.151 0 0 0-7.365-13.322Zm-45.032-43.861c5.044 5.465 10.096 11.566 15.065 18.186a322.04 322.04 0 0 0-30.257-.006c4.974-6.559 10.069-12.652 15.192-18.18ZM82.802 87.83a323.167 323.167 0 0 0-7.227 13.238c-3.184-7.553-5.909-14.98-8.134-22.152c7.304-1.634 15.093-2.97 23.209-3.984a321.524 321.524 0 0 0-7.848 12.897Zm8.081 65.352c-8.385-.936-16.291-2.203-23.593-3.793c2.26-7.3 5.045-14.885 8.298-22.6a321.187 321.187 0 0 0 7.257 13.246c2.594 4.48 5.28 8.868 8.038 13.147Zm37.542 31.03c-5.184-5.592-10.354-11.779-15.403-18.433c4.902.192 9.899.29 14.978.29c5.218 0 10.376-.117 15.453-.343c-4.985 6.774-10.018 12.97-15.028 18.486Zm52.198-57.817c3.422 7.8 6.306 15.345 8.596 22.52c-7.422 1.694-15.436 3.058-23.88 4.071a382.417 382.417 0 0 0 7.859-13.026a347.403 347.403 0 0 0 7.425-13.565Zm-16.898 8.101a358.557 358.557 0 0 1-12.281 19.815a329.4 329.4 0 0 1-23.444.823c-7.967 0-15.716-.248-23.178-.732a310.202 310.202 0 0 1-12.513-19.846h.001a307.41 307.41 0 0 1-10.923-20.627a310.278 310.278 0 0 1 10.89-20.637l-.001.001a307.318 307.318 0 0 1 12.413-19.761c7.613-.576 15.42-.876 23.31-.876H128c7.926 0 15.743.303 23.354.883a329.357 329.357 0 0 1 12.335 19.695a358.489 358.489 0 0 1 11.036 20.54a329.472 329.472 0 0 1-11 20.722Zm22.56-122.124c8.572 4.944 11.906 24.881 6.52 51.026c-.344 1.668-.73 3.367-1.15 5.09c-10.622-2.452-22.155-4.275-34.23-5.408c-7.034-10.017-14.323-19.124-21.64-27.008a160.789 160.789 0 0 1 5.888-5.4c18.9-16.447 36.564-22.941 44.612-18.3ZM128 90.808c12.625 0 22.86 10.235 22.86 22.86s-10.235 22.86-22.86 22.86s-22.86-10.235-22.86-22.86s10.235-22.86 22.86-22.86Z"></path></svg>

src/DataStreamSuite/Bts.DataStreamWebApp.Client/src/css/dark-hc.css

@@ -1,51 +0,0 @@
-.dark-high-contrast {
-  --md-sys-color-primary: rgb(251 250 255);
-  --md-sys-color-surface-tint: rgb(165 200 255);
-  --md-sys-color-on-primary: rgb(0 0 0);
-  --md-sys-color-primary-container: rgb(173 204 255);
-  --md-sys-color-on-primary-container: rgb(0 0 0);
-  --md-sys-color-secondary: rgb(250 250 255);
-  --md-sys-color-on-secondary: rgb(0 0 0);
-  --md-sys-color-secondary-container: rgb(159 208 255);
-  --md-sys-color-on-secondary-container: rgb(0 0 0);
-  --md-sys-color-tertiary: rgb(255 249 251);
-  --md-sys-color-on-tertiary: rgb(0 0 0);
-  --md-sys-color-tertiary-container: rgb(222 193 231);
-  --md-sys-color-on-tertiary-container: rgb(0 0 0);
-  --md-sys-color-error: rgb(255 249 249);
-  --md-sys-color-on-error: rgb(0 0 0);
-  --md-sys-color-error-container: rgb(255 186 177);
-  --md-sys-color-on-error-container: rgb(0 0 0);
-  --md-sys-color-background: rgb(17 19 24);
-  --md-sys-color-on-background: rgb(225 226 233);
-  --md-sys-color-surface: rgb(17 19 24);
-  --md-sys-color-on-surface: rgb(255 255 255);
-  --md-sys-color-surface-variant: rgb(67 71 78);
-  --md-sys-color-on-surface-variant: rgb(251 250 255);
-  --md-sys-color-outline: rgb(200 202 212);
-  --md-sys-color-outline-variant: rgb(200 202 212);
-  --md-sys-color-shadow: rgb(0 0 0);
-  --md-sys-color-scrim: rgb(0 0 0);
-  --md-sys-color-inverse-surface: rgb(225 226 233);
-  --md-sys-color-inverse-on-surface: rgb(0 0 0);
-  --md-sys-color-inverse-primary: rgb(0 42 83);
-  --md-sys-color-primary-fixed: rgb(219 231 255);
-  --md-sys-color-on-primary-fixed: rgb(0 0 0);
-  --md-sys-color-primary-fixed-dim: rgb(173 204 255);
-  --md-sys-color-on-primary-fixed-variant: rgb(0 22 49);
-  --md-sys-color-secondary-fixed: rgb(214 233 255);
-  --md-sys-color-on-secondary-fixed: rgb(0 0 0);
-  --md-sys-color-secondary-fixed-dim: rgb(159 208 255);
-  --md-sys-color-on-secondary-fixed-variant: rgb(0 24 42);
-  --md-sys-color-tertiary-fixed: rgb(249 222 255);
-  --md-sys-color-on-tertiary-fixed: rgb(0 0 0);
-  --md-sys-color-tertiary-fixed-dim: rgb(222 193 231);
-  --md-sys-color-on-tertiary-fixed-variant: rgb(33 14 42);
-  --md-sys-color-surface-dim: rgb(17 19 24);
-  --md-sys-color-surface-bright: rgb(55 57 62);
-  --md-sys-color-surface-container-lowest: rgb(12 14 19);
-  --md-sys-color-surface-container-low: rgb(25 28 32);
-  --md-sys-color-surface-container: rgb(29 32 36);
-  --md-sys-color-surface-container-high: rgb(40 42 47);
-  --md-sys-color-surface-container-highest: rgb(50 53 58);
-}

src/DataStreamSuite/Bts.DataStreamWebApp.Client/src/css/dark-mc.css

@@ -1,51 +0,0 @@
-.dark-medium-contrast {
-  --md-sys-color-primary: rgb(173 204 255);
-  --md-sys-color-surface-tint: rgb(165 200 255);
-  --md-sys-color-on-primary: rgb(0 22 49);
-  --md-sys-color-primary-container: rgb(112 146 198);
-  --md-sys-color-on-primary-container: rgb(0 0 0);
-  --md-sys-color-secondary: rgb(159 208 255);
-  --md-sys-color-on-secondary: rgb(0 24 42);
-  --md-sys-color-secondary-container: rgb(100 149 193);
-  --md-sys-color-on-secondary-container: rgb(0 0 0);
-  --md-sys-color-tertiary: rgb(222 193 231);
-  --md-sys-color-on-tertiary: rgb(33 14 42);
-  --md-sys-color-tertiary-container: rgb(162 136 171);
-  --md-sys-color-on-tertiary-container: rgb(0 0 0);
-  --md-sys-color-error: rgb(255 186 177);
-  --md-sys-color-on-error: rgb(55 0 1);
-  --md-sys-color-error-container: rgb(255 84 73);
-  --md-sys-color-on-error-container: rgb(0 0 0);
-  --md-sys-color-background: rgb(17 19 24);
-  --md-sys-color-on-background: rgb(225 226 233);
-  --md-sys-color-surface: rgb(17 19 24);
-  --md-sys-color-on-surface: rgb(251 250 255);
-  --md-sys-color-surface-variant: rgb(67 71 78);
-  --md-sys-color-on-surface-variant: rgb(200 202 212);
-  --md-sys-color-outline: rgb(160 163 171);
-  --md-sys-color-outline-variant: rgb(128 131 139);
-  --md-sys-color-shadow: rgb(0 0 0);
-  --md-sys-color-scrim: rgb(0 0 0);
-  --md-sys-color-inverse-surface: rgb(225 226 233);
-  --md-sys-color-inverse-on-surface: rgb(40 42 47);
-  --md-sys-color-inverse-primary: rgb(35 73 120);
-  --md-sys-color-primary-fixed: rgb(212 227 255);
-  --md-sys-color-on-primary-fixed: rgb(0 17 40);
-  --md-sys-color-primary-fixed-dim: rgb(165 200 255);
-  --md-sys-color-on-primary-fixed-variant: rgb(10 55 100);
-  --md-sys-color-secondary-fixed: rgb(206 229 255);
-  --md-sys-color-on-secondary-fixed: rgb(0 18 34);
-  --md-sys-color-secondary-fixed-dim: rgb(154 203 250);
-  --md-sys-color-on-secondary-fixed-variant: rgb(0 57 91);
-  --md-sys-color-tertiary-fixed: rgb(247 216 255);
-  --md-sys-color-on-tertiary-fixed: rgb(28 9 37);
-  --md-sys-color-tertiary-fixed-dim: rgb(218 189 226);
-  --md-sys-color-on-tertiary-fixed-variant: rgb(67 46 76);
-  --md-sys-color-surface-dim: rgb(17 19 24);
-  --md-sys-color-surface-bright: rgb(55 57 62);
-  --md-sys-color-surface-container-lowest: rgb(12 14 19);
-  --md-sys-color-surface-container-low: rgb(25 28 32);
-  --md-sys-color-surface-container: rgb(29 32 36);
-  --md-sys-color-surface-container-high: rgb(40 42 47);
-  --md-sys-color-surface-container-highest: rgb(50 53 58);
-}

src/DataStreamSuite/Bts.DataStreamWebApp.Client/src/css/dark.css

@@ -1,51 +0,0 @@
-.dark {
-  --md-sys-color-primary: rgb(165 200 255);
-  --md-sys-color-surface-tint: rgb(165 200 255);
-  --md-sys-color-on-primary: rgb(0 49 94);
-  --md-sys-color-primary-container: rgb(34 72 118);
-  --md-sys-color-on-primary-container: rgb(212 227 255);
-  --md-sys-color-secondary: rgb(154 203 250);
-  --md-sys-color-on-secondary: rgb(0 51 83);
-  --md-sys-color-secondary-container: rgb(12 74 114);
-  --md-sys-color-on-secondary-container: rgb(206 229 255);
-  --md-sys-color-tertiary: rgb(218 189 226);
-  --md-sys-color-on-tertiary: rgb(61 41 70);
-  --md-sys-color-tertiary-container: rgb(85 63 93);
-  --md-sys-color-on-tertiary-container: rgb(247 216 255);
-  --md-sys-color-error: rgb(255 180 171);
-  --md-sys-color-on-error: rgb(105 0 5);
-  --md-sys-color-error-container: rgb(147 0 10);
-  --md-sys-color-on-error-container: rgb(255 218 214);
-  --md-sys-color-background: rgb(17 19 24);
-  --md-sys-color-on-background: rgb(225 226 233);
-  --md-sys-color-surface: rgb(17 19 24);
-  --md-sys-color-on-surface: rgb(225 226 233);
-  --md-sys-color-surface-variant: rgb(67 71 78);
-  --md-sys-color-on-surface-variant: rgb(195 198 207);
-  --md-sys-color-outline: rgb(141 145 153);
-  --md-sys-color-outline-variant: rgb(67 71 78);
-  --md-sys-color-shadow: rgb(0 0 0);
-  --md-sys-color-scrim: rgb(0 0 0);
-  --md-sys-color-inverse-surface: rgb(225 226 233);
-  --md-sys-color-inverse-on-surface: rgb(46 48 53);
-  --md-sys-color-inverse-primary: rgb(60 96 144);
-  --md-sys-color-primary-fixed: rgb(212 227 255);
-  --md-sys-color-on-primary-fixed: rgb(0 28 58);
-  --md-sys-color-primary-fixed-dim: rgb(165 200 255);
-  --md-sys-color-on-primary-fixed-variant: rgb(34 72 118);
-  --md-sys-color-secondary-fixed: rgb(206 229 255);
-  --md-sys-color-on-secondary-fixed: rgb(0 29 50);
-  --md-sys-color-secondary-fixed-dim: rgb(154 203 250);
-  --md-sys-color-on-secondary-fixed-variant: rgb(12 74 114);
-  --md-sys-color-tertiary-fixed: rgb(247 216 255);
-  --md-sys-color-on-tertiary-fixed: rgb(39 20 48);
-  --md-sys-color-tertiary-fixed-dim: rgb(218 189 226);
-  --md-sys-color-on-tertiary-fixed-variant: rgb(85 63 93);
-  --md-sys-color-surface-dim: rgb(17 19 24);
-  --md-sys-color-surface-bright: rgb(55 57 62);
-  --md-sys-color-surface-container-lowest: rgb(12 14 19);
-  --md-sys-color-surface-container-low: rgb(25 28 32);
-  --md-sys-color-surface-container: rgb(29 32 36);
-  --md-sys-color-surface-container-high: rgb(40 42 47);
-  --md-sys-color-surface-container-highest: rgb(50 53 58);
-}

src/DataStreamSuite/Bts.DataStreamWebApp.Client/src/css/light-hc.css

@@ -1,51 +0,0 @@
-.light-high-contrast {
-  --md-sys-color-primary: rgb(0 34 70);
-  --md-sys-color-surface-tint: rgb(60 96 144);
-  --md-sys-color-on-primary: rgb(255 255 255);
-  --md-sys-color-primary-container: rgb(29 68 114);
-  --md-sys-color-on-primary-container: rgb(255 255 255);
-  --md-sys-color-secondary: rgb(0 36 61);
-  --md-sys-color-on-secondary: rgb(255 255 255);
-  --md-sys-color-secondary-container: rgb(3 70 110);
-  --md-sys-color-on-secondary-container: rgb(255 255 255);
-  --md-sys-color-tertiary: rgb(46 26 55);
-  --md-sys-color-on-tertiary: rgb(255 255 255);
-  --md-sys-color-tertiary-container: rgb(81 59 89);
-  --md-sys-color-on-tertiary-container: rgb(255 255 255);
-  --md-sys-color-error: rgb(78 0 2);
-  --md-sys-color-on-error: rgb(255 255 255);
-  --md-sys-color-error-container: rgb(140 0 9);
-  --md-sys-color-on-error-container: rgb(255 255 255);
-  --md-sys-color-background: rgb(249 249 255);
-  --md-sys-color-on-background: rgb(25 28 32);
-  --md-sys-color-surface: rgb(249 249 255);
-  --md-sys-color-on-surface: rgb(0 0 0);
-  --md-sys-color-surface-variant: rgb(224 226 236);
-  --md-sys-color-on-surface-variant: rgb(32 36 43);
-  --md-sys-color-outline: rgb(63 67 74);
-  --md-sys-color-outline-variant: rgb(63 67 74);
-  --md-sys-color-shadow: rgb(0 0 0);
-  --md-sys-color-scrim: rgb(0 0 0);
-  --md-sys-color-inverse-surface: rgb(46 48 53);
-  --md-sys-color-inverse-on-surface: rgb(255 255 255);
-  --md-sys-color-inverse-primary: rgb(228 236 255);
-  --md-sys-color-primary-fixed: rgb(29 68 114);
-  --md-sys-color-on-primary-fixed: rgb(255 255 255);
-  --md-sys-color-primary-fixed-dim: rgb(0 45 88);
-  --md-sys-color-on-primary-fixed-variant: rgb(255 255 255);
-  --md-sys-color-secondary-fixed: rgb(3 70 110);
-  --md-sys-color-on-secondary-fixed: rgb(255 255 255);
-  --md-sys-color-secondary-fixed-dim: rgb(0 47 77);
-  --md-sys-color-on-secondary-fixed-variant: rgb(255 255 255);
-  --md-sys-color-tertiary-fixed: rgb(81 59 89);
-  --md-sys-color-on-tertiary-fixed: rgb(255 255 255);
-  --md-sys-color-tertiary-fixed-dim: rgb(57 37 66);
-  --md-sys-color-on-tertiary-fixed-variant: rgb(255 255 255);
-  --md-sys-color-surface-dim: rgb(217 218 224);
-  --md-sys-color-surface-bright: rgb(249 249 255);
-  --md-sys-color-surface-container-lowest: rgb(255 255 255);
-  --md-sys-color-surface-container-low: rgb(243 243 250);
-  --md-sys-color-surface-container: rgb(237 237 244);
-  --md-sys-color-surface-container-high: rgb(231 232 238);
-  --md-sys-color-surface-container-highest: rgb(225 226 233);
-}

src/DataStreamSuite/Bts.DataStreamWebApp.Client/src/css/light-mc.css

@@ -1,51 +0,0 @@
-.light-medium-contrast {
-  --md-sys-color-primary: rgb(29 68 114);
-  --md-sys-color-surface-tint: rgb(60 96 144);
-  --md-sys-color-on-primary: rgb(255 255 255);
-  --md-sys-color-primary-container: rgb(83 118 167);
-  --md-sys-color-on-primary-container: rgb(255 255 255);
-  --md-sys-color-secondary: rgb(3 70 110);
-  --md-sys-color-on-secondary: rgb(255 255 255);
-  --md-sys-color-secondary-container: rgb(71 121 163);
-  --md-sys-color-on-secondary-container: rgb(255 255 255);
-  --md-sys-color-tertiary: rgb(81 59 89);
-  --md-sys-color-on-tertiary: rgb(255 255 255);
-  --md-sys-color-tertiary-container: rgb(133 108 141);
-  --md-sys-color-on-tertiary-container: rgb(255 255 255);
-  --md-sys-color-error: rgb(140 0 9);
-  --md-sys-color-on-error: rgb(255 255 255);
-  --md-sys-color-error-container: rgb(218 52 46);
-  --md-sys-color-on-error-container: rgb(255 255 255);
-  --md-sys-color-background: rgb(249 249 255);
-  --md-sys-color-on-background: rgb(25 28 32);
-  --md-sys-color-surface: rgb(249 249 255);
-  --md-sys-color-on-surface: rgb(25 28 32);
-  --md-sys-color-surface-variant: rgb(224 226 236);
-  --md-sys-color-on-surface-variant: rgb(63 67 74);
-  --md-sys-color-outline: rgb(91 95 103);
-  --md-sys-color-outline-variant: rgb(119 123 131);
-  --md-sys-color-shadow: rgb(0 0 0);
-  --md-sys-color-scrim: rgb(0 0 0);
-  --md-sys-color-inverse-surface: rgb(46 48 53);
-  --md-sys-color-inverse-on-surface: rgb(240 240 247);
-  --md-sys-color-inverse-primary: rgb(165 200 255);
-  --md-sys-color-primary-fixed: rgb(83 118 167);
-  --md-sys-color-on-primary-fixed: rgb(255 255 255);
-  --md-sys-color-primary-fixed-dim: rgb(58 93 141);
-  --md-sys-color-on-primary-fixed-variant: rgb(255 255 255);
-  --md-sys-color-secondary-fixed: rgb(71 121 163);
-  --md-sys-color-on-secondary-fixed: rgb(255 255 255);
-  --md-sys-color-secondary-fixed-dim: rgb(43 96 137);
-  --md-sys-color-on-secondary-fixed-variant: rgb(255 255 255);
-  --md-sys-color-tertiary-fixed: rgb(133 108 141);
-  --md-sys-color-on-tertiary-fixed: rgb(255 255 255);
-  --md-sys-color-tertiary-fixed-dim: rgb(107 84 116);
-  --md-sys-color-on-tertiary-fixed-variant: rgb(255 255 255);
-  --md-sys-color-surface-dim: rgb(217 218 224);
-  --md-sys-color-surface-bright: rgb(249 249 255);
-  --md-sys-color-surface-container-lowest: rgb(255 255 255);
-  --md-sys-color-surface-container-low: rgb(243 243 250);
-  --md-sys-color-surface-container: rgb(237 237 244);
-  --md-sys-color-surface-container-high: rgb(231 232 238);
-  --md-sys-color-surface-container-highest: rgb(225 226 233);
-}

src/DataStreamSuite/Bts.DataStreamWebApp.Client/src/css/light.css

@@ -1,51 +0,0 @@
-.light {
-  --md-sys-color-primary: rgb(60 96 144);
-  --md-sys-color-surface-tint: rgb(60 96 144);
-  --md-sys-color-on-primary: rgb(255 255 255);
-  --md-sys-color-primary-container: rgb(212 227 255);
-  --md-sys-color-on-primary-container: rgb(0 28 58);
-  --md-sys-color-secondary: rgb(46 98 140);
-  --md-sys-color-on-secondary: rgb(255 255 255);
-  --md-sys-color-secondary-container: rgb(206 229 255);
-  --md-sys-color-on-secondary-container: rgb(0 29 50);
-  --md-sys-color-tertiary: rgb(110 86 118);
-  --md-sys-color-on-tertiary: rgb(255 255 255);
-  --md-sys-color-tertiary-container: rgb(247 216 255);
-  --md-sys-color-on-tertiary-container: rgb(39 20 48);
-  --md-sys-color-error: rgb(186 26 26);
-  --md-sys-color-on-error: rgb(255 255 255);
-  --md-sys-color-error-container: rgb(255 218 214);
-  --md-sys-color-on-error-container: rgb(65 0 2);
-  --md-sys-color-background: rgb(249 249 255);
-  --md-sys-color-on-background: rgb(25 28 32);
-  --md-sys-color-surface: rgb(249 249 255);
-  --md-sys-color-on-surface: rgb(25 28 32);
-  --md-sys-color-surface-variant: rgb(224 226 236);
-  --md-sys-color-on-surface-variant: rgb(67 71 78);
-  --md-sys-color-outline: rgb(116 119 127);
-  --md-sys-color-outline-variant: rgb(195 198 207);
-  --md-sys-color-shadow: rgb(0 0 0);
-  --md-sys-color-scrim: rgb(0 0 0);
-  --md-sys-color-inverse-surface: rgb(46 48 53);
-  --md-sys-color-inverse-on-surface: rgb(240 240 247);
-  --md-sys-color-inverse-primary: rgb(165 200 255);
-  --md-sys-color-primary-fixed: rgb(212 227 255);
-  --md-sys-color-on-primary-fixed: rgb(0 28 58);
-  --md-sys-color-primary-fixed-dim: rgb(165 200 255);
-  --md-sys-color-on-primary-fixed-variant: rgb(34 72 118);
-  --md-sys-color-secondary-fixed: rgb(206 229 255);
-  --md-sys-color-on-secondary-fixed: rgb(0 29 50);
-  --md-sys-color-secondary-fixed-dim: rgb(154 203 250);
-  --md-sys-color-on-secondary-fixed-variant: rgb(12 74 114);
-  --md-sys-color-tertiary-fixed: rgb(247 216 255);
-  --md-sys-color-on-tertiary-fixed: rgb(39 20 48);
-  --md-sys-color-tertiary-fixed-dim: rgb(218 189 226);
-  --md-sys-color-on-tertiary-fixed-variant: rgb(85 63 93);
-  --md-sys-color-surface-dim: rgb(217 218 224);
-  --md-sys-color-surface-bright: rgb(249 249 255);
-  --md-sys-color-surface-container-lowest: rgb(255 255 255);
-  --md-sys-color-surface-container-low: rgb(243 243 250);
-  --md-sys-color-surface-container: rgb(237 237 244);
-  --md-sys-color-surface-container-high: rgb(231 232 238);
-  --md-sys-color-surface-container-highest: rgb(225 226 233);
-}

src/DataStreamSuite/Bts.DataStreamWebApp.Client/src/index.css

@@ -1,107 +0,0 @@
-:root {
-  font-family: Inter, system-ui, Avenir, Helvetica, Arial, sans-serif;
-  line-height: 1.5;
-  font-weight: 400;
-
-  color-scheme: light dark;
-  color: rgba(255, 255, 255, 0.87);
-  background-color: #242424;
-
-  font-synthesis: none;
-  text-rendering: optimizeLegibility;
-  -webkit-font-smoothing: antialiased;
-  -moz-osx-font-smoothing: grayscale;
-}
-
-a {
-  font-weight: 500;
-  color: #646cff;
-  text-decoration: inherit;
-}
-a:hover {
-  color: #535bf2;
-}
-
-body {
-  margin: 0;
-  display: flex;
-  place-items: center;
-  min-width: 320px;
-  min-height: 100vh;
-}
-
-button {
-  border-radius: 8px;
-  border: 1px solid transparent;
-  padding: 0.6em 1.2em;
-  font-size: 1em;
-  font-weight: 500;
-  font-family: inherit;
-  background-color: #1a1a1a;
-  cursor: pointer;
-  transition: border-color 0.25s;
-}
-button:hover {
-  border-color: #646cff;
-}
-button:focus,
-button:focus-visible {
-  outline: 4px auto -webkit-focus-ring-color;
-}
-
-button#query-button {
-  height: 5em;
-  vertical-align: top;
-}
-
-button#play-button {
-  padding: '10px 20px';
-  font-size: '16px';
-  cursor: 'pointer';
-  background-color: '#28a745';
-  color: '#fff';
-  border: 'none';
-  border-radius: '5px';
-};
-
-button#pause-button {
-  background-color: #ffc107;
-}
-
-button#stop-button {
-  background-color: #dc3545;
-}
-
-h1 {
-  font-size: 3.2em;
-  line-height: 1.1;
-}
-
-p#footer {
-  float: none;
-  clear: both;
-  vertical-align: bottom;
-}
-
-table#query-output {
-  float: none;
-  clear: both;
-}
-
-textarea#query-input {
-  height: 5em;
-  width: 50em;
-}
-
-@media (prefers-color-scheme: light) {
-  :root {
-    color: #213547;
-    background-color: #ffffff;
-  }
-  a:hover {
-    color: #747bff;
-  }
-  button {
-    background-color: #f9f9f9;
-  }
-}

src/DataStreamSuite/Bts.DataStreamWebApp.Client/src/main.jsx

@@ -1,7 +0,0 @@
-import React from 'react';
-import ReactDOM from 'react-dom/client';
-import App from './App';
-
-ReactDOM.createRoot(document.getElementById('root')).render(
-  <App />
-)

src/DataStreamSuite/Bts.DataStreamWebApp.Client/vite.config.js

@@ -1,69 +0,0 @@
-import { fileURLToPath, URL } from 'node:url';
-import { defineConfig } from 'vite';
-import plugin from '@vitejs/plugin-react';
-import fs from 'fs';
-import path from 'path';
-import child_process from 'child_process';
-import process, { env } from 'process';
-
-const baseFolder =
-  process.env.APPDATA !== undefined && process.env.APPDATA !== ''
-    ? `${process.env.APPDATA}/ASP.NET/https`
-    : `${process.env.HOME}/.aspnet/https`;
-
-const certificateName = "bts.datastreamwebapp.client";
-const certFilePath = path.join(baseFolder, `${certificateName}.pem`);
-const keyFilePath = path.join(baseFolder, `${certificateName}.key`);
-
-if (!fs.existsSync(certFilePath) || !fs.existsSync(keyFilePath)) {
-  if (0 !== child_process.spawnSync('dotnet', [
-    'dev-certs',
-    'https',
-    '--export-path',
-    certFilePath,
-    '--format',
-    'Pem',
-    '--no-password',
-  ], { stdio: 'inherit', }).status) {
-    throw new Error("Could not create certificate.");
-  }
-}
-
-const target = env.ASPNETCORE_HTTPS_PORT
-  ? `https://localhost:${env.ASPNETCORE_HTTPS_PORT}`
-  : env.ASPNETCORE_URLS
-    ? env.ASPNETCORE_URLS.split(';')[0]
-    : 'https://localhost:8443';
-
-export default defineConfig(({ $$command, mode, $$isSsrBuild, $$isPreview }) => {
-  return {
-    plugins: [plugin()],
-    resolve: {
-      alias: {
-        '@': fileURLToPath(new URL('./src', import.meta.url))
-      }
-    },
-    build: {
-      minify: false,
-      sourcemap: true,
-      outDir: `../../../build/Bts.DataStreamWebApp.Client/${mode}/dist`
-    },
-    server: {
-      proxy: {
-        '^/api': {
-          target,
-          secure: false
-        },
-        '^/ws': {
-          target,
-          secure: false
-        }
-      },
-      port: 5173,
-      https: {
-        key: fs.readFileSync(keyFilePath),
-        cert: fs.readFileSync(certFilePath),
-      }
-    }
-  };
-});

src/DataStreamSuite/Bts.DataStreamWebApp.Server/Bts.DataStreamWebApp.Server.csproj

@@ -1,40 +0,0 @@
-<Project Sdk="Microsoft.NET.Sdk.Web">
-
-  <PropertyGroup>
-    <TargetFramework>net7.0</TargetFramework>
-    <Nullable>enable</Nullable>
-    <ImplicitUsings>enable</ImplicitUsings>
-    <InvariantGlobalization>true</InvariantGlobalization>
-    <SpaRoot>..\Bts.DataStreamWebApp.Client</SpaRoot>
-    <SpaProxyLaunchCommand>npm run dev</SpaProxyLaunchCommand>
-    <SpaProxyServerUrl>https://localhost:5173</SpaProxyServerUrl>
-  </PropertyGroup>
-
-  <ItemGroup>
-    <PackageReference Include="CommandLineParser" Version="2.9.1" />
-    <PackageReference Include="Microsoft.AspNetCore.SpaProxy" Version="7.0.20" />
-    <PackageReference Include="Microsoft.AspNetCore.SpaServices.Extensions" Version="7.0.20" />
-    <PackageReference Include="Newtonsoft.Json" Version="13.0.3" />
-    <PackageReference Include="Swashbuckle.AspNetCore" Version="6.4.0" />
-  </ItemGroup>
-
-  <ItemGroup>
-    <ProjectReference Include="..\Bts.DataStreamWebApp.Client\Bts.DataStreamWebApp.Client.esproj">
-      <ReferenceOutputAssembly>false</ReferenceOutputAssembly>
-    </ProjectReference>
-    <ProjectReference Include="..\DataStream\DataStream.csproj" />
-    <ProjectReference Include="..\SharedStreams\SharedStreams.csproj" />
-    <ProjectReference Include="..\..\client\spark_api\spark_api.csproj" />
-    <ProjectReference Include="..\..\client\spark_core\spark_core.csproj" />
-    <ProjectReference Include="..\..\client\spark_websocket\spark_websocket.csproj" />
-  </ItemGroup>
-
-  <ItemGroup>
-    <ConfigurationProfile Include="appsettings.Development.json" />
-    <ConfigurationProfile Include="appsettings.Production.json" />
-    <ConfigurationProfile Include="appsettings.ProductionAws.json" />
-    <ConfigurationProfile Include="appsettings.Staging.json" />
-    <ConfigurationProfile Include="appsettings.json" />
-  </ItemGroup>
-
-</Project>

src/DataStreamSuite/Bts.DataStreamWebApp.Server/CannedQueriesConfig.cs

@@ -1,5 +0,0 @@
-namespace Bts.DataStreamWebApp.Server;
-
-public class CannedQueriesConfig {
-   public List<CannedQuery>? Items { get; set; }
-}

src/DataStreamSuite/Bts.DataStreamWebApp.Server/CannedQuery.cs

@@ -1,6 +0,0 @@
-namespace Bts.DataStreamWebApp.Server;
-
-public class CannedQuery {
-   public string? Name { get; set; }
-   public List<string>? Query { get; set; }
-}

src/DataStreamSuite/Bts.DataStreamWebApp.Server/Controllers/CannedQueriesController.cs

@@ -1,18 +0,0 @@
-namespace Bts.DataStreamWebApp.Server;
-
-using Microsoft.AspNetCore.Mvc;
-
-[ApiController]
-[Route("api/[controller]")]
-public class CannedQueriesController : ControllerBase {
-   readonly IConfiguration config;
-
-   public CannedQueriesController(IConfiguration config) =>
-      this.config = config;
-
-   [HttpGet(Name = "GetCannedQueries")]
-   public IEnumerable<CannedQuery>? Get() {
-      var cannedQueries = config.GetSection("CannedQueries").Get<List<CannedQuery>>()?.ToArray() ?? Array.Empty<CannedQuery>();
-      return cannedQueries;
-   }
-}

src/DataStreamSuite/Bts.DataStreamWebApp.Server/Controllers/ConfigurationController.cs

@@ -1,22 +0,0 @@
-// using Bts.DataStreamWebApp.Server;
-
-// using Microsoft.AspNetCore.Mvc;
-
-// [ApiController]
-// [Route("api/[controller]")]
-// public class ConfigController : ControllerBase {
-//    private readonly IConfiguration _configuration;
-
-//    public ConfigController(IConfiguration configuration) {
-//       _configuration = configuration;
-//    }
-
-//    [HttpGet("websocket")]
-//    public ActionResult<DataStreamWebSocketConfig> GetWebSocketConfig() {
-//       var endpoint = $"{_configuration["Kestrel:Endpoints:Http:Url"]}/ws";
-//       var webSocketConfig = new DataStreamWebSocketConfig {
-//          Endpoint = endpoint // or Https based on your setup
-//       };
-//       return Ok(webSocketConfig);
-//    }
-// }

src/DataStreamSuite/Bts.DataStreamWebApp.Server/Controllers/DataStreamSubscriptionController.cs

@@ -1,30 +0,0 @@
-namespace Bts.DataStreamWebApp.Server.Controllers;
-
-using Microsoft.AspNetCore.Mvc;
-
-[Route("/ws")]
-public class DataStreamSubscriptionController : ControllerBase {
-   readonly ILogger logger;
-
-   public DataStreamSubscriptionController(ILogger<DataStreamSubscriptionController> logger) {
-      this.logger = logger;
-      logger.LogInformation("Creating {ControllerType}", GetType().Name);
-   }
-
-   [Route("/ws")]
-   public async Task Get(CancellationToken cancellationToken) {
-      if (!HttpContext.WebSockets.IsWebSocketRequest) {
-         logger.LogError("Expected websocket request on path '{}'", HttpContext.Request.Path);
-         return;
-      }
-      logger.LogInformation("Establishing websocket connection...");
-      try {
-         using var webSocket = await HttpContext.WebSockets.AcceptWebSocketAsync();
-         logger.LogInformation("Websocket connection established.");
-         using var server = new DataStreamWebSocketServer(webSocket, logger);
-         await server.Run(cancellationToken);
-      } catch (Exception ex) {
-         logger.LogError("DataStreamSubscriptionController.Run threw exception: {Exception}", ex);
-      }
-   }
-}

src/DataStreamSuite/Bts.DataStreamWebApp.Server/DataStreamSubscription.cs

@@ -1,133 +0,0 @@
-namespace Bts.DataStreamWebApp.Server.Controllers;
-
-using BCG;
-
-using bts.websocket_core.json_rpc;
-
-using Bts.DataStreamCore;
-using Bts.DataStreamWebApp.Server.Messages;
-
-using System;
-using System.Net.WebSockets;
-using System.Text;
-using System.Text.Json;
-using System.Text.Json.Serialization;
-
-class DataStreamSubscription : IDisposable, IHostedService {
-   public readonly ViewStream ViewStream;
-   public DataPacket? LastFired;
-
-   readonly ILogger logger;
-   readonly WebSocket webSocket;
-
-   public DataStreamSubscription(string id, JsonRpcId requestId, ViewStream viewStream, WebSocket webSocket, ILogger logger) {
-      ViewStream = viewStream;
-      RequestId = requestId;
-      Id = id;
-      this.webSocket = webSocket;
-      this.logger = logger;
-   }
-
-   public string Id { get; }
-   public JsonRpcId RequestId { get; }
-   public SubscriptionStatus Status { get; private set; }
-
-   public void Start() {
-      if (Status == SubscriptionStatus.Subscribed) {
-         logger.LogWarning("Ignoring redundant call to 'Start'. Id = {Id} Status = {Status}", Id, Status);
-         return;
-      }
-      logger.LogInformation("Subscribing to ViewStream. SubscriptionId = {SubscriptionId}", Id);
-      Status = SubscriptionStatus.Subscribed;
-      ViewStream.onChange += onChange;
-   }
-
-   public Task StartAsync(CancellationToken cancellationToken) {
-      cancellationToken.ThrowIfCancellationRequested();
-      Start();
-      return Task.CompletedTask;
-   }
-
-   public Task Clear() {
-      if (LastFired is null) {
-         return Task.CompletedTask;
-      }
-      logger.LogInformation("Clearing subscription (firing DataPacket to delete all rows). Id = {Id} Status = {Status}", Id, Status);
-      return sendDataPacket("clear", new(new BlueTable(LastFired.schema), new BlueTable(LastFired.schema), LastFired.newRows));
-   }
-
-   public void Stop() {
-      if (Status == SubscriptionStatus.Unsubscribed) {
-         logger.LogWarning("Ignoring redundant call to 'Stop'. Id = {Id} Status = {Status}", Id, Status);
-         return;
-      }
-      logger.LogInformation("Unsubscribing from ViewStream. SubscriptionId = {SubscriptionId}", Id);
-      Status = SubscriptionStatus.Unsubscribed;
-      ViewStream.onChange -= onChange;
-   }
-
-   public Task StopAsync(CancellationToken cancellationToken) {
-      cancellationToken.ThrowIfCancellationRequested();
-      Stop();
-      return Task.CompletedTask;
-   }
-
-   public void Dispose() =>
-      Stop();
-
-   async void onChange(object newValue) {
-      if (newValue is not DataPacket dp) {
-         logger.LogError("Expected DataPacket in stream callback (actually received {}).", newValue?.GetType());
-         return;
-      }
-      if (webSocket.State != WebSocketState.Open) {
-         logger.LogError(
-            "Unsubscribing from stream early because WebSocket state = {WebSocketState}", webSocket.State);
-         Dispose();
-         return;
-      }
-      await sendDataPacket("subscribe", dp);
-   }
-
-   async Task sendDataPacket(string method, DataPacket dp) {
-      var serializerOpts = new JsonSerializerOptions {
-         NumberHandling = JsonNumberHandling.AllowNamedFloatingPointLiterals
-      };
-      serializerOpts.Converters.Add(new DataPacketJsonConverter() {
-         IncludeCurrent = true,
-         UseObjectRows = true,
-      });
-      var res = JsonRpcRequest.CreateNotification(method, new DataStreamSubscriptionUpdate() {
-         SubscriptionId = Id,
-         Table = dp
-      });
-      var str = serialize(res, serializerOpts);
-      if (str is null) {
-         logger.LogError("Unsubscribing from stream early because we failed to serialize an outgoing DataPacket");
-         Dispose();
-         return;
-      }
-      logger.LogInformation(
-         "Sending DataPacket. NumRows = {NumRows} NumCols = {NumCols} MessageLength = {MessageLength}",
-         dp.newRows.Count,
-         dp.schema.columnNames.Length,
-         str.Length);
-      var bytes = Encoding.UTF8.GetBytes(str);
-      try {
-         await webSocket.SendAsync(bytes, WebSocketMessageType.Text, true, CancellationToken.None);
-         LastFired = dp;
-      } catch (WebSocketException ex) {
-         logger.LogError($"sendDataPacket failed on webSocket.SendAsync: {ex}");
-      }
-   }
-
-   string? serialize<T>(T message, JsonSerializerOptions? opts) {
-      opts ??= JsonSerializerOptions.Default;
-      try {
-         return JsonSerializer.Serialize(message, opts);
-      } catch (Exception ex) {
-         logger.LogError("Failed to serialize message of type {MessageType}: {Exception}", typeof(T), ex);
-      }
-      return null;
-   }
-}

src/DataStreamSuite/Bts.DataStreamWebApp.Server/DataStreamSubscriptionUpdate.cs

@@ -1,13 +0,0 @@
-namespace Bts.DataStreamWebApp.Server;
-
-using BCG;
-
-using System.Text.Json.Serialization;
-
-public class DataStreamSubscriptionUpdate {
-   [JsonPropertyName("subscription_id")]
-   public string? SubscriptionId { get; set; }
-
-   [JsonPropertyName("table")]
-   public DataPacket? Table { get; set; }
-}

src/DataStreamSuite/Bts.DataStreamWebApp.Server/DataStreamWebAppServerConfig.cs

@@ -1,89 +0,0 @@
-namespace Bts.DataStreamWebApp.Server;
-
-using Bts.PubSub;
-
-using CommandLine;
-
-using System.Text.Json;
-using System.Text.Json.Serialization;
-
-class DataStreamWebAppServerConfig {
-   /// <summary>
-   /// The login user for sparkd
-   /// </summary>
-   [JsonPropertyName("user")]
-   [Option("user", HelpText = "The sparkd login user")]
-   public string? User { get; set; }
-
-   /// <summary>
-   /// The login password for sparkd
-   /// </summary>
-   [JsonPropertyName("password")]
-   [Option("pass", HelpText = "The sparkd login password")]
-   public string? Password { get; set; }
-
-   /// <summary>
-   /// The config server url for sparkd
-   /// </summary>
-   [JsonPropertyName("sparkd")]
-   [Option("sparkd", HelpText = "The sparkd config server")]
-   public string? Sparkd { get; set; }
-
-   [JsonPropertyName("pubsub")]
-   public PubSubConfig? PubSub { get; set; }
-
-   [Option("content-root-path")]
-   [JsonPropertyName("content_root_path")]
-   public string? ContentRootPath { get; set; }
-
-   [Option("web-root-path")]
-   [JsonPropertyName("web_root_path")]
-   public string? WebRootPath { get; set; }
-
-   [Option("environment-name")]
-   [JsonPropertyName("environment_name")]
-   public string? EnvironmentName { get; set; }
-
-   public DirectoryBrowserOptions? DirectoryBrowser { get; set; }
-
-   [Option(
-      "config",
-      MetaValue = "PATH",
-      HelpText = "Path to json file representing these options (DataStreamPublisherConfig)")]
-   [JsonIgnore]
-   public string? ConfigFile { get; set; }
-
-   public static DataStreamWebAppServerConfig Default() =>
-      new() {
-         User = "bts",
-         PubSub = PubSubConfig.Default(),
-      };
-
-   public void MergeWith(DataStreamWebAppServerConfig other) {
-      User ??= other.User;
-      Password ??= other.Password;
-      PubSub ??= other.PubSub;
-      Sparkd ??= other.Sparkd;
-      ContentRootPath ??= other.ContentRootPath;
-      WebRootPath ??= other.WebRootPath;
-      ConfigFile ??= other.ConfigFile;
-   }
-
-   /// <summary>
-   /// Merges options with those defined in `ConfigFile` (does not overwrite
-   /// non-null existing values).
-   /// </summary>
-   /// <exception cref="InvalidOperationException">
-   /// JsonSerializer.Deserialize returns null
-   /// </exception>
-   public void MergeWithConfigFile() {
-      if (ConfigFile is null) {
-         return;
-      }
-      var cfgString = File.ReadAllText(ConfigFile);
-      var parsedCfg = JsonSerializer.Deserialize<DataStreamWebAppServerConfig>(cfgString) ??
-         throw new InvalidOperationException($"Failed to parse json from {ConfigFile}");
-      MergeWith(parsedCfg);
-      ConfigFile = null;
-   }
-}

src/DataStreamSuite/Bts.DataStreamWebApp.Server/DataStreamWebSocketConfig.cs

@@ -1,5 +0,0 @@
-namespace Bts.DataStreamWebApp.Server;
-
-public class DataStreamWebSocketConfig {
-   public string? Endpoint { get; set; }
-}

src/DataStreamSuite/Bts.DataStreamWebApp.Server/DataStreamWebSocketServer.cs

@@ -1,148 +0,0 @@
-namespace Bts.DataStreamWebApp.Server.Controllers;
-
-using bts.websocket_core.json_rpc;
-
-using Bts.DataStreamWebApp.Server.Messages;
-
-using System.Net.WebSockets;
-using System.Text;
-using System.Text.Json;
-using System.Text.Json.Nodes;
-
-public class DataStreamWebSocketServer : IDisposable, IAsyncDisposable {
-   readonly ILogger logger;
-   readonly WebSocket webSocket;
-   readonly Dictionary<string, DataStreamSubscription> subscriptions = new();
-
-   public DataStreamWebSocketServer(WebSocket webSocket, ILogger logger) {
-      this.webSocket = webSocket;
-      this.logger = logger;
-   }
-
-   public async Task<int> Run(CancellationToken cancellationToken) {
-      logger.LogInformation("DataStreamWebSocketServer running");
-      await foreach (var req in JsonRpcRequestAsyncEnumerable.Create(webSocket, logger, cancellationToken)) {
-         await (req.Method switch {
-            "subscribe" => onSubscribe(req, cancellationToken),
-            "unsubscribe" => onUnsubscribe(req.Deserialize<UnsubscribeRequest>(), cancellationToken),
-            _ => Task.CompletedTask,
-         });
-      }
-      logger.LogInformation("DataStreamWebSocketServer stopped");
-      return 0;
-   }
-
-   /// <summary>
-   /// Disposes internal subscriptions. To additionally notify the client of the subscription status
-   /// change(s), use <see cref="DisposeAsync"/>.
-   /// </summary>
-   public void Dispose() {
-      GC.SuppressFinalize(this);
-      disposeSubscriptions();
-   }
-
-   /// <summary>
-   /// Disposes internal subscriptions (like <see cref="Dispose"/>), then sends 'subscription status change'
-   /// notifications to the client (unlike <see cref="Dispose"/>).
-   /// </summary>
-   public async ValueTask DisposeAsync() {
-      GC.SuppressFinalize(this);
-      var finalSubs = subscriptions.Values.Select(s => (s.RequestId, s.Id)).ToList();
-      disposeSubscriptions();
-      // Send a 'subscription status change' msg to the client for each one.
-      foreach (var (requestId, id) in finalSubs) {
-         await sendSubscriptionStatus(requestId, id, "unsubscribe", SubscriptionStatus.Unsubscribed, CancellationToken.None);
-      }
-   }
-
-   async Task onSubscribe(JsonRpcRequest<JsonNode> req, CancellationToken cancellationToken) {
-      ArgumentNullException.ThrowIfNull(req.Id);
-      ArgumentException.ThrowIfNullOrEmpty(req.Method);
-      if (!SubscribeRequest.Parse(req, out var subReq, out var parseErr, out var errCode)) {
-         logger.LogError("Invalid 'subscribe' request: {Reason}", parseErr);
-         var rpcError = JsonRpcResponse.CreateError<SubscriptionError>(
-            req.Id, new() { Code = (long)errCode, Message = "Failed to parse subscription request", Data = parseErr });
-         await send(rpcError, cancellationToken);
-         return;
-      }
-      var subId = subReq.SubscriptionId ?? throw new InvalidOperationException("SubscriptionId is null (parse false)");
-      disposeSubscription(subId);
-      if (!subReq.ParseQuery(out var vs, out parseErr, out errCode)) {
-         logger.LogError("Invalid 'subscribe' request: {Reason}", parseErr);
-         var rpcError = JsonRpcResponse.CreateError<SubscriptionError>(
-            req.Id, new() { Code = (long)errCode, Message = "Failed to parse subscription request", Data = parseErr });
-         await send(rpcError, cancellationToken);
-         return;
-      }
-      if (subscriptions.TryGetValue(subId, out var oldSub)) {
-         logger.LogInformation("Disposing existing subscription with id {SubscriptionId}", subId);
-         oldSub.Dispose();
-      }
-      var sub = new DataStreamSubscription(subId, req.Id.Value, vs, webSocket, logger);
-      subscriptions[subId] = sub;
-      await sendSubscriptionStatus(req.Id.Value, subId, req.Method, SubscriptionStatus.Subscribed, cancellationToken);
-      vs.ErrorChanged += OnStreamError;
-      await sub.StartAsync(cancellationToken);
-
-      async void OnStreamError(Exception exception) {
-         logger.LogError("Error in ViewStream (disposing subscription): {Exception}", exception);
-         subscriptions.Remove(sub.Id);
-         sub.Dispose();
-         var error = new SubscriptionError() {
-            ErrorMessage = "Error in ViewStream (disposing subscription)"
-         };
-         if (exception is not null) {
-            error.ErrorMessage += ". " + exception.Message;
-            error.StackTrace = exception.StackTrace?.Split("\n")?.ToList();
-         }
-         var rpcError = JsonRpcResponse.CreateError<SubscriptionError>(
-            req.Id, new() { Code = (long)ErrorCode.ViewStreamError, Message = "Error in ViewStream (disposing subscription)", Data = error });
-         await send(rpcError, cancellationToken);
-      }
-   }
-
-   async Task onUnsubscribe(JsonRpcRequest<UnsubscribeRequest> req, CancellationToken cancellationToken) {
-      ArgumentNullException.ThrowIfNull(req.Id);
-      ArgumentException.ThrowIfNullOrEmpty(req.Method);
-      var subId = req.Params?.SubscriptionId ?? string.Empty;
-      if (!disposeSubscription(subId)) {
-         logger.LogInformation("Unable to dispose subscription. Subscription with id {Id} not found.", subId);
-      }
-      await sendSubscriptionStatus(req.Id.Value, subId, "unsubscribe", SubscriptionStatus.Unsubscribed, cancellationToken);
-   }
-
-   bool disposeSubscription(string subId) {
-      if (!subscriptions.TryGetValue(subId, out var sub)) {
-         return false;
-      }
-      logger.LogInformation("Disposing subscription with id {Id}", subId);
-      sub.Dispose();
-      subscriptions.Remove(subId);
-      return true;
-   }
-
-   Task sendSubscriptionStatus(JsonRpcId reqId, string subId, string method, SubscriptionStatus status, CancellationToken cancellationToken) {
-      logger.LogInformation("Sending {Method} ack. SubscriptionId = {SubscriptionId}", method, subId);
-      var response = JsonRpcResponse.Create(reqId, new SubscriptionStatusMessage() {
-         SubscriptionId = subId,
-         Status = status,
-      });
-      return send(response, cancellationToken);
-   }
-
-   Task send<T>(T message, CancellationToken cancellationToken) {
-      if (message is null) {
-         return Task.CompletedTask;
-      }
-      var str = JsonSerializer.Serialize(message);
-      var bytes = Encoding.UTF8.GetBytes(str);
-      logger.LogTrace("Sending data. Length = {Length}", bytes.Length);
-      return webSocket.SendAsync(bytes, WebSocketMessageType.Text, true, cancellationToken);
-   }
-
-   void disposeSubscriptions() {
-      logger.LogInformation("Disposing DataStreamWebSocketServer subscriptions. Count: {Count}", subscriptions.Count);
-      subscriptions.Values.DisposeAll();
-      subscriptions.Clear();
-   }
-}

src/DataStreamSuite/Bts.DataStreamWebApp.Server/Dockerfile

@@ -1,11 +0,0 @@
-FROM bluetradingsystems/linux_dotnet7
-
-ARG SOURCE_DIR="build/Release/bin/net7.0/linux-x64/publish"
-ARG WEB_ROOT="build/Bts.DataStreamWebApp.Client/production/dist"
-ARG CONF_FILE="src/DataStreamSuite/Bts.DataStreamWebApp.Server/spark.conf"
-
-RUN dotnet dev-certs https --trust
-COPY --chown=bts:bts "${SOURCE_DIR}" "bin"
-COPY --chown=bts:bts "${WEB_ROOT}" "bin/wwwroot"
-COPY --chown=bts:bts "${CONF_FILE}" ".local/share/bts/Bts.DataStreamWebApp.Server/conf/spark.conf"
-WORKDIR "/home/bts/bin"

src/DataStreamSuite/Bts.DataStreamWebApp.Server/Messages/CannedQueriesResponse.cs

@@ -1,5 +0,0 @@
-namespace Bts.DataStreamWebApp.Server;
-
-public class CannedQueriesResponse {
-   public List<CannedQuery>? Items { get; set; }
-}

src/DataStreamSuite/Bts.DataStreamWebApp.Server/Messages/ErrorCode.cs

@@ -1,7 +0,0 @@
-namespace Bts.DataStreamWebApp.Server.Messages;
-
-enum ErrorCode {
-   None = 0,
-   InvalidSubscriptionRequest,
-   ViewStreamError,
-}

src/DataStreamSuite/Bts.DataStreamWebApp.Server/Messages/SubscribeRequest.cs

@@ -1,72 +0,0 @@
-namespace Bts.DataStreamWebApp.Server.Messages;
-
-using BCG;
-
-using bts.websocket_core.json_rpc;
-
-using Bts.DataStreamExpressions;
-
-using System.Diagnostics.CodeAnalysis;
-using System.Text.Json;
-using System.Text.Json.Nodes;
-using System.Text.Json.Serialization;
-
-public class SubscribeRequest {
-   [JsonPropertyName("query")]
-   public string? Query { get; set; }
-
-   [JsonPropertyName("subscription_id")]
-   public string? SubscriptionId { get; set; }
-
-   internal static bool Parse(JsonRpcRequest<JsonNode> req, [NotNullWhen(true)]out SubscribeRequest? subscription, [NotNullWhen(false)]out SubscriptionError? error, out ErrorCode errorCode) {
-      subscription = null;
-      errorCode = ErrorCode.None;
-      var paramsJsonObj = req.Params?.AsObject();
-      error = null;
-      if (paramsJsonObj is null) {
-         errorCode = ErrorCode.InvalidSubscriptionRequest;
-         error = new() { ErrorMessage = "'params' object not found" };
-         return false;
-      }
-      subscription = paramsJsonObj.Deserialize<SubscribeRequest>();
-      if (subscription is null) {
-         errorCode = ErrorCode.InvalidSubscriptionRequest;
-         error = new() { ErrorMessage = "'params' object invalid" };
-         return false;
-      }
-      return true;
-   }
-
-   internal bool ParseQuery([NotNullWhen(true)] out ViewStream? stream, [NotNullWhen(false)] out SubscriptionError? error, out ErrorCode errorCode) {
-      stream = null;
-      errorCode = ErrorCode.None;
-      error = null;
-      if (Query is null) {
-         errorCode = ErrorCode.InvalidSubscriptionRequest;
-         error = new() { ErrorMessage = "'params.query' field not found (expected a string)" };
-         return false;
-      }
-      var exp = DataStreamExpressionsInterpreter.Evaluate(Query);
-      if (exp is null) {
-         errorCode = ErrorCode.InvalidSubscriptionRequest;
-         error = new() { ErrorMessage = "'params.query' expression has invalid syntax" };
-         return false;
-      }
-      Exception? exception = null;
-      try {
-         stream = exp.getView(new Dictionary<string, StatementExpression>(0));
-      } catch (Exception ex) {
-         exception = ex;
-      }
-      if (stream is null) {
-         errorCode = ErrorCode.InvalidSubscriptionRequest;
-         error = new() { ErrorMessage = "Failed to create ViewStream from 'params.query' expression (check for typos in arguments/typenames)" };
-         if (exception is not null) {
-            error.ErrorMessage += ". " + exception.Message;
-            error.StackTrace = exception.StackTrace?.Split("\n")?.ToList();
-         }
-         return false;
-      }
-      return true;
-   }
-}

src/DataStreamSuite/Bts.DataStreamWebApp.Server/Messages/SubscriptionError.cs

@@ -1,11 +0,0 @@
-namespace Bts.DataStreamWebApp.Server.Messages;
-
-using System.Text.Json.Serialization;
-
-public class SubscriptionError {
-   [JsonPropertyName("error_message")]
-   public string? ErrorMessage { get; set; }
-
-   [JsonPropertyName("stack_trace")]
-   public List<string>? StackTrace { get; set; }
-}

src/DataStreamSuite/Bts.DataStreamWebApp.Server/Messages/SubscriptionStatusMessage.cs

@@ -1,18 +0,0 @@
-namespace Bts.DataStreamWebApp.Server.Messages;
-
-using System.Text.Json.Serialization;
-
-enum SubscriptionStatus {
-   None,
-   Subscribed,
-   Unsubscribed,
-}
-
-class SubscriptionStatusMessage {
-   [JsonPropertyName("subscription_id")]
-   public string? SubscriptionId { get; set; }
-
-   [JsonPropertyName("status")]
-   [JsonConverter(typeof(JsonStringEnumConverter))]
-   public SubscriptionStatus Status { get; set; } = SubscriptionStatus.None;
-}

src/DataStreamSuite/Bts.DataStreamWebApp.Server/Messages/UnsubscribeRequest.cs

@@ -1,8 +0,0 @@
-namespace Bts.DataStreamWebApp.Server.Messages;
-
-using System.Text.Json.Serialization;
-
-public class UnsubscribeRequest {
-   [JsonPropertyName("subscription_id")]
-   public string? SubscriptionId { get; set; }
-}

src/DataStreamSuite/Bts.DataStreamWebApp.Server/Program.cs

@@ -1,82 +0,0 @@
-using Bts.DataStreamCore.Services;
-using Bts.PubSub.Services;
-
-using Spark.Api.Services;
-using Spark.Core.Services;
-
-using System.Reactive.Linq;
-using System.Reflection;
-
-var builder = WebApplication.CreateBuilder(new WebApplicationOptions {
-   Args = args,
-   ApplicationName = "Bts.DataStreamWebApp.Server",
-});
-ConfigureApp(builder.Configuration, builder.Environment.EnvironmentName);
-ConfigureServices(builder.Services, builder.Configuration);
-
-var app = builder.Build();
-var wsOpts = new WebSocketOptions {
-   KeepAliveInterval = TimeSpan.FromMinutes(2)
-};
-if (app.Environment.IsDevelopment()) {
-   app.UseDirectoryBrowser("/browse");
-   app.UseSwagger();
-   app.UseSwaggerUI();
-}
-app.UseHttpsRedirection();
-app.UseWebSockets(wsOpts);
-app.UseDefaultFiles();
-app.UseStaticFiles();
-app.UseAuthorization();
-app.MapControllers();
-try {
-   Console.Write($"Starting Application");
-   app.Run();
-   Console.Write($"Application exited gracefully.");
-} catch (Exception ex) {
-   Console.Error.Write($"Application crashed: {ex}");
-   return 1;
-}
-return 0;
-
-static void ConfigureApp(ConfigurationManager configBuilder, string environment) {
-   // We prefix appsettings with the exe name because multiple exes will usually be in the build dir
-   // (this way, each exe can still load its own appsettings files from the current directory).
-   var exeName = Assembly.GetCallingAssembly().GetName().Name;
-   AddJsonFile($"{exeName}.appsettings.json");
-   AddJsonFile($"{exeName}.appsettings.{environment}.json");
-
-   var cfg = configBuilder.GetSection("config")?.Value;
-   if (!SetBasePath(cfg)) {
-      AddJsonFile(cfg);
-   }
-
-   bool SetBasePath(string? path) {
-      if (!Directory.Exists(path)) {
-         return false;
-      }
-      configBuilder.SetBasePath(path);
-      return true;
-   }
-
-   bool AddJsonFile(string? path) {
-      if (!File.Exists(path)) {
-         return false;
-      }
-      configBuilder.AddJsonFile(path);
-      return true;
-   }
-}
-
-static void ConfigureServices(IServiceCollection services, IConfiguration config) {
-   services.AddDirectoryBrowser()
-      .AddControllers().Services
-      .AddEndpointsApiExplorer()
-      .AddSwaggerGen();
-   services.AddSpaStaticFiles();
-   services.AddSpark(config.GetSection("Spark"))
-      .AddSparkApi(config.GetSection("SparkApi"))
-      .AddMasterWorker(config.GetSection("MasterWorker"))
-      .AddViewStreamDatabase()
-      .AddPubSubClient(config.GetSection("PubSub"));
-}

src/DataStreamSuite/Bts.DataStreamWebApp.Server/Properties/launchSettings.json

@@ -1,29 +0,0 @@
-{
-  "$schema": "http://json.schemastore.org/launchsettings.json",
-  "profiles": {
-    "http": {
-      "commandName": "Project",
-      "dotnetRunMessages": true,
-      "launchBrowser": true,
-      "launchUrl": "swagger",
-      "applicationUrl": "http://localhost:8080",
-      "environmentVariables": {
-        "ASPNETCORE_HTTP_PORT": "8080",
-        "ASPNETCORE_ENVIRONMENT": "Development",
-        "ASPNETCORE_HOSTINGSTARTUPASSEMBLIES": "Microsoft.AspNetCore.SpaProxy",
-      }
-    },
-    "https": {
-      "commandName": "Project",
-      "dotnetRunMessages": true,
-      "launchBrowser": true,
-      "applicationUrl": "http://localhost:8080;https://localhost:8443",
-      "environmentVariables": {
-        "ASPNETCORE_HTTP_PORT": "8080",
-        "ASPNETCORE_HTTPS_PORT": "8443",
-        "ASPNETCORE_ENVIRONMENT": "Development",
-        "ASPNETCORE_HOSTINGSTARTUPASSEMBLIES": "Microsoft.AspNetCore.SpaProxy",
-      }
-    },
-  },
-}

src/DataStreamSuite/Bts.DataStreamWebApp.Server/appsettings.Development.json

@@ -1,52 +0,0 @@
-{
-  "PubSub": {
-    "Routing": {
-      "PublisherRoutes": [
-        {
-          "Name": "default",
-          "ServerAddress": "bts-cloud://localhost:22019",
-          "TagRegExp": ".*"
-        }
-      ],
-      "SubscriberRoutes": [
-        {
-          "Name": "default",
-          "ServerAddress": "bts-cloud://localhost:22019",
-          "TagRegExp": ".*"
-        }
-      ]
-    }
-  },
-  "CannedQueries": [
-    {
-      "Name": "BBO",
-      "Query": [
-        "bbo"
-      ]
-    },
-    {
-      "Name": "Trades",
-      "Query": [
-        "timesales"
-      ]
-    },
-    {
-      "Name": "Instruments",
-      "Query": [
-        "instruments"
-      ]
-    },
-    {
-      "Name": "Books",
-      "Query": [
-        "books"
-      ]
-    },
-    {
-      "Name": "Benzinga News",
-      "Query": [
-        "benzinga_news"
-      ]
-    }
-  ]
-}

src/DataStreamSuite/Bts.DataStreamWebApp.Server/appsettings.Production.json

@@ -1,8 +0,0 @@
-{
-  "Spark": {
-    "ConfigServer": "http://sparkd:8000/config"
-  },
-  "SparkApi": {
-    "IsEnabled": true
-  }
-}

src/DataStreamSuite/Bts.DataStreamWebApp.Server/appsettings.Staging.json

@@ -1,44 +0,0 @@
-{
-  "CannedQueries": [
-    {
-      "Name": "BBO",
-      "Query": [
-        "bbo"
-      ]
-    },
-    {
-      "Name": "Trades",
-      "Query": [
-        "timesales"
-      ]
-    },
-    {
-      "Name": "Instruments",
-      "Query": [
-        "instruments"
-      ]
-    },
-    {
-      "Name": "Books",
-      "Query": [
-        "books"
-      ]
-    },
-    {
-      "Name": "Benzinga News",
-      "Query": [
-        "benzinga_news"
-      ]
-    }
-  ],
-  "Spark": {
-    "ConfigServer": "http://sparkd:8000/config"
-  },
-  "SparkApi": {
-    "IsEnabled": true,
-    "Credentials": {
-      "User": "bts",
-      "Password": ""
-    }
-  }
-}

src/DataStreamSuite/Bts.DataStreamWebApp.Server/spark.conf

@@ -1,15 +0,0 @@
-Logging {
-  value {
-    console {
-      default_level: "Info"
-    }
-    syslog {
-      default_level: "Info"
-    }
-    file {
-      default_level: "Info"
-    }
-    initial_queue_size: 120001
-    max_queue_growth: 4
-  }
-}

src/DataStreamSuite/Bts.DataStreamWebApp.Server/wwwroot/alt-index.html

@@ -1,168 +0,0 @@
-<!DOCTYPE html>
-<html>
-<head>
-    <meta charset="utf-8" />
-    <title></title>
-    <style>
-        table {
-            border: 0
-        }
-
-        .commslog-data {
-            font-family: Consolas, Courier New, Courier, monospace;
-        }
-
-        .commslog-server {
-            background-color: red;
-            color: white
-        }
-
-        .commslog-client {
-            background-color: green;
-            color: white
-        }
-    </style>
-</head>
-<body>
-    <h1>ALT WebSocket Sample Application</h1>
-    <p id="stateLabel">Ready to connect...</p>
-    <div>
-        <label for="connectionUrl">WebSocket Server URL:</label>
-        <input id="connectionUrl" />
-        <button id="connectButton" type="submit">Connect</button>
-    </div>
-    <p></p>
-    <div>
-        <label for="sendMessage">Message to send:</label>
-        <input id="sendMessage" disabled />
-        <button id="sendButton" type="submit" disabled>Send</button>
-        <button id="closeButton" disabled>Close Socket</button>
-    </div>
-
-    <h2>Communication Log</h2>
-    <table style="width: 800px">
-        <thead>
-            <tr>
-                <td style="width: 100px">From</td>
-                <td style="width: 100px">To</td>
-                <td>Data</td>
-            </tr>
-        </thead>
-        <tbody id="commsLog">
-        </tbody>
-    </table>
-
-    <script>
-        var connectionUrl = document.getElementById("connectionUrl");
-        var connectButton = document.getElementById("connectButton");
-        var stateLabel = document.getElementById("stateLabel");
-        var sendMessage = document.getElementById("sendMessage");
-        var sendButton = document.getElementById("sendButton");
-        var commsLog = document.getElementById("commsLog");
-        var closeButton = document.getElementById("closeButton");
-        var socket;
-
-        var scheme = document.location.protocol === "https:" ? "wss" : "ws";
-        var port = document.location.port ? (":" + document.location.port) : "";
-
-        connectionUrl.value = scheme + "://" + document.location.hostname + port + "/ws" ;
-
-        function updateState() {
-            function disable() {
-                sendMessage.disabled = true;
-                sendButton.disabled = true;
-                closeButton.disabled = true;
-            }
-            function enable() {
-                sendMessage.disabled = false;
-                sendButton.disabled = false;
-                closeButton.disabled = false;
-            }
-
-            connectionUrl.disabled = true;
-            connectButton.disabled = true;
-
-            if (!socket) {
-                disable();
-            } else {
-                switch (socket.readyState) {
-                    case WebSocket.CLOSED:
-                        stateLabel.innerHTML = "Closed";
-                        disable();
-                        connectionUrl.disabled = false;
-                        connectButton.disabled = false;
-                        break;
-                    case WebSocket.CLOSING:
-                        stateLabel.innerHTML = "Closing...";
-                        disable();
-                        break;
-                    case WebSocket.CONNECTING:
-                        stateLabel.innerHTML = "Connecting...";
-                        disable();
-                        break;
-                    case WebSocket.OPEN:
-                        stateLabel.innerHTML = "Open";
-                        enable();
-                        break;
-                    default:
-                        stateLabel.innerHTML = "Unknown WebSocket State: " + htmlEscape(socket.readyState);
-                        disable();
-                        break;
-                }
-            }
-        }
-
-        closeButton.onclick = function () {
-            if (!socket || socket.readyState !== WebSocket.OPEN) {
-                alert("socket not connected");
-            }
-            socket.close(1000, "Closing from client");
-        };
-
-        sendButton.onclick = function () {
-            if (!socket || socket.readyState !== WebSocket.OPEN) {
-                alert("socket not connected");
-            }
-            var data = sendMessage.value;
-            socket.send(data);
-            commsLog.innerHTML += '<tr>' +
-                '<td class="commslog-client">Client</td>' +
-                '<td class="commslog-server">Server</td>' +
-                '<td class="commslog-data">' + htmlEscape(data) + '</td></tr>';
-        };
-
-        connectButton.onclick = function() {
-            stateLabel.innerHTML = "Connecting";
-            socket = new WebSocket(connectionUrl.value);
-            socket.onopen = function (event) {
-                updateState();
-                commsLog.innerHTML += '<tr>' +
-                    '<td colspan="3" class="commslog-data">Connection opened</td>' +
-                '</tr>';
-            };
-            socket.onclose = function (event) {
-                updateState();
-                commsLog.innerHTML += '<tr>' +
-                    '<td colspan="3" class="commslog-data">Connection closed. Code: ' + htmlEscape(event.code) + '. Reason: ' + htmlEscape(event.reason) + '</td>' +
-                '</tr>';
-            };
-            socket.onerror = updateState;
-            socket.onmessage = function (event) {
-                commsLog.innerHTML += '<tr>' +
-                    '<td class="commslog-server">Server</td>' +
-                    '<td class="commslog-client">Client</td>' +
-                    '<td class="commslog-data">' + htmlEscape(event.data) + '</td></tr>';
-            };
-        };
-
-        function htmlEscape(str) {
-            return str.toString()
-                .replace(/&/g, '&amp;')
-                .replace(/"/g, '&quot;')
-                .replace(/'/g, '&#39;')
-                .replace(/</g, '&lt;')
-                .replace(/>/g, '&gt;');
-        }
-    </script>
-</body>
-</html>

src/DataStreamSuite/Bts.DataStreamWebApp.Server/wwwroot/index.html

@@ -1,176 +0,0 @@
-<!DOCTYPE html>
-<html>
-<head>
-    <meta charset="utf-8" />
-    <title></title>
-    <style>
-        table {
-            border: 0
-        }
-
-        .commslog-data {
-            font-family: Consolas, Courier New, Courier, monospace;
-        }
-
-        .commslog-server {
-            background-color: red;
-            color: white
-        }
-
-        .commslog-client {
-            background-color: green;
-            color: white
-        }
-    </style>
-</head>
-<body>
-    <h1>DataStreams via WebSocket</h1>
-    <p id="stateLabel">Ready to connect...</p>
-    <div>
-        <label for="connectionUrl">WebSocket Server URL:</label>
-        <input id="connectionUrl" />
-        <button id="connectButton" type="submit">Connect</button>
-    </div>
-    <p></p>
-    <div>
-        <label for="sendMessage">Query:</label>
-        <input id="sendMessage" disabled />
-        <button id="sendButton" type="submit" disabled>Send</button>
-        <button id="closeButton" disabled>Close Socket</button>
-    </div>
-
-    <h2>Communication Log</h2>
-    <table style="width: 800px">
-        <thead>
-            <tr>
-                <td style="width: 100px">From</td>
-                <td style="width: 100px">To</td>
-                <td>Data</td>
-            </tr>
-        </thead>
-        <tbody id="commsLog">
-        </tbody>
-    </table>
-
-    <script>
-        var connectionUrl = document.getElementById("connectionUrl");
-        var connectButton = document.getElementById("connectButton");
-        var stateLabel = document.getElementById("stateLabel");
-        var sendMessage = document.getElementById("sendMessage");
-        var sendButton = document.getElementById("sendButton");
-        var commsLog = document.getElementById("commsLog");
-        var closeButton = document.getElementById("closeButton");
-        var socket;
-
-        var scheme = document.location.protocol === "https:" ? "wss" : "ws";
-        var port = document.location.port ? (":" + document.location.port) : "";
-
-        connectionUrl.value = scheme + "://" + document.location.hostname + port + "/ws" ;
-
-        function updateState() {
-            function disable() {
-                sendMessage.disabled = true;
-                sendButton.disabled = true;
-                closeButton.disabled = true;
-            }
-            function enable() {
-                sendMessage.disabled = false;
-                sendButton.disabled = false;
-                closeButton.disabled = false;
-            }
-
-            connectionUrl.disabled = true;
-            connectButton.disabled = true;
-
-            if (!socket) {
-                disable();
-            } else {
-                switch (socket.readyState) {
-                    case WebSocket.CLOSED:
-                        stateLabel.innerHTML = "Closed";
-                        disable();
-                        connectionUrl.disabled = false;
-                        connectButton.disabled = false;
-                        break;
-                    case WebSocket.CLOSING:
-                        stateLabel.innerHTML = "Closing...";
-                        disable();
-                        break;
-                    case WebSocket.CONNECTING:
-                        stateLabel.innerHTML = "Connecting...";
-                        disable();
-                        break;
-                    case WebSocket.OPEN:
-                        stateLabel.innerHTML = "Open";
-                        enable();
-                        break;
-                    default:
-                        stateLabel.innerHTML = "Unknown WebSocket State: " + htmlEscape(socket.readyState);
-                        disable();
-                        break;
-                }
-            }
-        }
-
-        closeButton.onclick = function () {
-            if (!socket || socket.readyState !== WebSocket.OPEN) {
-                alert("socket not connected");
-            }
-            socket.close(1000, "Closing from client");
-        };
-
-        sendButton.onclick = function () {
-            if (!socket || socket.readyState !== WebSocket.OPEN) {
-                alert("socket not connected");
-            }
-            const rpcReq = {
-              jsonrpc: "2.0",
-              method: "subscribe",
-              params: {
-                query: sendMessage.value,
-              },
-              id: 1,
-            };
-            const rpcReqStr = JSON.stringify(rpcReq);
-            socket.send(rpcReqStr);
-            commsLog.innerHTML += '<tr>' +
-                '<td class="commslog-client">Client</td>' +
-                '<td class="commslog-server">Server</td>' +
-                '<td class="commslog-data">' + htmlEscape(rpcReqStr) + '</td></tr>';
-        };
-
-        connectButton.onclick = function() {
-            stateLabel.innerHTML = "Connecting";
-            socket = new WebSocket(connectionUrl.value);
-            socket.onopen = function (event) {
-                updateState();
-                commsLog.innerHTML += '<tr>' +
-                    '<td colspan="3" class="commslog-data">Connection opened</td>' +
-                '</tr>';
-            };
-            socket.onclose = function (event) {
-                updateState();
-                commsLog.innerHTML += '<tr>' +
-                    '<td colspan="3" class="commslog-data">Connection closed. Code: ' + htmlEscape(event.code) + '. Reason: ' + htmlEscape(event.reason) + '</td>' +
-                '</tr>';
-            };
-            socket.onerror = updateState;
-            socket.onmessage = function (event) {
-                commsLog.innerHTML += '<tr>' +
-                    '<td class="commslog-server">Server</td>' +
-                    '<td class="commslog-client">Client</td>' +
-                    '<td class="commslog-data">' + htmlEscape(event.data) + '</td></tr>';
-            };
-        };
-
-        function htmlEscape(str) {
-            return str.toString()
-                .replace(/&/g, '&amp;')
-                .replace(/"/g, '&quot;')
-                .replace(/'/g, '&#39;')
-                .replace(/</g, '&lt;')
-                .replace(/>/g, '&gt;');
-        }
-    </script>
-</body>
-</html>

src/DataStreamSuite/Bts.Extensions.Hosting/Bts.Extensions.Hosting.csproj

@@ -1,13 +1,13 @@
 <Project Sdk="Microsoft.NET.Sdk">
 
   <PropertyGroup>
-    <TargetFramework>net7.0</TargetFramework>
+    <TargetFramework>net8.0</TargetFramework>
     <ImplicitUsings>enable</ImplicitUsings>
     <Nullable>enable</Nullable>
   </PropertyGroup>
 
   <ItemGroup>
-    <PackageReference Include="Microsoft.Extensions.Hosting" Version="7.0.1" />
+    <PackageReference Include="Microsoft.Extensions.Hosting" Version="10.0.0" />
   </ItemGroup>
 
 </Project>

src/DataStreamSuite/Bts.Extensions.Hosting/HostBuilderExtensions.cs

@@ -2,6 +2,7 @@
 
 using Microsoft.Extensions.Configuration;
 using Microsoft.Extensions.Hosting;
+using Microsoft.Extensions.Logging;
 
 using System.Diagnostics;
 
@@ -47,6 +48,17 @@ public static class HostBuilderExtensions {
       }
    }
 
+   public static void ConfigureLogging(ILoggingBuilder loggingBuilder, ILoggerProvider loggerProvider) =>
+      loggingBuilder.AddFilter("Microsoft", LogLevel.Warning)
+         .AddFilter("System", LogLevel.Warning)
+         .AddFilter("NToastNotify", LogLevel.Warning)
+         .AddProvider(loggerProvider)
+         .AddSimpleConsole(opts => {
+            opts.SingleLine = true;
+            opts.TimestampFormat = "yyyy-MM-dd HH:mm:ss.ffffff|";
+         })
+         .AddDebug();
+
    public static string GetCallingAssemblyName() =>
       new StackTrace()
          .GetFrame(2)

src/DataStreamSuite/Bts.Net/Bts.Net.csproj

@@ -1,7 +1,7 @@
 <Project Sdk="Microsoft.NET.Sdk">
 
   <PropertyGroup>
-    <TargetFramework>net7.0</TargetFramework>
+    <TargetFramework>net8.0</TargetFramework>
     <ImplicitUsings>enable</ImplicitUsings>
     <Nullable>enable</Nullable>
   </PropertyGroup>

src/DataStreamSuite/DataStream/BlueOrb/BlueAddIndexColumnTable.cs

@@ -0,0 +1,118 @@
+namespace Bts.DataStream;
+
+using BCG;
+
+using bts.collections;
+using bts.utils;
+
+using Bts.DataStreamCore;
+
+using Spark.Core.Collections;
+
+using System;
+using System.Collections;
+using System.Collections.Generic;
+using System.Linq;
+
+public class BlueAddIndexColumnTable : IBlueRowCollection {
+   readonly IBlueTableSchema schema;
+   readonly IList<object?>[] columnValues;
+   readonly int count;
+
+   BlueAddIndexColumnTable(IBlueTableSchema schema, IList<object?>[] columnValues, int? count = null) {
+      this.schema = schema;
+      this.columnValues = columnValues;
+      this.count = count ?? columnValues[0].Count;
+   }
+
+   IBlueTableSchema IBlueRowCollection.schema => schema;
+
+   public int Count => count;
+
+   public IBlueRow this[int id] =>
+      id > 0 && id <= count
+         ? GetRow(id - 1)
+         : throw new KeyNotFoundException();
+
+   public static BlueAddIndexColumnTable Create(string tableName, ColumnInfo[] columnInfos, IList<object?>[] columnValues) {
+      if (columnInfos.Any(c => c.colName == "id")) {
+         throw new ArgumentException("'id' column already exists", nameof(columnInfos));
+      }
+      if (columnInfos.Length != columnValues.Length) {
+         throw new ArgumentException("columnValues length differs from columnInfos length", nameof(columnValues));
+      }
+      int count = columnValues.FirstOrDefault()?.Count ?? 0;
+      var origCols = columnInfos;
+      var origColValues = columnValues;
+      columnInfos = new ColumnInfo[origCols.Length + 1];
+      columnInfos[0] = new ColumnInfo("id", 0, typeof(int));
+      columnValues = new IList<object?>[origCols.Length + 1];
+      columnValues[0] = new StepList<int>(1, 1, count).WrapSelector(id => (object?)id, null);
+      for (int i = 1; i < columnInfos.Length; i++) {
+         columnInfos[i] = new ColumnInfo(origCols[i - 1].colName, i, origCols[i - 1].colType);
+         columnValues[i] = origColValues[i - 1];
+      }
+      var schema = new BlueTableSchema(columnInfos, tableName);
+      return new(schema, columnValues, count);
+   }
+
+   public static BlueAddIndexColumnTable Create(IBlueTableSchema schema, IList<object?>[] columnValues) {
+      int count = columnValues.FirstOrDefault()?.Count ?? 0;
+      int? idColIdx = schema.TryResolveIndices("id");
+      var idColValues = new StepList<int>(1, 1, count).WrapSelector(id => (object?)id, null);
+      var columnInfos = schema.columnInfo;
+      if (idColIdx.HasValue) {
+         // Replace existing id column. columnValues may or may not contain the old ids (if its
+         // length is one less column than that of columnInfos, it doesn't contain them).
+         switch (columnValues.Length - schema.columnInfo.Length) {
+            case 0:
+               columnValues[idColIdx.Value] = idColValues;
+               break;
+            case -1:
+               columnValues = ArrayHelper.InsertExact(columnValues, idColIdx.Value, true, idColValues);
+               break;
+            default:
+               throw new ArgumentException("ColumnInfos length disagrees with columnValues length");
+         }
+      } else {
+         if (columnValues.Length != columnInfos.Length) {
+            throw new ArgumentException("ColumnInfos length disagrees with columnValues length");
+         }
+         var origColInfos = columnInfos;
+         columnInfos = new ColumnInfo[columnInfos.Length + 1];
+         columnInfos[0] = new("id", 0, typeof(int));
+         for (int i = 1; i < columnInfos.Length; i++) {
+            var c = origColInfos[i - 1];
+            columnInfos[i] = new(c.colName, c.colNo + 1, c.colType);
+         }
+         columnValues = ArrayHelper.InsertExact(columnValues, 0, true, idColValues);
+      }
+      return new(new BlueTableSchema(schema.tableName, columnInfos), columnValues);
+   }
+
+   public bool containsID(int id) =>
+      id > 0 && id <= count;
+
+   public IList<object?> GetColumn(int index) =>
+      columnValues[index];
+
+   public IBlueRow GetRow(int index) =>
+      new BlueRowColumnWrapper(schema, index, columnValues);
+
+   public int IndexOf(IBlueRow item) {
+      int id = item.Id();
+      return id > 0 && id <= count && BlueRowEqualityComparer.Equals(GetRow(id - 1), item)
+         ? id - 1
+         : -1;
+   }
+
+   public IEnumerator<IBlueRow> GetEnumerator() {
+      int count = Count;
+      for (int i = 0; i < count; i++) {
+         yield return GetRow(i);
+      }
+   }
+
+   IEnumerator IEnumerable.GetEnumerator() =>
+      GetEnumerator();
+}

src/DataStreamSuite/DataStream/BlueOrb/BlueIndexedColumnTable.cs

@@ -0,0 +1,56 @@
+namespace Bts.DataStream;
+
+using BCG;
+
+using Bts.DataStreamCore;
+
+using mbts.utils;
+
+using System.Collections;
+using System.Collections.Generic;
+using System.Linq;
+
+public class BlueIndexedColumnTable : IBlueRowCollection {
+   readonly IBlueTableSchema schema;
+   readonly IList<object?>[] columnValues;
+   readonly Dictionary<int, int> rowIdToIndex;
+
+   public BlueIndexedColumnTable(IBlueTableSchema schema, IList<object?>[] columnValues) {
+      this.schema = schema;
+      this.columnValues = columnValues;
+      int idColumnIndex = schema.ResolveIndices("id");
+      rowIdToIndex = columnValues[idColumnIndex]
+         .Select((idObj, idx) => new KeyValuePair<int, int>(ExceptionAssert.Argument.Is<int>(idObj), idx))
+         .ToDictionary();
+   }
+
+   public IBlueRow this[int id] =>
+      rowIdToIndex.TryGetValue(id, out var idx)
+         ? GetRow(idx)
+         : throw new KeyNotFoundException();
+
+   IBlueTableSchema IBlueRowCollection.schema => schema;
+
+   public int Count => rowIdToIndex.Count;
+
+   public bool containsID(int id) =>
+      rowIdToIndex.ContainsKey(id);
+
+   public IEnumerator<IBlueRow> GetEnumerator() {
+      int count = Count;
+      for (int i = 0; i < count; i++) {
+         yield return GetRow(i);
+      }
+   }
+
+   public IBlueRow GetRow(int index) =>
+      new BlueRowColumnWrapper(schema, index, columnValues);
+
+   public int IndexOf(IBlueRow item) =>
+      rowIdToIndex.TryGetValue(item.Id(), out var idx) && BlueRowEqualityComparer.Equals(GetRow(idx), item)
+         ? idx
+         : -1;
+
+   IEnumerator IEnumerable.GetEnumerator() =>
+      GetEnumerator();
+}

src/DataStreamSuite/DataStream/BlueOrb/BlueRowColumnWrapper.cs

@@ -0,0 +1,21 @@
+namespace Bts.DataStream;
+
+using BCG;
+
+using System.Collections.Generic;
+
+public class BlueRowColumnWrapper : IBlueRow {
+   readonly int index;
+   readonly IList<object?>[] columns;
+
+   public BlueRowColumnWrapper(IBlueTableSchema schema, int index, IList<object?>[] columns) {
+      this.schema = schema;
+      this.index = index;
+      this.columns = columns;
+   }
+
+   public object? this[int index] =>
+      columns[index][this.index];
+
+   public IBlueTableSchema schema { get; }
+}

src/DataStreamSuite/DataStream/BlueOrb/CSVReaderStream.cs

@@ -0,0 +1,18 @@
+namespace bts.BlueOrb {
+   using System;
+
+   using BCG;
+
+   [DataStreamUserRespresentation("CSVReaderStream")]
+   public class CSVReaderStream : ViewStream {
+      public CSVReaderStream(string file_name)
+         : this(file_name, false) { }
+      public CSVReaderStream(string file_name, bool guess_schema) {
+         ViewStream rs = new BCG.CSVReaderStream(file_name);
+         if (guess_schema) {
+            rs = new SchemaGuesserStream(rs);
+         }
+         mySetRealSource(rs);
+      }
+   }
+}

src/DataStreamSuite/DataStream/BlueOrb/CSVSaverStream.cs

@@ -0,0 +1,13 @@
+namespace bts.BlueOrb {
+   using BCG;
+
+   using System;
+
+   [DataStreamUserRespresentation("CSVSaverStream")]
+   public class CSVSaverStream: ViewStream {
+      public CSVSaverStream(ViewStream parent_str, string file_name) {
+         BCG.CSVSaverStream rs = new BCG.CSVSaverStream(parent_str, file_name, true, Int32.MaxValue, false);
+         mySetRealSource(rs);
+      }
+   }
+}

src/DataStreamSuite/DataStream/BlueOrb/CountStream.cs

@@ -14,14 +14,11 @@ namespace BCG {
          myVS = vs;
       }
 
-      public CountStream(string tableName)
-         : this(new TableStream(tableName)) { }
-
       protected override void myOnFirstSubscriber() {
-         myVS.onChange += myHandleChanges;
+         myVS.onChange += new ChangeDelegate(myHandleChanges);
       }
       protected override void myOnLastSubscriber() {
-         myVS.onChange -= myHandleChanges;
+         myVS.onChange -= new ChangeDelegate(myHandleChanges);
       }
 
 

src/DataStreamSuite/DataStream/BlueOrb/FilterStream.cs

@@ -78,12 +78,11 @@ namespace bts.BlueOrb {
       #endregion
 
       #region overridden methods
-      protected override void myOnFirstSubscriber() =>
-         myParentStream.onChange += myHandleCallbacks;
+      protected override void myOnFirstSubscriber() { myParentStream.onChange += new ChangeDelegate(myHandleCallbacks); }
 
       protected override void myOnLastSubscriber() {
          myLastFiredRows = null;
-         myParentStream.onChange -= myHandleCallbacks;
+         myParentStream.onChange -= new ChangeDelegate(myHandleCallbacks);
       }
 
       protected override void myFireEvent(object o) {

src/DataStreamSuite/DataStream/BlueOrb/GroupByStream.cs

@@ -34,7 +34,8 @@ namespace bts.BlueOrb {
                if (out_rows != null && out_rows.Count > 0) {
                   rv = rv ?? new BlueTable(out_rows.ElementAt(0).schema);
                }
-               rv.addRows(out_rows);
+               if (rv != null)
+                  rv.addRows(out_rows);
             }
             return rv;
          }
@@ -61,20 +62,24 @@ namespace bts.BlueOrb {
                                  Func<List<IBlueRow>, List<IBlueRow>> evaluate_func)
                                  : this(main_stream, new DelNonIncGroupByEvaluator(groupby_cols_func, evaluate_func)) { }
 
-      public GroupByStream(ViewStream parent_str, string groupby_col, string standard_group_by_text)
-      : this(parent_str, groupby_col, standard_group_by_text, String.Empty) { }
+      public GroupByStream(ViewStream parent_str, string groupby_cols, string standard_group_by_text)
+      : this(parent_str, groupby_cols, standard_group_by_text, String.Empty) { }
 
       // this is the main constructor that does everything
-      public GroupByStream(ViewStream parent_str, string groupby_col, string standard_group_by_ops, string optional_sort_col) {
+      public GroupByStream(ViewStream parent_str, string groupby_cols, string standard_group_by_ops, string optional_sort_col) {
          MultiRowOpHelper std_group_by = MultiRowOpHelper.parse_ops(standard_group_by_ops);
          Func<string[]> groupby_cols_func = () => {
-            return String.IsNullOrEmpty(groupby_col) ? null : new string[] { groupby_col };
+            if (String.IsNullOrEmpty(groupby_cols))
+               return null;
+            return groupby_cols.Split(',').Select(x => x.Trim()).ToArray();
          };
          Func<List<IBlueRow>, List<IBlueRow>> evaluate_func = (group_rows) => {
             if (!String.IsNullOrEmpty(optional_sort_col))
                group_rows.Sort((r1, r2) => BlueDataUtility.compare_rows(r2, r1, optional_sort_col));
             // this si where the group by aggregation happens
             IBlueRow rv = MultiRowOpHelper.group_by_compute(group_rows, std_group_by);
+            if (rv == null)
+               return new List<IBlueRow>();
             return new IBlueRow[] { rv }.ToList();
          };
          ViewStream rs = new GroupByStream(parent_str, groupby_cols_func, evaluate_func);
@@ -105,6 +110,9 @@ namespace bts.BlueOrb {
          /// reducing
          _timer.Restart();
          IBlueRowCollection total_state = _groupby_func.evaluate(grouped_rows);
+         if (total_state == null)
+            // mayeb returna n empty table ?
+            return;
          _timer.Stop();
          var applyElapsedMs = _timer.ElapsedMilliseconds;
          _n_total_fires++;
@@ -113,9 +121,7 @@ namespace bts.BlueOrb {
             Log.DataFlow.info(
                $"Time to evaluate {grouped_rows.Count} groups {_timer.ElapsedMilliseconds}ms, ratio {_n_expensive_fires}/{_n_total_fires}");
          }
-         // TODO,
-         if (total_state != null)
-            myFireEvent(new DataPacket(total_state, total_state, myLastValue == null ? new BlueTable(total_state.schema) : (myLastValue as DataPacket).newRows));
+         myFireEvent(new DataPacket(total_state, total_state, myLastValue == null ? new BlueTable(total_state.schema) : (myLastValue as DataPacket).newRows));
       }
    }
 }

src/DataStreamSuite/DataStream/BlueOrb/Join/SimpleJoinStream.cs

@@ -20,9 +20,9 @@
          private IBlueTableSchema _schema;
          private int[] _join_cols;
          private IDictionary<RowHash, List<IBlueRow>> _hash_row_map = new Dictionary<RowHash, List<IBlueRow>>();
-         
+
          public JoinBookkeepingData(ViewStream vs, List<string> join_cols, bool is_left, BlueRowHasher row_hasher) {
-            _join_col_names = join_cols.Where (c => !String.IsNullOrEmpty(c)).ToList ();
+            _join_col_names = join_cols.Where(c => !String.IsNullOrEmpty(c)).ToList();
             _is_left = is_left;
             _hasher = row_hasher;
          }
@@ -31,23 +31,23 @@
 
          public IDictionary<RowHash, List<IBlueRow>> rows => _hash_row_map;
 
-         public List<IBlueRow> get_matching_rows (int hash) {
+         public List<IBlueRow> get_matching_rows(int hash) {
             List<IBlueRow> rv = null;
             _hash_row_map.TryGetValue(hash, out rv);
             return rv;
          }
 
          private RowHash _get_hash(IBlueRow r) {
-            if (_join_cols == null || _join_cols.Length == 0) { 
+            if (_join_cols == null || _join_cols.Length == 0) {
                return 0;
             }
             return _hasher.get_hash(r, _join_cols);
          }
 
          public void update_data(DataPacket dp) {
-            if (!ready) {               
+            if (!ready) {
                _schema = dp.newRows.schema;
-               _join_cols = _join_col_names.Count == 0 ? new int[] { }: _join_col_names.Select(n => dp.schema[n].colNo).ToArray();
+               _join_cols = _join_col_names.Count == 0 ? new int[] { } : _join_col_names.Select(n => dp.schema[n].colNo).ToArray();
             }
             _hash_row_map.Clear();
             foreach (IBlueRow r in dp.newRows) {
@@ -82,7 +82,7 @@
          _rs = rs;
          List<string> lj_col_list = ljc.Split(',').Where(x => !String.IsNullOrEmpty(x)).ToList();
          List<string> rj_col_list = rjc.Split(',').Where(x => !String.IsNullOrEmpty(x)).ToList();
-         Log.assert(lj_col_list.Count ==  rj_col_list.Count);
+         Log.assert(lj_col_list.Count == rj_col_list.Count);
          int n_cols = lj_col_list.Count;
          BlueRowHasher hasher = new BlueRowHasher(n_cols);
          _ldata = new JoinBookkeepingData(ls, lj_col_list, true, hasher);
@@ -101,7 +101,7 @@
       }
 
       private void _rs_onchange(object dp_obj) {
-         DataPacket dp = dp_obj as DataPacket ;
+         DataPacket dp = dp_obj as DataPacket;
          _rdata.update_data(dp);
          _form_and_fire();
       }
@@ -128,11 +128,11 @@
       }
 
       private void _form_and_fire() {
-         if (!_ldata.ready||!_rdata.ready)
+         if (!_ldata.ready || !_rdata.ready)
             return;
          _build_schema();
          List<object[]> res_rows = new List<object[]>();
-         
+
          Action<object[], IBlueRow, bool> copy_row = (rval, r, is_left) => {
             int offset = 1;
             JoinBookkeepingData sdata = _ldata;
@@ -148,7 +148,7 @@
             }
          };
          Func<int, IBlueRow, object[]> build_row_array = (id, l_row) => {
-            object[] rv = new object[_schema.columnNames.Length];            
+            object[] rv = new object[_schema.columnNames.Length];
             for (int i = 0; i < _schema.columnNames.Length; i++) {
                rv[i] = DBNull.Value;
             }
@@ -162,7 +162,7 @@
             List<IBlueRow> l_rows = kv.Value;
             List<IBlueRow> r_rows = _rdata.get_matching_rows(hash);
 
-            foreach (IBlueRow lr in l_rows) {                              
+            foreach (IBlueRow lr in l_rows) {
                if (r_rows != null) {
                   foreach (IBlueRow r in r_rows) {
                      object[] rowvals = build_row_array(res_rows.Count, lr);
@@ -182,6 +182,15 @@
          var dp = BlueDataUtility.build_dp(rv, myLastValue as DataPacket);
          myFireEvent(dp);
       }
-
    }
+
+   [DataStreamUserRespresentation("FullJoinStream")]
+   public class FullJoinStream : ViewStream {
+      public FullJoinStream(ViewStream l_str, string ljcs, ViewStream r_str, string rjcs) {
+         ViewStream js = new SimpleJoinStream(l_str, ljcs, r_str, rjcs);
+         ViewStream rs = new bts.BlueOrb.FilterStream(js, r => !DBNull.Value.Equals(r["right_id"]));
+         mySetRealSource(rs);
+      }
+   }
+
 }

src/DataStreamSuite/DataStream/BlueOrb/MillisecondStream.cs

@@ -43,7 +43,7 @@ namespace bts.BlueOrb {
             _last_fire_time = DateTime.Now;
             IBlueRow nr = new BlueRow(_schema, _id++, _last_fire_time,
             _last_fire_time.ToString("yyyy-MM-dd HH:mm:ss.fff"));
-            _table.add_row(nr);
+            _table = _table.Add(nr);
             var dp = BlueDataUtility.build_dp_inc(_table, myLastValue as DataPacket);
             myFireEvent(dp);
          }

src/DataStreamSuite/DataStream/BlueOrb/MultiRowOpHelper.cs

@@ -150,6 +150,14 @@ namespace bts.BlueOrb {
                return col_vals.Last();
             case MultiRowOp.StdDev:
                return MathNet.Numerics.Statistics.Statistics.StandardDeviation(col_vals);
+            case MultiRowOp.EdgeDiff:
+               return col_vals.First() - col_vals.Last();
+            case MultiRowOp.EdgeDiffPercentage: {
+                  double first = col_vals.First();
+                  double last = col_vals.Last();
+                  return ((first - last) / last) * 100.0;
+                  return col_vals.First() - col_vals.Last();
+               }
             default:
                // TODO
                Log.assert(false);

src/DataStreamSuite/DataStream/BlueOrb/Parquet/IParquetColumnReader.cs

@@ -0,0 +1,29 @@
+namespace Bts.DataStream.Parquet;
+
+using bts.collections;
+
+using System;
+using System.Collections.Generic;
+
+public interface IParquetColumnReader : IDisposable {
+   IList<object?> Read();
+   IList<object?> Read(int? start, int? stop);
+}
+
+public interface IParquetColumnReader<TLogical, TPhysical> : IParquetColumnReader {
+   new IList<TLogical?> Read();
+   new IList<TLogical?> Read(int? start, int? stop);
+
+   IList<object?> IParquetColumnReader.Read() =>
+      Read().WrapSelector(v => (object?)v, null);
+
+   IList<object?> IParquetColumnReader.Read(int? start, int? stop) =>
+      Read(start, stop).WrapSelector(v => (object?)v, null);
+}
+
+public interface IParquetColumnReader<T> : IParquetColumnReader<T, T> {
+   new IList<T?> Read();
+
+   IList<object?> IParquetColumnReader.Read() =>
+      Read().WrapSelector(v => (object?)v, null);
+}

src/DataStreamSuite/DataStream/BlueOrb/Parquet/ParquetAdapter.cs

@@ -0,0 +1,507 @@
+namespace Bts.DataStream.Parquet;
+
+using bts;
+using bts.collections;
+
+using mbts.utils;
+
+using ParquetSharp;
+
+using Spark.Core.Collections;
+
+using System;
+using System.Collections.Generic;
+
+public static class ParquetAdapter {
+   public static IParquetColumnReader GetColumnReader(this RowGroupReader rowGroupReader, int index, Type? colType = null) {
+      int rowCount = checked((int)rowGroupReader.MetaData.NumRows);
+      var colReader = rowGroupReader.Column(index);
+      return getColumnReader(colReader, rowCount, colType: colType);
+   }
+
+   public static IList<object?>[] ReadColumns(this ParquetFileReader fileReader) {
+      var schema = fileReader.FileMetaData.Schema.ToBlueSchema("parquetReader");
+      var colInfos = schema.columnInfo;
+      var colTypes = Array.ConvertAll(colInfos, c => c.colType);
+      var colValues = new IList<object?>[fileReader.FileMetaData.NumColumns];
+      int rowCount = checked((int)fileReader.FileMetaData.NumRows);
+      for (int i = 0; i < colValues.Length; i++) {
+         colValues[i] = new List<object?>(rowCount);
+      }
+      int rowGroupCount = fileReader.FileMetaData.NumRowGroups;
+      for (int rowGroup = 0; rowGroup < rowGroupCount; ++rowGroup) {
+         using var rowGroupReader = fileReader.RowGroup(rowGroup);
+         rowGroupReader.ReadColumnsTo(ref colValues, colTypes);
+      }
+      return colValues;
+   }
+
+   public static IList<object?>[] ReadColumns(this ParquetFileReader fileReader, int? start, int? stop) {
+      if (!start.HasValue && !stop.HasValue) {
+         return ReadColumns(fileReader);
+      }
+      var schema = fileReader.FileMetaData.ToBlueSchema("parquetReader");
+      var colInfos = schema.columnInfo;
+      var colTypes = Array.ConvertAll(colInfos, c => c.colType);
+      var colValues = new IList<object?>[fileReader.FileMetaData.NumColumns];
+      int rowCount = checked((int)fileReader.FileMetaData.NumRows);
+      start ??= 0;
+      stop ??= rowCount;
+      var sliceLength = stop.Value - start.Value;
+      for (int i = 0; i < colValues.Length; i++) {
+         colValues[i] = new List<object?>(sliceLength);
+      }
+      int numRowGroups = fileReader.FileMetaData.NumRowGroups;
+      int? numRowsPerGroup = null;
+      int rowIdx = 0;
+      for (int rowGroup = 0; rowGroup < numRowGroups; ++rowGroup) {
+         if (stop.Value <= rowIdx) {
+            break;
+         }
+         using var rowGroupReader = fileReader.RowGroup(rowGroup);
+         int numRowsInGroup = checked((int)rowGroupReader.MetaData.NumRows);
+         numRowsPerGroup ??= numRowsInGroup;
+         if (rowGroup < numRowGroups - 1) {
+            Log.assert(numRowsInGroup == numRowsPerGroup.Value);
+         } else {
+            Log.assert(numRowsInGroup <= numRowsPerGroup.Value);
+         }
+         int startIdx = start.Value - rowIdx;
+         int stopIdx = stop.Value - rowIdx;
+         rowIdx += numRowsInGroup;
+         if (startIdx >= numRowsInGroup) {
+            continue;
+         }
+         startIdx = Math.Max(0, startIdx);
+         stopIdx = Math.Min(stopIdx, numRowsInGroup);
+         int n = rowGroupReader.ReadColumnsTo(ref colValues, colTypes, startIdx, stopIdx);
+         Log.assert(n == (stopIdx - startIdx));
+      }
+      return colValues;
+   }
+
+   public static int ReadColumnsTo(this RowGroupReader rowGroupReader, ref IList<object?>[] columnValues, Type[] colTypes) {
+      int rowCount = checked((int)rowGroupReader.MetaData.NumRows);
+      for (int colIdx = rowGroupReader.MetaData.NumColumns - 1; colIdx > -1; colIdx--) {
+         using var colReader = rowGroupReader.GetColumnReader(colIdx, colTypes[colIdx]);
+         columnValues[colIdx].AddRange(colReader.Read());
+      }
+      return rowCount;
+   }
+
+   public static int ReadColumnsTo(this RowGroupReader rowGroupReader, ref IList<object?>[] columnValues, Type[] colTypes, int? start, int? stop) {
+      if (!start.HasValue && !stop.HasValue) {
+         return ReadColumnsTo(rowGroupReader, ref columnValues, colTypes);
+      }
+      int numRows = checked((int)rowGroupReader.MetaData.NumRows);
+      if (start.HasValue && start.Value >= numRows) {
+         return 0;
+      }
+      int? readRowCount = null;
+      for (int colIdx = rowGroupReader.MetaData.NumColumns - 1; colIdx > -1; colIdx--) {
+         using var colReader = rowGroupReader.GetColumnReader(colIdx, colTypes[colIdx]);
+         var colVals = colReader.Read(start, stop);
+         readRowCount ??= colVals.Count;
+         Log.assert(readRowCount == colVals.Count);
+         columnValues[colIdx].AddRange(colVals);
+      }
+      return readRowCount ?? throw new ArgumentException("Row group has no columns", nameof(rowGroupReader));
+   }
+
+   static IParquetColumnReader getColumnReader(ColumnReader reader, int bufferSize = 4096, Type? colType = null) {
+      var desc = reader.ColumnDescriptor;
+      var logicalType = desc.LogicalType;
+      return logicalType.Type switch {
+         LogicalTypeEnum.Int => IntReader(),
+         LogicalTypeEnum.String => new ParquetColumnReader<string?>(reader, bufferSize),
+         LogicalTypeEnum.Date => new ParquetColumnReader<DateTime?>(reader, bufferSize),
+         LogicalTypeEnum.Decimal => DecimalReader(),
+         LogicalTypeEnum.Timestamp => TimestampReader(),
+         LogicalTypeEnum.None => DefaultReader(),
+         _ => throw new NotImplementedException(),
+      };
+
+      IParquetColumnReader DefaultReader() {
+         return desc.PhysicalType switch {
+            PhysicalType.Boolean => new ParquetColumnReader<bool?>(reader, bufferSize),
+            PhysicalType.Int32 => new ParquetColumnReader<int?>(reader, bufferSize),
+            PhysicalType.Int64 => new ParquetColumnReader<long?>(reader, bufferSize),
+            PhysicalType.Float => new ParquetColumnReader<float?>(reader, bufferSize),
+            PhysicalType.Double => new ParquetColumnReader<double?>(reader, bufferSize),
+            _ => throw new NotImplementedException(),
+         };
+      }
+
+      IParquetColumnReader TimestampReader() {
+         var tlt = ExceptionAssert.of_type<TimestampLogicalType>(logicalType);
+         return new ParquetColumnReader<DateTimeNanos?, DateTime?>(reader, dtn => dtn?.DateTime, bufferSize);
+      }
+
+      IParquetColumnReader IntReader() {
+         var ilt = ExceptionAssert.of_type<IntLogicalType>(logicalType);
+         if (!ilt.IsSigned) {
+            if (colType is null) {
+               return ilt.BitWidth switch {
+                  8 => new ParquetColumnReader<byte?>(reader, bufferSize),
+                  16 => new ParquetColumnReader<ushort?>(reader, bufferSize),
+                  32 => new ParquetColumnReader<uint?>(reader, bufferSize),
+                  64 => new ParquetColumnReader<ulong?>(reader, bufferSize),
+                  _ => throw new NotImplementedException(),
+               };
+            } else if (colType == typeof(byte)) {
+               return ilt.BitWidth switch {
+                  8 => new ParquetColumnReader<byte?>(reader, bufferSize),
+                  16 => new ParquetColumnReader<ushort?, byte?>(reader, v => v.HasValue ? checked((byte)v.Value) : null, bufferSize),
+                  32 => new ParquetColumnReader<uint?, ushort?>(reader, v => v.HasValue ? checked((byte)v.Value) : null, bufferSize),
+                  64 => new ParquetColumnReader<ulong?, ushort?>(reader, v => v.HasValue ? checked((byte)v.Value) : null, bufferSize),
+                  _ => throw new NotImplementedException(),
+               };
+            } else if (colType == typeof(ushort)) {
+               return ilt.BitWidth switch {
+                  8 => new ParquetColumnReader<byte?, ushort?>(reader, v => v.HasValue ? v.Value : null, bufferSize),
+                  16 => new ParquetColumnReader<ushort?>(reader, bufferSize),
+                  32 => new ParquetColumnReader<uint?, ushort?>(reader, v => v.HasValue ? checked((ushort)v.Value) : null, bufferSize),
+                  64 => new ParquetColumnReader<ulong?, ushort?>(reader, v => v.HasValue ? checked((ushort)v.Value) : null, bufferSize),
+                  _ => throw new NotImplementedException(),
+               };
+            } else if (colType == typeof(uint)) {
+               return ilt.BitWidth switch {
+                  8 => new ParquetColumnReader<byte?, uint?>(reader, v => v.HasValue ? v.Value : null, bufferSize),
+                  16 => new ParquetColumnReader<ushort?, uint?>(reader, v => v.HasValue ? v.Value : null, bufferSize),
+                  32 => new ParquetColumnReader<uint?>(reader, bufferSize),
+                  64 => new ParquetColumnReader<ulong?, uint?>(reader, v => v.HasValue ? checked((uint)v.Value) : null, bufferSize),
+                  _ => throw new NotImplementedException(),
+               };
+            } else if (colType == typeof(ulong)) {
+               return ilt.BitWidth switch {
+                  8 => new ParquetColumnReader<byte?, ulong?>(reader, v => v.HasValue ? v.Value : null, bufferSize),
+                  16 => new ParquetColumnReader<ushort?, ulong?>(reader, v => v.HasValue ? v.Value : null, bufferSize),
+                  32 => new ParquetColumnReader<uint?, ulong?>(reader, v => v.HasValue ? v.Value : null, bufferSize),
+                  64 => new ParquetColumnReader<ulong?>(reader, bufferSize),
+                  _ => throw new NotImplementedException(),
+               };
+            }
+            throw new NotImplementedException($"Expected signed int column type, but got {colType}");
+         } else {
+            if (colType is null) {
+               return ilt.BitWidth switch {
+                  8 => new ParquetColumnReader<sbyte?>(reader, bufferSize),
+                  16 => new ParquetColumnReader<short?>(reader, bufferSize),
+                  32 => new ParquetColumnReader<int?>(reader, bufferSize),
+                  64 => new ParquetColumnReader<long?>(reader, bufferSize),
+                  _ => throw new NotImplementedException(),
+               };
+            } else if (colType == typeof(sbyte)) {
+               return ilt.BitWidth switch {
+                  8 => new ParquetColumnReader<sbyte?>(reader, bufferSize),
+                  16 => new ParquetColumnReader<short?, sbyte?>(reader, v => v.HasValue ? checked((sbyte)v.Value) : null, bufferSize),
+                  32 => new ParquetColumnReader<int?, sbyte?>(reader, v => v.HasValue ? checked((sbyte)v.Value) : null, bufferSize),
+                  64 => new ParquetColumnReader<long?, sbyte?>(reader, v => v.HasValue ? checked((sbyte)v.Value) : null, bufferSize),
+                  _ => throw new NotImplementedException(),
+               };
+            } else if (colType == typeof(short)) {
+               return ilt.BitWidth switch {
+                  8 => new ParquetColumnReader<char?, short?>(reader, v => v.HasValue ? checked((short)v.Value) : null, bufferSize),
+                  16 => new ParquetColumnReader<short?>(reader, bufferSize),
+                  32 => new ParquetColumnReader<int?, short?>(reader, v => v.HasValue ? checked((short)v.Value) : null, bufferSize),
+                  64 => new ParquetColumnReader<long?, short?>(reader, v => v.HasValue ? checked((short)v.Value) : null, bufferSize),
+                  _ => throw new NotImplementedException(),
+               };
+            } else if (colType == typeof(int)) {
+               return ilt.BitWidth switch {
+                  8 => new ParquetColumnReader<char?, int?>(reader, v => v.HasValue ? v.Value : null, bufferSize),
+                  16 => new ParquetColumnReader<short?, int?>(reader, v => v.HasValue ? v.Value : null, bufferSize),
+                  32 => new ParquetColumnReader<int?>(reader, bufferSize),
+                  64 => new ParquetColumnReader<long?, int?>(reader, v => v.HasValue ? checked((int)v.Value) : null, bufferSize),
+                  _ => throw new NotImplementedException(),
+               };
+            } else if (colType == typeof(long)) {
+               return ilt.BitWidth switch {
+                  8 => new ParquetColumnReader<char?, long?>(reader, v => v.HasValue ? v.Value : null, bufferSize),
+                  16 => new ParquetColumnReader<short?, long?>(reader, v => v.HasValue ? v.Value : null, bufferSize),
+                  32 => new ParquetColumnReader<int?, long?>(reader, v => v.HasValue ? v.Value : null, bufferSize),
+                  64 => new ParquetColumnReader<long?>(reader, bufferSize),
+                  _ => throw new NotImplementedException(),
+               };
+            }
+            throw new NotImplementedException($"Expected unsigned int column type, but got {colType}");
+         }
+      }
+
+      IParquetColumnReader DecimalReader() {
+         var dlt = ExceptionAssert.of_type<DecimalLogicalType>(logicalType);
+         return new ParquetColumnReader<double?>(reader, bufferSize);
+      }
+   }
+
+   public static IColumnWriter GetColumnWriter(TypeCode typeCode, ColumnWriter writer, int bufferSize = 4096) {
+      return typeCode switch {
+         TypeCode.Boolean => new ColumnWriter<bool?>(writer, bufferSize),
+         TypeCode.SByte => new ColumnWriter<sbyte?>(writer, bufferSize, GetInt8Converter((IntLogicalType)writer.ColumnDescriptor.LogicalType)),
+         TypeCode.Int16 => new ColumnWriter<short?>(writer, bufferSize, GetInt16Converter((IntLogicalType)writer.ColumnDescriptor.LogicalType)),
+         TypeCode.Int32 => new ColumnWriter<int?>(writer, bufferSize, GetInt32Converter((IntLogicalType)writer.ColumnDescriptor.LogicalType)),
+         TypeCode.Int64 => new ColumnWriter<long?>(writer, bufferSize, GetInt64Converter((IntLogicalType)writer.ColumnDescriptor.LogicalType)),
+         TypeCode.Byte => new ColumnWriter<byte?>(writer, bufferSize, GetUInt8Converter((IntLogicalType)writer.ColumnDescriptor.LogicalType)),
+         TypeCode.UInt16 => new ColumnWriter<ushort?>(writer, bufferSize, GetUInt16Converter((IntLogicalType)writer.ColumnDescriptor.LogicalType)),
+         TypeCode.UInt32 => new ColumnWriter<uint?>(writer, bufferSize, GetUInt32Converter((IntLogicalType)writer.ColumnDescriptor.LogicalType)),
+         TypeCode.UInt64 => new ColumnWriter<ulong?>(writer, bufferSize, GetUInt64Converter((IntLogicalType)writer.ColumnDescriptor.LogicalType)),
+         TypeCode.Single => new ColumnWriter<float?>(writer, bufferSize),
+         TypeCode.Double => new ColumnWriter<double?>(writer, bufferSize),
+         TypeCode.DateTime => new ColumnWriter<DateTimeNanos?>(writer, bufferSize, ConvertDateTimeObj),
+         TypeCode.String => new ColumnWriter<string?>(writer, bufferSize),
+         _ => throw new NotSupportedException($"Unsupported column TypeCode {typeCode}"),
+      };
+
+      static DateTimeNanos? ConvertDateTimeObj(object? obj) {
+         if (obj is DateTime dt) {
+            return new(dt);
+         }
+         if (obj is DateTimeNanos dtn) {
+            return dtn;
+         }
+         throw new InvalidCastException($"Unable to convert object of type {obj?.GetType()} to DateTimeNanos");
+      }
+   }
+
+   public static Func<object?, sbyte?> GetInt8Converter(IntLogicalType logicalType) =>
+      GetInt8Converter(logicalType.IsSigned, logicalType.BitWidth);
+
+   public static Func<object?, sbyte?> GetInt8Converter(bool isSigned, int bitWidth) =>
+      isSigned
+         ? bitWidth switch {
+            8 => obj => obj as sbyte?,
+            16 => obj => obj is short v ? checked((sbyte)v) : null,
+            32 => obj => obj is int v ? checked((sbyte)v) : null,
+            64 => obj => obj is long v ? checked((sbyte)v) : null,
+            _ => throw new NotImplementedException(),
+         }
+         : bitWidth switch {
+            8 => obj => obj is byte v ? checked((sbyte)v) : null,
+            16 => obj => obj is ushort v ? checked((sbyte)v) : null,
+            32 => obj => obj is uint v ? checked((sbyte)v) : null,
+            64 => obj => obj is ulong v ? checked((sbyte)v) : null,
+            _ => throw new NotImplementedException(),
+         };
+
+   public static Func<object?, short?> GetInt16Converter(IntLogicalType logicalType) =>
+      GetInt16Converter(logicalType.IsSigned, logicalType.BitWidth);
+
+   public static Func<object?, short?> GetInt16Converter(bool isSigned, int bitWidth) =>
+      isSigned
+         ? bitWidth switch {
+            8 => obj => obj is sbyte v ? v : null,
+            16 => obj => obj as short?,
+            32 => obj => obj is int v ? checked((short)v) : null,
+            64 => obj => obj is long v ? checked((short)v) : null,
+            _ => throw new NotImplementedException(),
+         }
+         : bitWidth switch {
+            8 => obj => obj is byte v ? v : null,
+            16 => obj => obj is ushort v ? checked((short)v) : null,
+            32 => obj => obj is uint v ? checked((short)v) : null,
+            64 => obj => obj is ulong v ? checked((short)v) : null,
+            _ => throw new NotImplementedException(),
+         };
+
+   public static Func<object?, int?> GetInt32Converter(IntLogicalType logicalType) =>
+      GetInt32Converter(logicalType.IsSigned, logicalType.BitWidth);
+
+   public static Func<object?, int?> GetInt32Converter(bool isSigned, int bitWidth) =>
+      isSigned
+         ? bitWidth switch {
+            8 => obj => obj is sbyte v ? v : null,
+            16 => obj => obj is short v ? v : null,
+            32 => obj => obj as int?,
+            64 => obj => obj is long v ? checked((int)v) : null,
+            _ => throw new NotImplementedException(),
+         }
+         : bitWidth switch {
+            8 => obj => obj is byte v ? v : null,
+            16 => obj => obj is ushort v ? v : null,
+            32 => obj => obj is uint v ? checked((int)v) : null,
+            64 => obj => obj is ulong v ? checked((int)v) : null,
+            _ => throw new NotImplementedException(),
+         };
+
+   public static Func<object?, long?> GetInt64Converter(IntLogicalType logicalType) =>
+      GetInt64Converter(logicalType.IsSigned, logicalType.BitWidth);
+
+   public static Func<object?, long?> GetInt64Converter(bool isSigned, int bitWidth) =>
+      isSigned
+         ? bitWidth switch {
+            8 => obj => obj is sbyte v ? v : null,
+            16 => obj => obj is short v ? v : null,
+            32 => obj => obj is int v ? v : null,
+            64 => obj => obj as long?,
+            _ => throw new NotImplementedException(),
+         }
+         : bitWidth switch {
+            8 => obj => obj is byte v ? v : null,
+            16 => obj => obj is ushort v ? v : null,
+            32 => obj => obj is uint v ? v : null,
+            64 => obj => obj is ulong v ? checked((long)v) : null,
+            _ => throw new NotImplementedException(),
+         };
+
+   public static Func<object?, byte?> GetUInt8Converter(IntLogicalType logicalType) =>
+      GetUInt8Converter(logicalType.IsSigned, logicalType.BitWidth);
+
+   public static Func<object?, byte?> GetUInt8Converter(bool isSigned, int bitWidth) =>
+      isSigned
+         ? bitWidth switch {
+            8 => obj => obj is sbyte v ? checked((byte)v) : null,
+            16 => obj => obj is short v ? checked((byte)v) : null,
+            32 => obj => obj is int v ? checked((byte)v) : null,
+            64 => obj => obj is long v ? checked((byte)v) : null,
+            _ => throw new NotImplementedException(),
+         }
+         : bitWidth switch {
+            8 => obj => obj as byte?,
+            16 => obj => obj is ushort v ? checked((byte)v) : null,
+            32 => obj => obj is uint v ? checked((byte)v) : null,
+            64 => obj => obj is ulong v ? checked((byte)v) : null,
+            _ => throw new NotImplementedException(),
+         };
+
+   public static Func<object?, ushort?> GetUInt16Converter(IntLogicalType logicalType) =>
+      GetUInt16Converter(logicalType.IsSigned, logicalType.BitWidth);
+
+   public static Func<object?, ushort?> GetUInt16Converter(bool isSigned, int bitWidth) =>
+      isSigned
+         ? bitWidth switch {
+            8 => obj => obj is sbyte v ? checked((ushort)v) : null,
+            16 => obj => obj is short v ? checked((ushort)v) : null,
+            32 => obj => obj is int v ? checked((ushort)v) : null,
+            64 => obj => obj is long v ? checked((ushort)v) : null,
+            _ => throw new NotImplementedException(),
+         }
+         : bitWidth switch {
+            8 => obj => obj is byte v ? v : null,
+            16 => obj => obj is ushort v ? v : null,
+            32 => obj => obj is uint v ? checked((ushort)v) : null,
+            64 => obj => obj is ulong v ? checked((ushort)v) : null,
+            _ => throw new NotImplementedException(),
+         };
+
+   public static Func<object?, uint?> GetUInt32Converter(IntLogicalType logicalType) =>
+      GetUInt32Converter(logicalType.IsSigned, logicalType.BitWidth);
+
+   public static Func<object?, uint?> GetUInt32Converter(bool isSigned, int bitWidth) =>
+      isSigned
+         ? bitWidth switch {
+            8 => obj => obj is sbyte v ? checked((uint)v) : null,
+            16 => obj => obj is short v ? checked((uint)v) : null,
+            32 => obj => obj is int v ? checked((uint)v) : null,
+            64 => obj => obj is long v ? checked((uint)v) : null,
+            _ => throw new NotImplementedException(),
+         }
+         : bitWidth switch {
+            8 => obj => obj is byte v ? v : null,
+            16 => obj => obj is ushort v ? v : null,
+            32 => obj => obj is uint v ? v : null,
+            64 => obj => obj is ulong v ? checked((uint)v) : null,
+            _ => throw new NotImplementedException(),
+         };
+
+   public static Func<object?, ulong?> GetUInt64Converter(IntLogicalType logicalType) =>
+      GetUInt64Converter(logicalType.IsSigned, logicalType.BitWidth);
+
+   public static Func<object?, ulong?> GetUInt64Converter(bool isSigned, int bitWidth) =>
+      isSigned
+         ? bitWidth switch {
+            8 => obj => obj is sbyte v ? checked((ulong)v) : null,
+            16 => obj => obj is short v ? checked((ulong)v) : null,
+            32 => obj => obj is int v ? checked((ulong)v) : null,
+            64 => obj => obj is long v ? checked((ulong)v) : null,
+            _ => throw new NotImplementedException(),
+         }
+         : bitWidth switch {
+            8 => obj => obj is byte v ? v : null,
+            16 => obj => obj is ushort v ? v : null,
+            32 => obj => obj is uint v ? v : null,
+            64 => obj => obj is ulong v ? v : null,
+            _ => throw new NotImplementedException(),
+         };
+
+   public static string ColumnEnumTypeKey(string colName) =>
+      $"{colName}.EnumType";
+
+   public static KeyValuePair<string, string> ColumnEnumTypeEntry(string colName, Type enumType) {
+      if (enumType.IsNullable()) {
+         enumType = enumType.GetGenericArguments()[0];
+      }
+      if (!enumType.IsEnum) {
+         throw new ArgumentException($"Must be enum type (actually {enumType})", nameof(enumType));
+      }
+      return new(ColumnEnumTypeKey(colName), enumType.ToString());
+   }
+
+   public static Type? GetColumnEnumType(IReadOnlyDictionary<string, string>? keyValueMetadata, string columnName) {
+      if (keyValueMetadata is not null) {
+         var key = ColumnEnumTypeKey(columnName);
+         if (keyValueMetadata.TryGetValue(key, out var typeName)) {
+            return Type.GetType(typeName);
+         }
+      }
+      return null;
+   }
+
+   public interface IColumnWriter : IDisposable {
+      void Write(IReadOnlyCollection<object?> values);
+   }
+
+   public interface IColumnWriter<T> : IColumnWriter {
+      void Write(IReadOnlyCollection<T?> values);
+   }
+
+   class ColumnWriter<T> : IColumnWriter<T> {
+      readonly T?[] buffer;
+      readonly LogicalColumnWriter<T> writer;
+      readonly Func<object?, T?> convert;
+
+      public ColumnWriter(ColumnWriter writer, int bufferSize)
+         : this(writer, new T?[bufferSize]) { }
+
+      public ColumnWriter(ColumnWriter writer, int bufferSize, Func<object?, T?> convert)
+         : this(writer, new T?[bufferSize], convert) { }
+
+      public ColumnWriter(ColumnWriter writer, T?[] buffer)
+         : this(writer, buffer, obj => (T?)obj) { }
+
+      public ColumnWriter(ColumnWriter writer, T?[] buffer, Func<object?, T?> convert) {
+         this.buffer = buffer;
+         this.writer = writer.LogicalWriter<T>();
+         this.convert = convert;
+      }
+
+      public void Write(IReadOnlyCollection<T?> values) {
+         if (buffer.Length == 0) {
+            throw new InvalidOperationException("Cannot write because buffer length is 0");
+         }
+         var count = values.Count;
+         var nPages = count / buffer.Length;
+         using var en = values.GetEnumerator();
+         for (int pg = 0; pg < nPages; pg++) {
+            for (int i = 0; i < buffer.Length; i++) {
+               Log.assert(en.MoveNext(), "Enumeration failed before reaching expected count");
+               buffer[i] = en.Current;
+            }
+            writer.WriteBatch(buffer!);
+         }
+         var remainder = count - nPages * buffer.Length;
+         if (remainder != 0) {
+            for (int i = 0; i < remainder; i++) {
+               Log.assert(en.MoveNext(), "Enumeration failed before reaching expected count");
+               buffer[i] = en.Current;
+            }
+            writer.WriteBatch(buffer!, 0, remainder);
+         }
+      }
+
+      public void Dispose() =>
+         writer.Dispose();
+
+      void IColumnWriter.Write(IReadOnlyCollection<object?> values) =>
+         Write(ReadOnlySelectorCollection.Create(values, convert));
+   }
+}

src/DataStreamSuite/DataStream/BlueOrb/Parquet/ParquetColumnReader.cs

@@ -0,0 +1,111 @@
+namespace Bts.DataStream.Parquet;
+
+using bts;
+using bts.collections;
+
+using ParquetSharp;
+
+using System;
+using System.Collections.Generic;
+
+class ParquetColumnReader<T> : IParquetColumnReader<T> {
+   protected readonly LogicalColumnReader<T> reader;
+   protected readonly T?[] buffer;
+
+   public ParquetColumnReader(ColumnReader reader, int bufferSize)
+      : this(reader, new T?[bufferSize]) { }
+
+   protected ParquetColumnReader(ColumnReader reader, T?[] buffer) {
+      this.buffer = buffer;
+      this.reader = reader.LogicalReader<T>();
+   }
+
+   ~ParquetColumnReader() =>
+      Dispose();
+
+   public IList<T?> Read() {
+      if (buffer.Length == 0) {
+         throw new InvalidOperationException("Cannot write because buffer length is 0");
+      }
+      int n = reader.ReadBatch(buffer!);
+      return buffer.WrapSubRange(0, n);
+   }
+
+   public IList<T?> Read(int? start, int? stop) {
+      if (buffer.Length == 0) {
+         throw new InvalidOperationException("Cannot write because buffer length is 0");
+      }
+      start ??= 0;
+      stop ??= buffer.Length;
+      int length = stop.Value - start.Value;
+      if (length <= 0) {
+         return new List<T?>(0);
+      }
+      if (start.Value > 0) {
+         var nSkipped = checked((int)reader.Skip(start.Value));
+         Log.assert(nSkipped >= 0 && nSkipped <= start.Value);
+         if (nSkipped < start.Value) {
+            return new List<T?>(0);
+         }
+      }
+      int n = reader.ReadBatch(buffer!, 0, length);
+      return buffer.WrapSubRange(0, n);
+   }
+
+   public void Dispose() {
+      GC.SuppressFinalize(this);
+      reader.Dispose();
+   }
+}
+
+class ParquetColumnReader<TIn, TOut> : IParquetColumnReader<TOut> {
+   readonly Func<TIn, TOut?> convert;
+   readonly LogicalColumnReader<TIn> reader;
+   readonly TIn[] buffer;
+
+   public ParquetColumnReader(ColumnReader reader, Func<TIn, TOut?> convert, int bufferSize)
+      : this(reader, convert, new TIn[bufferSize]) { }
+
+   public ParquetColumnReader(ColumnReader reader, Func<TIn, TOut?> convert, TIn[] buffer) {
+      this.convert = convert;
+      this.reader = reader.LogicalReader<TIn>();
+      this.buffer = buffer;
+   }
+
+   ~ParquetColumnReader() =>
+      Dispose();
+
+   public IList<TOut?> Read() {
+      if (buffer.Length == 0) {
+         throw new InvalidOperationException("Cannot write because buffer length is 0");
+      }
+      int n = reader.ReadBatch(buffer!);
+      return buffer.WrapSelector(convert, null).WrapSubRange(0, n);
+   }
+
+   public IList<TOut?> Read(int? start, int? stop) {
+      if (buffer.Length == 0) {
+         throw new InvalidOperationException("Cannot write because buffer length is 0");
+      }
+      start ??= 0;
+      stop ??= buffer.Length;
+      int length = stop.Value - start.Value;
+      if (length <= 0) {
+         return new List<TOut?>(0);
+      }
+      if (start.Value > 0) {
+         var nSkipped = checked((int)reader.Skip(start.Value));
+         Log.assert(nSkipped >= 0 && nSkipped <= start.Value);
+         if (nSkipped < start.Value) {
+            return new List<TOut?>(0);
+         }
+      }
+      int n = reader.ReadBatch(buffer!, 0, length);
+      return buffer.WrapSelector(convert, null).WrapSubRange(0, n);
+   }
+
+   public void Dispose() {
+      GC.SuppressFinalize(this);
+      reader.Dispose();
+   }
+}

src/DataStreamSuite/DataStream/BlueOrb/Parquet/ParquetExtensions.cs

@@ -0,0 +1,159 @@
+namespace Bts.DataStream.Parquet;
+
+using BCG;
+
+using bts;
+using bts.collections;
+
+using Microsoft.Scripting.Utils;
+
+using ParquetSharp;
+
+using System;
+using System.Collections.Generic;
+using System.Linq;
+
+public static class ParquetExtensions {
+   public static int RowCount(this FileMetaData fileMetaData) =>
+      checked((int)fileMetaData.NumRows);
+
+   public static IBlueTableSchema ToBlueSchema(this SchemaDescriptor schemaDescriptor, string tableName = "parquet", IReadOnlyDictionary<string, string>? keyValueMetadata = null) {
+      var colInfos = new ColumnInfo[schemaDescriptor.NumColumns];
+      for (int i = 0; i < colInfos.Length; i++) {
+         colInfos[i] = schemaDescriptor.BlueColumn(i, keyValueMetadata);
+      }
+      return new BlueTableSchema(tableName, colInfos);
+   }
+
+   public static IBlueTableSchema ToBlueSchema(this FileMetaData fileMetaData, string tableName = "parquet") =>
+      ToBlueSchema(fileMetaData.Schema, tableName, fileMetaData.KeyValueMetadata);
+
+   public static ColumnInfo BlueColumn(this SchemaDescriptor schemaDescriptor, int index, IReadOnlyDictionary<string, string>? keyValueMetadata = null) {
+      var c = schemaDescriptor.Column(index);
+      var lt = c.LogicalType;
+      if (lt.Type == LogicalTypeEnum.Enum) {
+         var enumType = ParquetAdapter.GetColumnEnumType(keyValueMetadata, c.Name);
+         if (enumType is not null) {
+            return new(c.Name, index, enumType);
+         }
+      }
+      return new(c.Name, index, getTypeCode(c.LogicalType, c.PhysicalType));
+   }
+
+   public static IBlueRowCollection ReadBlueTable(this ParquetFileReader fileReader, int? start = null, int? stop = null) {
+      var schema = fileReader.FileMetaData.ToBlueSchema("parquetReader");
+      var colInfos = schema.columnInfo;
+      IBlueRowCollection? rv;
+      if (!schema.columnInfo.Any(c => c.colName == "id")) {
+         rv = BlueAddIndexColumnTable.Create(schema, fileReader.ReadColumns(start, stop));
+      } else {
+         rv = new BlueIndexedColumnTable(schema, fileReader.ReadColumns(start, stop));
+      }
+      Log.assert(!schema.columnInfo.Any(c => c is null));
+      return rv;
+   }
+
+   public static (Column[] Column, IReadOnlyDictionary<string, string> Meta) ToParquetSchema(this IBlueTableSchema schema) {
+      var blueColumns = schema.columnInfo;
+      var parquetColumns = new Column[blueColumns.Length];
+      var parquetMeta = new Dictionary<string, string>();
+      KeyValuePair<string, string>[] parquetColMeta;
+      for (int i = 0; i < blueColumns.Length; i++) {
+         (parquetColumns[i], parquetColMeta) = ToParquetColumn(blueColumns[i]);
+         parquetMeta.AddRange(parquetColMeta);
+      }
+      return (parquetColumns, parquetMeta);
+   }
+
+   public static (Column Column, KeyValuePair<string, string>[] Meta) ToParquetColumn(this ColumnInfo columnInfo) {
+      var colType = columnInfo.colType;
+      bool isNullable = colType.IsNullable();
+      if (isNullable) {
+         colType = colType.GetGenericArguments()[0];
+      }
+      KeyValuePair<string, string>[] meta;
+      TypeCode typeCode;
+      var colName = columnInfo.colName;
+      if (colType.IsEnum) {
+         typeCode = colType.GetEnumUnderlyingType().GetTypeCode();
+         meta = new[] { ParquetAdapter.ColumnEnumTypeEntry(columnInfo.colName, colType) };
+      } else {
+         typeCode = colType.GetTypeCode();
+         meta = Array.Empty<KeyValuePair<string, string>>();
+      }
+      return (ToParquetColumn(colName, typeCode), meta);
+
+      static Column ToParquetColumn(string colName, TypeCode typeCode) =>
+         typeCode switch {
+            TypeCode.Boolean => new Column<bool?>(colName),
+            TypeCode.Byte => new Column<byte?>(colName),
+            TypeCode.Int16 => new Column<short?>(colName),
+            TypeCode.Int32 => new Column<int?>(colName),
+            TypeCode.Int64 => new Column<long?>(colName),
+            TypeCode.UInt16 => new Column<ushort?>(colName),
+            TypeCode.UInt32 => new Column<uint?>(colName),
+            TypeCode.UInt64 => new Column<ulong?>(colName),
+            TypeCode.Single => new Column<float?>(colName),
+            TypeCode.Double => new Column<double?>(colName),
+            TypeCode.DateTime => new Column<DateTimeNanos?>(colName),
+            TypeCode.String => new Column<string?>(colName),
+            _ => throw new NotSupportedException($"Unsupported column TypeCode {typeCode}"),
+         };
+   }
+
+   static TypeCode getTypeCode(LogicalType logicalType, PhysicalType physicalType) {
+      return logicalType.Type switch {
+         // LogicalTypeEnum.Int => TypeCode.Int32,
+         // LogicalTypeEnum.Decimal => TypeCode.Double,
+         LogicalTypeEnum.Date or LogicalTypeEnum.Time or LogicalTypeEnum.Timestamp => TypeCode.DateTime,
+         LogicalTypeEnum.String => TypeCode.String,
+         LogicalTypeEnum.Int => FromInt(),
+         LogicalTypeEnum.Decimal => FromDecimal(),
+         _ => FromPhysicalType(),
+      };
+
+      TypeCode FromPhysicalType() =>
+         physicalType switch {
+            PhysicalType.Boolean => TypeCode.Boolean,
+            PhysicalType.Int32 => TypeCode.Int32,
+            PhysicalType.Int64 => TypeCode.Int64,
+            PhysicalType.Float => TypeCode.Single,
+            PhysicalType.Double => TypeCode.Double,
+            _ => throw new NotImplementedException($"TypeCode not implemented for {logicalType} (logical) {physicalType} (physical)"),
+         };
+
+      TypeCode FromInt() {
+         var ilt = (IntLogicalType)logicalType;
+         var bitWidth = ilt.BitWidth;
+         return ilt.IsSigned ? FromSignedInt() : FromUnsignedInt();
+
+         TypeCode FromSignedInt() =>
+            bitWidth switch {
+               8 => TypeCode.SByte,
+               16 => TypeCode.Int16,
+               32 => TypeCode.Int32,
+               64 => TypeCode.Int64,
+               _ => throw new NotSupportedException($"Bit width not supported for signed int ({bitWidth})"),
+            };
+
+         TypeCode FromUnsignedInt() =>
+            bitWidth switch {
+               8 => TypeCode.Byte,
+               16 => TypeCode.UInt16,
+               32 => TypeCode.UInt32,
+               64 => TypeCode.UInt64,
+               _ => throw new NotSupportedException($"Bit width not supported for unsigned int ({bitWidth})"),
+            };
+      }
+
+      TypeCode FromDecimal() {
+         var dlt = (DecimalLogicalType)logicalType;
+         var scale = dlt.Scale;
+         var precision = dlt.Precision;
+         return scale switch {
+            0 => throw new NotSupportedException($"Scale not supported for decimal ({scale})"),
+            _ => TypeCode.Double,
+         };
+      }
+   }
+}

src/DataStreamSuite/DataStream/BlueOrb/Parquet/ParquetGroupTable.cs

@@ -0,0 +1,234 @@
+namespace Bts.DataStream.Parquet;
+
+using BCG;
+
+using bts;
+using bts.collections;
+
+using Bts.DataStreamCore;
+
+using ParquetSharp;
+
+using Spark.Core.Collections;
+
+using System;
+using System.Collections;
+using System.Collections.Generic;
+using System.Diagnostics.CodeAnalysis;
+
+public class ParquetGroupTable : IBlueRowCollection, IDisposable {
+   readonly RowGroupReader reader;
+   readonly IBlueTableSchema schema;
+   readonly bool leaveOpen;
+   readonly int fullCount;
+   readonly object syncRoot = new();
+   readonly int? start;
+   readonly int? stop;
+   readonly int startIndex;
+   readonly int stopIndex;
+   readonly int idColumnIndex;
+   readonly int idColumnIndexInFile;
+   readonly Func<int, int>? rowIdFactory;
+   DataCache? dataCacheBacking;
+
+   public ParquetGroupTable(
+      RowGroupReader reader,
+      IBlueTableSchema schema,
+      int? start = null,
+      int? stop = null,
+      int idColumnIndex = -1,
+      int idColumnIndexInFile = -1,
+      Func<int, int>? rowIdFactory = null,
+      bool leaveOpen = false
+   ) {
+      this.reader = reader;
+      this.schema = schema;
+      this.start = start;
+      this.stop = stop;
+      this.idColumnIndex = idColumnIndex;
+      this.idColumnIndexInFile = idColumnIndexInFile;
+      this.rowIdFactory = rowIdFactory;
+      this.leaveOpen = leaveOpen;
+      fullCount = checked((int)reader.MetaData.NumRows);
+      Log.assert(fullCount >= 0);
+      startIndex = SliceList.GetStartIndex(start, fullCount, true);
+      stopIndex = SliceList.GetStopIndex(stop, fullCount, true);
+      stopIndex = Math.Max(stopIndex, startIndex);
+      Log.assert(
+         startIndex >= 0
+         && startIndex <= fullCount
+         && stopIndex >= 0
+         && stopIndex <= fullCount
+         && stopIndex >= startIndex);
+   }
+
+   ~ParquetGroupTable() =>
+      Dispose();
+
+   public int Count => stopIndex - startIndex;
+
+   IBlueTableSchema IBlueRowCollection.schema => schema;
+
+   DataCache dataCache =>
+      EnsureInit();
+
+   public IBlueRow this[int id] => dataCache[id];
+
+   public bool containsID(int id) =>
+      dataCache.RowIdToIndex.ContainsKey(id);
+
+   public IBlueRow GetRow(int index) =>
+      dataCache.Rows[index];
+
+   public int IndexOf(IBlueRow item) {
+      int id = item.Id();
+      return dataCache.RowIdToIndex.TryGetValue(id, out var idx) && BlueRowEqualityComparer.Equals(dataCache.Rows[idx], item)
+         ? idx
+         : -1;
+   }
+
+   public bool TryGetValue(int id, [NotNullWhen(true)] out IBlueRow? row) {
+      if (dataCache.RowIdToIndex.TryGetValue(id, out var idx)) {
+         row = dataCache.Rows[idx];
+         return true;
+      }
+      row = null;
+      return false;
+   }
+
+   public IEnumerator<IBlueRow> GetEnumerator() {
+      for (int i = 0; i < dataCache.Rows.Length; i++) {
+         yield return dataCache.Rows[i];
+      }
+   }
+
+   public void Dispose() {
+      GC.SuppressFinalize(this);
+      if (!leaveOpen) {
+         reader.Dispose();
+      }
+   }
+
+   [MemberNotNull(nameof(dataCacheBacking))]
+   internal DataCache EnsureInit() {
+      if (dataCacheBacking.HasValue) {
+         return dataCacheBacking.Value;
+      }
+      lock (syncRoot) {
+         if (dataCacheBacking.HasValue) {
+            return dataCacheBacking.Value;
+         }
+         var rows = new IBlueRow[Count];
+         int numColumns = reader.MetaData.NumColumns;
+         var blueColumns = schema.columnInfo;
+         Log.assert((blueColumns.Length - numColumns) switch {
+            0 or 1 => true,
+            _ => false,
+         });
+         var columnValues = allocateColumnValues(
+            reader, blueColumns.Length, Count, idColumnIndex, idColumnIndexInFile, startIndex, stopIndex, rowIdFactory);
+         var rowIdToIndex = new Dictionary<int, int>();
+         for (int i = 0; i < rows.Length; i++) {
+            rows[i] = new BlueRowColumnWrapper(schema, i, columnValues);
+         }
+         dataCacheBacking = new(rows, rowIdToIndex, columnValues);
+      }
+      return dataCacheBacking.Value;
+   }
+
+   static IList<object?>[] allocateColumnValues(
+      RowGroupReader reader,
+      int numColumns,
+      int numRows,
+      int idColIdx,
+      int idColIdxInFile,
+      int startIndex,
+      int stopIndex,
+      Func<int, int>? idFactory
+   ) {
+      if (idColIdx < 0 || idColIdx >= numColumns) {
+         throw new ArgumentException("Id column index out of bounds", nameof(idColIdx));
+      }
+      var columnValues = new IList<object?>[numColumns];
+      if (idColIdxInFile == -1) {
+         // Generate ids (there is no id column in the file)
+         ArgumentNullException.ThrowIfNull(idFactory);
+         for (int i = 0; i < numColumns; i++) {
+            if (i < idColIdx) {
+               ReadColumn(i, i);
+            } else if (i == idColIdx) {
+               columnValues[idColIdx] = new StepList<int>(0, 1, numRows).WrapSelector(idx => (object?)idFactory(idx), null);
+            } else {
+               ReadColumn(i - 1, i);
+            }
+         }
+      } else if (idFactory is not null) {
+         // Generate ids (skip id column in the file)
+         for (int i = 0; i < numColumns; i++) {
+            if (i == idColIdx) {
+               columnValues[idColIdx] = new StepList<int>(0, 1, numRows).WrapSelector(idx => (object?)idFactory(idx), null);
+            } else {
+               if (i > idColIdx && i < idColIdxInFile) {
+                  ReadColumn(i - 1, i);
+               } else if (i > idColIdxInFile && i < idColIdx) {
+                  ReadColumn(i + 1, i);
+               } else {
+                  ReadColumn(i, i);
+               }
+            }
+         }
+      } else {
+         // Get ids from the file (output table id column index may differ from file)
+         // Case 1) File id col index > output
+         //    src cols: a, b, c, d, (id), e, f, ...
+         //    dst cols: a, b, (id), c, d, e, f, ...
+         //
+         // Case 2) File id col index == output
+         //    src cols: a, b, (id), c, d, e, f, ...
+         //    dst cols: a, b, (id), c, d, e, f, ...
+         //
+         // Case 3) File id col index < output
+         //    src cols: a, b, (id), c, d, e, f, ...
+         //    dst cols: a, b, c, d, (id), e, f, ...
+
+         for (int i = 0; i < numColumns; i++) {
+            if (i == idColIdx) {
+               ReadColumn(idColIdxInFile, idColIdx);
+            } else if (i > idColIdx && i < idColIdxInFile) {
+               ReadColumn(i - 1, i);
+            } else if (i > idColIdxInFile && i < idColIdx) {
+               ReadColumn(i, i - 1);
+            } else {
+               ReadColumn(i, i);
+            }
+         }
+      }
+      return columnValues;
+
+      void ReadColumn(int srcIdx, int dstIdx) {
+         using var colReader = reader.GetColumnReader(srcIdx);
+         columnValues[dstIdx] = colReader.Read(startIndex, stopIndex);
+      }
+   }
+
+   IEnumerator IEnumerable.GetEnumerator() =>
+      GetEnumerator();
+
+   internal readonly struct DataCache {
+      public readonly IBlueRow[] Rows;
+      public readonly Dictionary<int, int> RowIdToIndex;
+      public readonly IList<object?>[] ColumnValues;
+
+      public DataCache(IBlueRow[] rows, Dictionary<int, int> rowIdToIndex, IList<object?>[] columnValues) {
+         Rows = rows;
+         RowIdToIndex = rowIdToIndex;
+         ColumnValues = columnValues;
+      }
+
+      public IBlueRow this[int id] =>
+         Rows[RowIdToIndex[id]];
+
+      public IBlueRow GetRow(int index) =>
+         Rows[index];
+   }
+}

src/DataStreamSuite/DataStream/BlueOrb/Parquet/ParquetReaderStream.cs

@@ -0,0 +1,65 @@
+namespace Bts.DataStream.Parquet;
+
+using BCG;
+
+using bts;
+using bts.collections;
+
+using Bts.DataStreamCore;
+using Bts.PubSub;
+
+using Spark.Core.Disposables;
+
+using System;
+using System.Threading;
+
+[DataStreamUserRespresentation(false)]
+public class ParquetReaderStream : ViewStream {
+   readonly string file;
+   readonly int? start;
+   readonly int? stop;
+   readonly IWorkScheduler workScheduler;
+   IBlueRowCollection? table;
+   IDisposable? workerSub;
+
+   public ParquetReaderStream(string file, IWorkScheduler workScheduler) {
+      this.file = file;
+      this.workScheduler = workScheduler;
+   }
+
+   public ParquetReaderStream(string file, int? start, int? stop, IWorkScheduler workScheduler) {
+      this.file = file;
+      this.start = start;
+      this.stop = stop;
+      this.workScheduler = workScheduler;
+   }
+
+   protected override void myOnFirstSubscriber() {
+      Start();
+      var worker = new AnonymousWorker(DoWork);
+      workScheduler.Add(worker);
+      workerSub = Disposable.Create(() => workScheduler.Remove(worker));
+
+      async void Start() {
+         try {
+            Volatile.Write(ref table, await ParquetTableReader.ReadAsync(file, start, stop));
+         } catch (Exception ex) {
+            Log.DataFlow.error($"[parquet_reader_stream_err.json]{{ \"err\": \"{ex}\" }}");
+         }
+      }
+
+      void DoWork() {
+         var table = Volatile.Read(ref this.table);
+         if (table is null) {
+            return;
+         }
+         Disposable.dispose_thread_safe(ref workerSub);
+         if (table.schema.columnInfo.Length > 1) {
+            myFireEvent(new DataPacket(table, table, new BlueTableImmutable(table.schema)));
+         }
+      }
+   }
+
+   protected override void myOnLastSubscriber() =>
+      Disposable.dispose_thread_safe(ref workerSub);
+}

src/DataStreamSuite/DataStream/BlueOrb/Parquet/ParquetRow.cs

@@ -0,0 +1,26 @@
+// namespace Bts.DataStream.Parquet;
+
+// using BCG;
+
+// using global::Parquet.Data;
+
+// public class ParquetRow : IBlueRow {
+//    readonly int id;
+//    readonly int rowIndex;
+//    readonly DataColumn[] data;
+
+//    public ParquetRow(int id, int rowIndex, DataColumn[] data, IBlueTableSchema schema) {
+//       this.id = id;
+//       this.rowIndex = rowIndex;
+//       this.data = data;
+//       Schema = schema;
+//    }
+
+//    public object? this[int index] =>
+//       index == 0
+//          ? id
+//          : data[index - 1].Data.GetValue(rowIndex);
+
+//    public IBlueTableSchema Schema { get; }
+//    IBlueTableSchema IBlueRow.schema => Schema;
+// }

src/DataStreamSuite/DataStream/BlueOrb/Parquet/ParquetTable.cs

@@ -0,0 +1,145 @@
+namespace Bts.DataStream.Parquet;
+
+using BCG;
+
+using bts;
+using bts.collections;
+
+using Bts.DataStreamCore;
+
+using ParquetSharp;
+
+using Spark.Core.Collections;
+
+using System;
+using System.Collections;
+using System.Collections.Generic;
+using System.Linq;
+using System.Threading.Tasks;
+
+public class ParquetTable : IBlueRowCollection, IDisposable {
+   readonly SortedList<int, ParquetGroupTable> groups = new();
+   readonly int fullCount;
+   readonly IBlueTableSchema schema;
+   readonly ParquetFileReader? reader;
+   readonly int startIndex;
+   readonly int stopIndex;
+
+   public ParquetTable(ParquetFileReader reader, string tableName = "parquet", int? start = null, int? stop = null, bool leaveOpen = false) {
+      schema = reader.FileMetaData.ToBlueSchema(tableName);
+      int idColIdx;
+      if (schema.TryResolveIndices("id", out int idColIdxInFile)) {
+         idColIdx = idColIdxInFile;
+      } else {
+         var origCols = schema.columnInfo;
+         var cols = new ColumnInfo[origCols.Length + 1];
+         idColIdx = 0;
+         cols[0] = new("id", 0, typeof(int));
+         for (int i = 0; i < origCols.Length; i++) {
+            var c = origCols[i];
+            cols[i + 1] = new(c.colName, i + 1, c.colType);
+         }
+         schema = new BlueTableSchema(cols, schema.tableName);
+      }
+      fullCount = checked((int)reader.FileMetaData.NumRows);
+      Log.assert(fullCount >= 0);
+      startIndex = SliceList.GetStartIndex(start, fullCount, true);
+      stopIndex = SliceList.GetStopIndex(stop, fullCount, true);
+      stopIndex = Math.Max(stopIndex, startIndex);
+      Log.assert(startIndex >= 0 && startIndex <= fullCount && stopIndex >= 0 && stopIndex <= fullCount && stopIndex >= startIndex);
+      int numRowGroups = reader.FileMetaData.NumRowGroups;
+      int rowGroupLastIdx = 0;
+      object syncRoot = new();
+      for (int i = 0; i < numRowGroups && rowGroupLastIdx < Count; i++) {
+         int? grpStart = start.HasValue ? startIndex - rowGroupLastIdx : null;
+         int? grpStop = stop.HasValue ? stopIndex - rowGroupLastIdx : null;
+         int offset = rowGroupLastIdx;
+         var grp = new ParquetGroupTable(reader.RowGroup(i), schema, grpStart, grpStop, idColIdx, idColIdxInFile, GetAutoId);
+         if (grp.Count == 0) {
+            continue;
+         }
+         rowGroupLastIdx += grp.Count;
+         groups.Add(rowGroupLastIdx, grp);
+
+         int GetAutoId(int index) =>
+            index + offset;
+      }
+      Parallel.For(0, groups.Count, i => groups.GetValueAtIndex(i).EnsureInit());
+      if (!leaveOpen) {
+         this.reader = reader;
+      }
+   }
+
+   ~ParquetTable() =>
+      Dispose();
+
+   public IBlueTableSchema Schema => schema;
+
+   public int Count => stopIndex - startIndex;
+
+   IBlueTableSchema IBlueRowCollection.schema => schema;
+
+   public IBlueRow this[int index] {
+      get {
+         foreach (var grp in groups.Values) {
+            if (grp.TryGetValue(index, out var r)) {
+               return r;
+            }
+         }
+         throw new KeyNotFoundException();
+      }
+   }
+
+   public static ParquetTable FromFile(string path, int? start = null, int? stop = null, string tableName = "parquet") =>
+      new(new(path), tableName, start, stop, false);
+
+   public IBlueRow GetRow(int index) {
+      if (index < 0 || index >= Count) {
+         throw new IndexOutOfRangeException();
+      }
+      int rowGroupIdx = groups.Keys.BinarySearch(index);
+      if (rowGroupIdx < 0) {
+         rowGroupIdx = ~rowGroupIdx;
+      }
+      Log.assert(rowGroupIdx <= groups.Count);
+      var grp = groups[rowGroupIdx];
+      int idxWithinGrp = index;
+      if (rowGroupIdx > 0) {
+         idxWithinGrp -= groups.Keys[rowGroupIdx - 1];
+      }
+      return grp[idxWithinGrp];
+   }
+
+   public bool containsID(int id) =>
+      groups.Values.Any(rgl => rgl.containsID(id));
+
+   public int IndexOf(IBlueRow item) {
+      int idx;
+      int grpFirstIdx = 0;
+      foreach (var (grpLastIdx, grp) in groups) {
+         idx = grp.IndexOf(item);
+         if (idx >= 0) {
+            return idx + grpFirstIdx;
+         }
+         grpFirstIdx = grpLastIdx;
+      }
+      return -1;
+   }
+
+   public IEnumerator<IBlueRow> GetEnumerator() {
+      foreach (var rgl in groups.Values) {
+         foreach (var r in rgl) {
+            yield return r;
+         }
+      }
+   }
+
+   public void Dispose() {
+      GC.SuppressFinalize(this);
+      groups.Values.DisposeAll();
+      reader?.Dispose();
+   }
+
+   IEnumerator IEnumerable.GetEnumerator() =>
+      GetEnumerator();
+}

src/DataStreamSuite/DataStream/BlueOrb/Parquet/ParquetTableReader.cs

@@ -0,0 +1,32 @@
+#nullable enable
+namespace Bts.DataStream.Parquet;
+
+using BCG;
+
+using Bts.DataStreamCore;
+
+using ParquetSharp;
+
+using System.IO;
+using System.Threading.Tasks;
+
+public class ParquetTableReader {
+   public static Task<IBlueRowCollection> ReadAsync(Stream stream, bool leaveOpen = false, int? start = null, int? stop = null) =>
+      Task.Run(() => Read(stream, leaveOpen, start, stop));
+
+   public static Task<IBlueRowCollection> ReadAsync(string path, int? start = null, int? stop = null) =>
+      Task.Run(() => Read(path, start, stop));
+
+   public static IBlueRowCollection Read(Stream stream, bool leaveOpen = false, int? start = null, int? stop = null) {
+      using var fileReader = new ParquetFileReader(stream, leaveOpen);
+      var rv = fileReader.ReadBlueTable(start, stop);
+      rv.schema.Validate();
+      return rv;
+   }
+
+   public static IBlueRowCollection Read(string path, int? start = null, int? stop = null) {
+      IBlueRowCollection rv = ParquetTable.FromFile(path, start, stop);
+      rv.schema.Validate();
+      return rv;
+   }
+}