feat: Add server side csharp sdk

For Ophil

.config/dotnet-tools.json

@@ -25,6 +25,12 @@
       "commands": [
         "fake"
       ]
+    },
+    "dotnet-t4": {
+      "version": "3.0.0",
+      "commands": [
+        "t4"
+      ]
     }
   }
 }

.editorconfig

@@ -207,17 +207,23 @@ dotnet_naming_symbols.parameters.applicable_accessibilities = public,internal,pr
 dotnet_naming_symbols.private_symbols.applicable_kinds = property,method,field,event,delegate
 dotnet_naming_symbols.private_symbols.applicable_accessibilities = protected,private
 
+# local symbols
+dotnet_naming_symbols.local_symbols.applicable_kinds = local
+
+# parameter symbols
+dotnet_naming_symbols.parameter_symbols.applicable_kinds = parameter
+
 # Naming styles
 
-dotnet_naming_style.snake_case.required_prefix =
-dotnet_naming_style.snake_case.required_suffix =
-dotnet_naming_style.snake_case.word_separator = _
-dotnet_naming_style.snake_case.capitalization = all_lower
+dotnet_naming_style.pascal_case.required_prefix =
+dotnet_naming_style.pascal_case.required_suffix =
+dotnet_naming_style.pascal_case.word_separator =
+dotnet_naming_style.pascal_case.capitalization = pascal_case
 
-dotnet_naming_style.private_snake_case.required_prefix = _
-dotnet_naming_style.private_snake_case.required_suffix =
-dotnet_naming_style.private_snake_case.word_separator = _
-dotnet_naming_style.private_snake_case.capitalization = all_lower
+dotnet_naming_style.camel_case.required_prefix =
+dotnet_naming_style.camel_case.required_suffix =
+dotnet_naming_style.camel_case.word_separator =
+dotnet_naming_style.camel_case.capitalization = camel_case
 
 dotnet_naming_style.user_types.required_prefix =
 dotnet_naming_style.user_types.required_suffix =
@@ -229,6 +235,10 @@ dotnet_naming_style.interface.required_suffix =
 dotnet_naming_style.interface.word_separator =
 dotnet_naming_style.interface.capitalization = pascal_case
 
+dotnet_naming_style.local.capitalization = camel_case
+
+dotnet_naming_style.parameter.capitalization = camel_case
+
 # rules
 dotnet_naming_rule.user_types.symbols = user_types
 dotnet_naming_rule.user_types.style = user_types
@@ -238,14 +248,18 @@ dotnet_naming_rule.interfaces.symbols = interface
 dotnet_naming_rule.interfaces.style = interface_style
 dotnet_naming_rule.interfaces.severity = suggestion
 
-dotnet_naming_rule.public_members_snake_case.symbols = public_symbols
-dotnet_naming_rule.public_members_snake_case.style = snake_case
-dotnet_naming_rule.public_members_snake_case.severity = suggestion
+dotnet_naming_rule.public_members.symbols = public_symbols
+dotnet_naming_rule.public_members.style = pascal_case
+dotnet_naming_rule.public_members.severity = suggestion
 
-dotnet_naming_rule.private_members_snake_case.symbols = private_symbols
-dotnet_naming_rule.private_members_snake_case.style = private_snake_case
-dotnet_naming_rule.private_members_snake_case.severity = suggestion
+dotnet_naming_rule.private_members.symbols = private_symbols
+dotnet_naming_rule.private_members.style = camel_case
+dotnet_naming_rule.private_members.severity = suggestion
 
-dotnet_naming_rule.parameters.symbols = parameters
-dotnet_naming_rule.parameters.style = snake_case
+dotnet_naming_rule.variables_are_camel_case.severity = suggestion
+dotnet_naming_rule.variables_are_camel_case.symbols = local_symbols
+dotnet_naming_rule.variables_are_camel_case.style = local
+
+dotnet_naming_rule.parameters.symbols = parameter_symbols
+dotnet_naming_rule.parameters.style = parameter
 dotnet_naming_rule.parameters.severity = suggestion

.env

@@ -46,4 +46,11 @@ SPARKD__SIM_FIX_PORT=8002
 SPARKD__SHM_SIZE=4gb
 SPARKD__PYTHON_REQUIREMENTS=requirements.txt
 
+### PubSubServer ###
+PUBSUB__CONFIG_DIR=src/DataStreamSuite/PubSubServer/Configurations/Development
+PUBSUB__SOURCE_DIR=build/Release/bin/net8.0/linux-x64/publish
+
+### MdPublisher ###
+MD_PUBLISHER__CONFIG_DIR=src/DataStreamSuite/DataStreamPublisher/Configurations/ReplayMarketData
+MD_PUBLISHER__SOURCE_DIR=build/Release/bin/net8.0/linux-x64/publish
 # TODO: add config for other services (pubsub, md-publisher, webapp, api-server, etc.)

.gitignore

@@ -55,6 +55,8 @@ trace_data/
 .cache/
 /src/Spark.Admin.WebApp/tsconfig.*.g.json
 /src/Spark.Admin.WebApp/src/bin/
+/src/**/*appsettings.Local.json
+/src/**/*/Configurations/Local/
 **/CMakeDeps.cmake
 **/CMakeDeps_win.cmake
 .devcontainer/

.vscode/launch.json

@@ -14,6 +14,49 @@
       "type": "coreclr",
       "request": "attach"
     },
+    {
+      "name": "Api Server",
+      "type": "coreclr",
+      "request": "launch",
+      "program": "",
+      "preLaunchTask": "Build Api Server (Debug)",
+      "windows": {
+        "program": "spark_api_server.exe",
+      },
+      "linux": {
+        "program": "spark_api_server",
+      },
+      "args": [
+        "run",
+      ],
+      "cwd": "build/Debug/bin/net8.0",
+      "env": {
+        "DOTNET_ENVIRONMENT": "${input:apiServerEnvironment}",
+      },
+      "stopAtEntry": false,
+      "console": "integratedTerminal"
+    },
+    {
+      "name": "Api Server (Skip Build)",
+      "type": "coreclr",
+      "request": "launch",
+      "program": "",
+      "windows": {
+        "program": "spark_api_server.exe",
+      },
+      "linux": {
+        "program": "spark_api_server",
+      },
+      "args": [
+        "run",
+      ],
+      "cwd": "build/Debug/bin/net8.0",
+      "env": {
+        "DOTNET_ENVIRONMENT": "${input:apiServerEnvironment}",
+      },
+      "stopAtEntry": false,
+      "console": "integratedTerminal"
+    },
     {
       "name": "DataStreamPublisher",
       "type": "coreclr",
@@ -26,11 +69,28 @@
       "windows": {
         "program": "DataStreamPublisher.exe",
       },
-      "args": [
-        "--config",
-        "${workspaceFolder}/src/DataStreamSuite/DataStreamPublisher/Configurations/${input:dataStreamPublisherConfig}.json"
-      ],
       "cwd": "build/Debug/bin/net8.0",
+      "env": {
+        "DOTNET_ENVIRONMENT": "${input:dataStreamPublisherEnvironment}",
+      },
+      "stopAtEntry": false,
+      "console": "integratedTerminal"
+    },
+    {
+      "name": "DataStreamPublisher (Skip Build)",
+      "type": "coreclr",
+      "request": "launch",
+      "program": "",
+      "linux": {
+        "program": "DataStreamPublisher",
+      },
+      "windows": {
+        "program": "DataStreamPublisher.exe",
+      },
+      "cwd": "${workspaceFolder}/build/Debug/bin/net8.0",
+      "env": {
+        "DOTNET_ENVIRONMENT": "${input:dataStreamPublisherEnvironment}",
+      },
       "stopAtEntry": false,
       "console": "integratedTerminal"
     },
@@ -82,16 +142,29 @@
       "name": "DataStreamQueryAnalyzer",
       "type": "coreclr",
       "request": "launch",
-      //"preLaunchTask": "Build DataStreamQueryAnalyzer (Debug)",
+      "preLaunchTask": "Build DataStreamQueryAnalyzer (Debug)",
       "program": "",
       "windows": {
         "program": "DataStreamQueryAnalyzer.exe",
-        "args": [
-          "--config",
-          "${workspaceFolder}/src/DataStreamSuite/DataStreamQueryAnalyzer/Configurations/DataStreamQueryAnalyzer.json"
-        ],
         "cwd": "build/Debug/bin/net8.0-windows",
         "env": {
+          "DOTNET_ENVIRONMENT": "${input:dataStreamQueryAnalyzerEnvironment}",
+          "PYTHON_EXE": "${workspaceFolder}/.venv/Scripts/python.exe"
+        },
+      },
+      "stopAtEntry": false,
+      "console": "integratedTerminal",
+    },
+    {
+      "name": "DataStreamQueryAnalyzer (Skip Build)",
+      "type": "coreclr",
+      "request": "launch",
+      "program": "",
+      "windows": {
+        "program": "DataStreamQueryAnalyzer.exe",
+        "cwd": "build/Debug/bin/net8.0-windows",
+        "env": {
+          "DOTNET_ENVIRONMENT": "${input:dataStreamQueryAnalyzerEnvironment}",
           "PYTHON_EXE": "${workspaceFolder}/.venv/Scripts/python.exe"
         },
       },
@@ -115,6 +188,27 @@
         "${workspaceFolder}/src/DataStreamSuite/PubSubServer/Configurations/${input:pubSubServerConfig}.json"
       ],
       "cwd": "build/Debug/bin/net8.0",
+      "env": {
+        "DOTNET_ENVIRONMENT": "${input:pubSubServerEnvironment}",
+      },
+      "stopAtEntry": false,
+      "console": "integratedTerminal"
+    },
+    {
+      "name": "PubSubServer (Skip Build)",
+      "type": "coreclr",
+      "request": "launch",
+      "program": "",
+      "linux": {
+        "program": "PubSubServer",
+      },
+      "windows": {
+        "program": "PubSubServer.exe",
+      },
+      "cwd": "build/Debug/bin/net8.0",
+      "env": {
+        "DOTNET_ENVIRONMENT": "${input:pubSubServerEnvironment}",
+      },
       "stopAtEntry": false,
       "console": "integratedTerminal"
     },
@@ -127,6 +221,9 @@
         "preLaunchTask": "Build Spark (Debug)",
         "program": "Spark.exe",
         "cwd": "build/Debug/bin/net8.0-windows",
+        "env": {
+          "DOTNET_ENVIRONMENT": "${input:sparkEnvironment}",
+        }
       },
       "stopAtEntry": false,
       "console": "integratedTerminal"
@@ -139,6 +236,9 @@
       "windows": {
         "program": "Spark.exe",
         "cwd": "build/Debug/bin/net8.0-windows",
+        "env": {
+          "DOTNET_ENVIRONMENT": "${input:sparkEnvironment}",
+        }
       },
       "stopAtEntry": false,
       "console": "integratedTerminal"
@@ -251,6 +351,17 @@
     },
   ],
   "inputs": [
+    {
+      "id": "apiServerEnvironment",
+      "type": "pickString",
+      "description": "Api Server Configuration",
+      "options": [
+        "Development",
+        "Local",
+        "Staging",
+      ],
+      "default": "Development",
+    },
     {
       "id": "dataStreamPublisherConfig",
       "type": "pickString",
@@ -263,6 +374,16 @@
       ],
       "default": "MarketDataPublisher",
     },
+    {
+      "id": "dataStreamPublisherEnvironment",
+      "type": "pickString",
+      "description": "DataStreamPublisher Configuration",
+      "options": [
+        "Local",
+        "ReplayMarketData",
+      ],
+      "default": "Local",
+    },
     {
       "id": "pubSubServerConfig",
       "type": "pickString",
@@ -273,6 +394,37 @@
       ],
       "default": "PubSubServer",
     },
+    {
+      "id": "pubSubServerEnvironment",
+      "type": "pickString",
+      "description": "Api Server Configuration",
+      "options": [
+        "Development",
+        "Local",
+        "Staging",
+      ],
+      "default": "Development",
+    },
+    {
+      "id": "dataStreamQueryAnalyzerEnvironment",
+      "type": "pickString",
+      "description": "DataStreamQueryAnalyzer Configuration",
+      "options": [
+        "Development",
+        "Local",
+      ],
+      "default": "Development",
+    },
+    {
+      "id": "sparkEnvironment",
+      "type": "pickString",
+      "description": "Spark Configuration",
+      "options": [
+        "Development",
+        "Local",
+      ],
+      "default": "Development",
+    },
     {
       "id": "sparkConfigServer",
       "type": "promptString",

.vscode/python.env

@@ -1 +1 @@
-PYTHONPATH=/workspaces/spark/build/sparkd-debug-clang/src/lib_python_interop;/workspaces/spark/src/lib_python_interop/python_interop_plugins
+PYTHONPATH=/workspaces/spark/build/sparkd-debug-clang/src/lib_python_interop;/workspaces/spark/src/lib_python_interop/python_interop_plugins;/workspaces/spark/tools/py

.vscode/tasks.json

@@ -121,6 +121,57 @@
         "color": "terminal.ansiCyan"
       },
     },
+    {
+      "label": "Build DataStreamQueryAnalyzer",
+      "group": "build",
+      "type": "process",
+      "command": "",
+      "windows": {
+        "command": "tools/build_data_stream_query_analyzer.cmd",
+      },
+      "args": [
+        "--configuration",
+        "${input:buildConfiguration}",
+      ],
+      "icon": {
+        "id": "terminal",
+        "color": "terminal.ansiCyan"
+      },
+    },
+    {
+      "label": "Build DataStreamQueryAnalyzer (Debug)",
+      "group": "build",
+      "type": "process",
+      "command": "",
+      "windows": {
+        "command": "tools/build_data_stream_query_analyzer.cmd",
+      },
+      "args": [
+        "--configuration",
+        "Debug",
+      ],
+      "icon": {
+        "id": "terminal",
+        "color": "terminal.ansiCyan"
+      },
+    },
+    {
+      "label": "Build DataStreamQueryAnalyzer (Release)",
+      "group": "build",
+      "type": "process",
+      "command": "",
+      "windows": {
+        "command": "tools/build_data_stream_query_analyzer.cmd",
+      },
+      "args": [
+        "--configuration",
+        "Release",
+      ],
+      "icon": {
+        "id": "terminal",
+        "color": "terminal.ansiCyan"
+      },
+    },
     {
       "label": "Build Spark",
       "group": "build",
@@ -351,9 +402,132 @@
         "id": "wrench"
       },
     },
+    {
+      "label": "Run Api Server",
+      "dependsOn": "Build Api Server",
+      "type": "process",
+      "command": "",
+      "windows": {
+        "command": "spark_api_server.exe",
+      },
+      "linux": {
+        "command": "spark_api_server",
+      },
+      "args": [
+        "run",
+      ],
+      "options": {
+        "cwd": "${workspaceFolder}/build/${input:buildConfiguration}/bin/net8.0",
+        "env": {
+          "DOTNET_ENVIRONMENT": "${input:apiServerEnvironment}",
+        }
+      },
+      "icon": {
+        "id": "run",
+        "color": "terminal.ansiGreen",
+      },
+    },
+    {
+      "label": "Run Api Server (Debug)",
+      "dependsOn": "Build Api Server (Debug)",
+      "type": "process",
+      "command": "",
+      "windows": {
+        "command": "spark_api_server.exe",
+      },
+      "linux": {
+        "command": "spark_api_server",
+      },
+      "args": [
+        "run",
+      ],
+      "options": {
+        "cwd": "${workspaceFolder}/build/Debug/bin/net8.0",
+        "env": {
+          "DOTNET_ENVIRONMENT": "${input:apiServerEnvironment}",
+        }
+      },
+      "icon": {
+        "id": "run",
+        "color": "terminal.ansiGreen",
+      },
+    },
+    {
+      "label": "Run Api Server (Release)",
+      "dependsOn": "Build Api Server (Release)",
+      "type": "process",
+      "command": "",
+      "windows": {
+        "command": "spark_api_server.exe",
+      },
+      "linux": {
+        "command": "spark_api_server",
+      },
+      "args": [
+        "run",
+      ],
+      "options": {
+        "cwd": "${workspaceFolder}/build/Release/bin/net8.0",
+        "env": {
+          "DOTNET_ENVIRONMENT": "${input:apiServerEnvironment}",
+        }
+      },
+      "icon": {
+        "id": "run",
+        "color": "terminal.ansiGreen",
+      },
+    },
+    {
+      "label": "Run Api Server (Debug | Skip Build)",
+      "type": "process",
+      "command": "",
+      "windows": {
+        "command": "spark_api_server.exe",
+      },
+      "linux": {
+        "command": "spark_api_server",
+      },
+      "args": [
+        "run",
+      ],
+      "options": {
+        "cwd": "${workspaceFolder}/build/Debug/bin/net8.0",
+        "env": {
+          "DOTNET_ENVIRONMENT": "${input:apiServerEnvironment}",
+        }
+      },
+      "icon": {
+        "id": "run",
+        "color": "terminal.ansiGreen",
+      },
+    },
+    {
+      "label": "Run Api Server (Release | Skip Build)",
+      "type": "process",
+      "command": "",
+      "windows": {
+        "command": "spark_api_server.exe",
+      },
+      "linux": {
+        "command": "spark_api_server",
+      },
+      "args": [
+        "run",
+      ],
+      "options": {
+        "cwd": "${workspaceFolder}/build/Release/bin/net8.0",
+        "env": {
+          "DOTNET_ENVIRONMENT": "${input:apiServerEnvironment}",
+        }
+      },
+      "icon": {
+        "id": "run",
+        "color": "terminal.ansiGreen",
+      },
+    },
     {
       "label": "Run DataStreamQueryAnalyzer",
-      "dependsOn": "Build DataStream",
+      "dependsOn": "Build DataStreamQueryAnalyzer",
       "type": "process",
       "command": "",
       "windows": {
@@ -373,7 +547,7 @@
     },
     {
       "label": "Run DataStreamQueryAnalyzer (Debug)",
-      "dependsOn": "Build DataStream (Debug)",
+      "dependsOn": "Build DataStreamQueryAnalyzer (Debug)",
       "type": "process",
       "command": "",
       "windows": {
@@ -393,7 +567,7 @@
     },
     {
       "label": "Run DataStreamQueryAnalyzer (Release)",
-      "dependsOn": "Build DataStream (Release)",
+      "dependsOn": "Build DataStreamQueryAnalyzer (Release)",
       "type": "process",
       "command": "",
       "windows": {
@@ -422,12 +596,11 @@
       "windows": {
         "command": "DataStreamPublisher.exe",
       },
-      "args": [
-        "--config",
-        "${workspaceFolder}/src/DataStreamSuite/DataStreamPublisher/Configurations/${input:dataStreamPublisherConfig}.json",
-      ],
       "options": {
-        "cwd": "${workspaceFolder}/build/${input:buildConfiguration}/bin/net8.0"
+        "cwd": "${workspaceFolder}/build/${input:buildConfiguration}/bin/net8.0",
+        "env": {
+          "DOTNET_ENVIRONMENT": "${input:dataStreamPublisherEnvironment}",
+        },
       },
       "icon": {
         "id": "run",
@@ -445,12 +618,11 @@
       "windows": {
         "command": "DataStreamPublisher.exe",
       },
-      "args": [
-        "--config",
-        "${workspaceFolder}/src/DataStreamSuite/DataStreamPublisher/Configurations/${input:dataStreamPublisherConfig}.json",
-      ],
       "options": {
-        "cwd": "${workspaceFolder}/build/Debug/bin/net8.0"
+        "cwd": "${workspaceFolder}/build/Debug/bin/net8.0",
+        "env": {
+          "DOTNET_ENVIRONMENT": "${input:dataStreamPublisherEnvironment}",
+        },
       },
       "icon": {
         "id": "run",
@@ -468,12 +640,11 @@
       "windows": {
         "command": "DataStreamPublisher.exe",
       },
-      "args": [
-        "--config",
-        "${workspaceFolder}/src/DataStreamSuite/DataStreamPublisher/Configurations/${input:dataStreamPublisherConfig}.json",
-      ],
       "options": {
-        "cwd": "${workspaceFolder}/build/Release/bin/net8.0"
+        "cwd": "${workspaceFolder}/build/Release/bin/net8.0",
+        "env": {
+          "DOTNET_ENVIRONMENT": "${input:dataStreamPublisherEnvironment}",
+        },
       },
       "icon": {
         "id": "run",
@@ -491,12 +662,11 @@
       "windows": {
         "command": "PubSubServer.exe",
       },
-      "args": [
-        "--config",
-        "${workspaceFolder}/src/DataStreamSuite/PubSubServer/Configurations/${input:pubSubServerConfig}.json",
-      ],
       "options": {
-        "cwd": "${workspaceFolder}/build/${input:buildConfiguration}/bin/net8.0"
+        "cwd": "${workspaceFolder}/build/${input:buildConfiguration}/bin/net8.0",
+        "env": {
+          "DOTNET_ENVIRONMENT": "${input:pubSubServerEnvironment}",
+        },
       },
       "icon": {
         "id": "run",
@@ -514,12 +684,11 @@
       "windows": {
         "command": "PubSubServer.exe",
       },
-      "args": [
-        "--config",
-        "${workspaceFolder}/src/DataStreamSuite/PubSubServer/Configurations/${input:pubSubServerConfig}.json",
-      ],
       "options": {
-        "cwd": "${workspaceFolder}/build/Debug/bin/net8.0"
+        "cwd": "${workspaceFolder}/build/Debug/bin/net8.0",
+        "env": {
+          "DOTNET_ENVIRONMENT": "${input:pubSubServerEnvironment}",
+        },
       },
       "icon": {
         "id": "run",
@@ -536,12 +705,11 @@
       "windows": {
         "command": "PubSubServer.exe",
       },
-      "args": [
-        "--config",
-        "${workspaceFolder}/src/DataStreamSuite/PubSubServer/Configurations/${input:pubSubServerConfig}.json",
-      ],
       "options": {
-        "cwd": "${workspaceFolder}/build/Debug/bin/net8.0"
+        "cwd": "${workspaceFolder}/build/Debug/bin/net8.0",
+        "env": {
+          "DOTNET_ENVIRONMENT": "${input:pubSubServerEnvironment}",
+        },
       },
       "icon": {
         "id": "run",
@@ -559,12 +727,11 @@
       "windows": {
         "command": "PubSubServer.exe",
       },
-      "args": [
-        "--config",
-        "${workspaceFolder}/src/DataStreamSuite/PubSubServer/Configurations/${input:pubSubServerConfig}.json",
-      ],
       "options": {
-        "cwd": "${workspaceFolder}/build/Release/bin/net8.0"
+        "cwd": "${workspaceFolder}/build/Release/bin/net8.0",
+        "env": {
+          "DOTNET_ENVIRONMENT": "${input:pubSubServerEnvironment}",
+        },
       },
       "icon": {
         "id": "run",
@@ -581,12 +748,11 @@
       "windows": {
         "command": "PubSubServer.exe",
       },
-      "args": [
-        "--config",
-        "${workspaceFolder}/src/DataStreamSuite/PubSubServer/Configurations/${input:pubSubServerConfig}.json",
-      ],
       "options": {
-        "cwd": "${workspaceFolder}/build/Release/bin/net8.0"
+        "cwd": "${workspaceFolder}/build/Release/bin/net8.0",
+        "env": {
+          "DOTNET_ENVIRONMENT": "${input:pubSubServerEnvironment}",
+        },
       },
       "icon": {
         "id": "run",
@@ -852,6 +1018,17 @@
     },
   ],
   "inputs": [
+    {
+      "id": "apiServerEnvironment",
+      "type": "pickString",
+      "description": "Api Server Configuration",
+      "options": [
+        "Development",
+        "Local",
+        "Staging",
+      ],
+      "default": "Development",
+    },
     {
       "id": "buildConfiguration",
       "type": "pickString",
@@ -885,26 +1062,31 @@
       "default": "sparkd-debug-clang",
     },
     {
-      "id": "dataStreamPublisherConfig",
+      "id": "dataStreamPublisherEnvironment",
       "type": "pickString",
       "description": "Publisher Configuration",
       "options": [
-        "MarketDataPublisher",
-        "SimpleEchoClient",
-        "SimpleTestDataPublisher",
-        "TestDataPublisher",
+        "Development",
+        "MarketData",
+        "Production",
+        "ProductionAws",
+        "Staging",
       ],
       "default": "MarketDataPublisher",
     },
     {
-      "id": "pubSubServerConfig",
+      "id": "pubSubServerEnvironment",
       "type": "pickString",
       "description": "PubSub Configuration",
       "options": [
-        "PubSubServer",
-        "SimpleEchoClient",
+        "Development",
+        "EchoServer",
+        "Local",
+        "Production",
+        "ProductionAws",
+        "Staging",
       ],
-      "default": "PubSubServer",
+      "default": "Development",
     },
     {
       "id": "sparkConfigQuery",

CMakeLists.txt

@@ -21,7 +21,7 @@ set(
 # SHARED_MEMORY_PREFIX serves as a kind of namespace so multiple applications
 # can run simultaneously without fighting over the same shm files (e.g., sparkd,
 # api server, etc.).
-set(SHARED_MEMORY_PREFIX "bts" CACHE STRING "Prefix to use when creating shared memory files")
+set(SHARED_MEMORY_PREFIX "\${BTS_SHARED_MEMORY_PREFIX}" CACHE STRING "Prefix to use when creating shared memory files")
 set(EXTERNALS_BIN "${EXTERNALS_ROOT}/bin")
 set(EXTERNALS_INCLUDE "${EXTERNALS_ROOT}/include")
 set(EXTERNALS_LIB "${EXTERNALS_ROOT}/lib/$<CONFIG>")
@@ -40,9 +40,8 @@ include_directories(BEFORE SYSTEM ${EXTERNALS_INCLUDE} ${EXTERNALS_ROOT}/include
 link_directories(${EXTERNALS_LIB})
 
 add_compile_definitions(SHARED_MEMORY_PREFIX="${SHARED_MEMORY_PREFIX}")
-add_compile_definitions(APP_DATA_DIR="${APP_DATA_DIR}")
-if(DEFINED APP_DATA_DIR_OVERRIDE)
-  add_compile_definitions(APP_DATA_DIR_OVERRIDE="${APP_DATA_DIR_OVERRIDE}")
+if(DEFINED APP_DATA_DIR)
+  add_compile_definitions(APP_DATA_DIR="${APP_DATA_DIR}")
 endif()
 add_compile_definitions(EXPORT_MD_CONSUMER="${EXPORT_MD_CONSUMER}")
 if(WIN32)

CMakePresets.json

@@ -20,8 +20,6 @@
         "SPARK_CMAKE_CXX_FLAGS": "/MP /WX /wd4250 /wd4251 /wd4244 /wd4267 /wd4355 /wd4275 /wd4345 /bigobj /EHsc /DCURL_STATICLIB /D_WINSOCK_DEPRECATED_NO_WARNINGS /D__TBB_NO_IMPLICIT_LINKAGE /DBOOST_ALL_NO_LIB /std:c++20 /D_SCL_SECURE_NO_WARNINGS /D_CRT_SECURE_NO_WARNINGS /D_WIN32_WINNT=0x0600"
       },
       "cacheVariables": {
-        "APP_DATA_DIR": "${dollar}{APPDATA}/bts",
-        "APP_DATA_DIR_OVERRIDE": "${dollar}{SPARK_DATA_DIR}",
         "CMAKE_CXX_FLAGS": "$env{SPARK_CMAKE_CXX_FLAGS}",
         "CMAKE_CXX_FLAGS_DEBUG": "$env{SPARK_CMAKE_CXX_FLAGS} /MTd",
         "CMAKE_CXX_FLAGS_RELEASE": "$env{SPARK_CMAKE_CXX_FLAGS} /MT",
@@ -36,8 +34,7 @@
         "EXPORT_MD_CONSUMER": {
           "type": "BOOL",
           "value": "ON"
-        },
-        "SHARED_MEMORY_PREFIX": "spark"
+        }
       }
     },
 
@@ -53,8 +50,6 @@
       "binaryDir": "${sourceDir}/build/${presetName}",
       "cacheVariables": {
         "platform": "",
-        "APP_DATA_DIR": "${dollar}{HOME}/sparkdata",
-        "APP_DATA_DIR_OVERRIDE": "${dollar}{SPARKD_DATA_DIR}",
         "CMAKE_BUILD_WITH_INSTALL_RPATH": {
           "type": "BOOL",
           "value": "OFF"
@@ -77,8 +72,7 @@
         "BUILD_ADMIN_WEB_APP": {
           "type": "BOOL",
           "value": "ON"
-        },
-        "SHARED_MEMORY_PREFIX": "sparkd"
+        }
       }
     },
 
@@ -159,11 +153,7 @@
     {
       "name": "spark-api-server-windows",
       "inherits": "spark",
-      "displayName": "Spark Api Server (Windows)",
-      "cacheVariables": {
-        "APP_DATA_DIR_OVERRIDE": "${dollar}{SPARK_API_SERVER_DATA_DIR}",
-        "SHARED_MEMORY_PREFIX": "spark_api_server"
-      }
+      "displayName": "Spark Api Server (Windows)"
     },
 
     {
@@ -172,7 +162,6 @@
       "displayName": "Spark Api Server (Linux)",
       "generator": "Ninja Multi-Config",
       "cacheVariables": {
-        "APP_DATA_DIR_OVERRIDE": "${dollar}{SPARK_API_SERVER_DATA_DIR}",
         "BUILD_ADMIN_WEB_APP": {
           "type": "BOOL",
           "value": "OFF"
@@ -229,10 +218,8 @@
       "description": "Data Stream utils ported over from Edge",
       "binaryDir": "${sourceDir}/build/${presetName}",
       "cacheVariables": {
-        "APP_DATA_DIR": "${dollar}{APPDATA}/bts/DataStream",
-        "APP_DATA_DIR_OVERRIDE": "${dollar}{DATA_STREAM_DATA_DIR}",
         "CONFIG_DEFAULTS_PATH": "spark/config/profiles/default/*.conf;spark/config/profiles/data_stream/*.conf",
-        "SHARED_MEMORY_PREFIX": "data_stream"
+        "DOTNET_BINARY_DIR": "${sourceDir}/build/$<CONFIG>/bin/net8.0;${sourceDir}/build/$<CONFIG>/bin/net8.0-windows"
       }
     },
 
@@ -248,8 +235,6 @@
       "inherits": "sparkd",
       "generator": "Ninja Multi-Config",
       "cacheVariables": {
-        "APP_DATA_DIR": "${dollar}{HOME}/.local/share/DataStream",
-        "APP_DATA_DIR_OVERRIDE": "${dollar}{DATA_STREAM_DATA_DIR}",
         "BUILD_ADMIN_WEB_APP": {
           "type": "BOOL",
           "value": "OFF"
@@ -267,12 +252,11 @@
         "CMAKE_CXX_FLAGS_DEBUG": "-g -fPIC -Wall -pedantic -Wextra -Werror -Wno-missing-braces -Wno-ignored-qualifiers --std=c++20 -fdiagnostics-color -DDEBUG",
         "CMAKE_CXX_FLAGS_RELEASE": "-fPIC -Wall -pedantic -Wextra -Werror -Wno-missing-braces -Wno-ignored-qualifiers --std=c++20 -fdiagnostics-color -fno-omit-frame-pointer -DNDEBUG -O3",
         "CONFIG_DEFAULTS_PATH": "spark/config/profiles/default/*.conf;spark/config/profiles/data_stream/*.conf",
-        "DOTNET_BINARY_DIR": "${sourceDir}/build/$<CONFIG>/bin/net7.0",
+        "DOTNET_BINARY_DIR": "${sourceDir}/build/$<CONFIG>/bin/net8.0",
         "EXPORT_MD_CONSUMER": {
           "type": "BOOL",
           "value": "ON"
-        },
-        "SHARED_MEMORY_PREFIX": "data_stream"
+        }
       }
     }
   ],

Spark.Api.Server.sln

@@ -42,6 +42,10 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Bts.ColumnExpressions", "sr
 EndProject
 Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "DataStream", "src\DataStreamSuite\DataStream\DataStream.csproj", "{39AEA075-6457-46C0-8821-1158BE6EFA57}"
 EndProject
+Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "sdk", "sdk", "{56ACFCD1-60C7-4E2C-B90D-B1F95ECE33A1}"
+EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Bts.Spark.Sdk", "src\sdk\Bts.Spark.Sdk\Bts.Spark.Sdk.csproj", "{ADA9E802-8EFA-4E72-8E9A-6AA1E4181184}"
+EndProject
 Global
 	GlobalSection(SolutionConfigurationPlatforms) = preSolution
 		Debug|x64 = Debug|x64
@@ -194,6 +198,14 @@ Global
 		{6BD3F6D8-B6FC-44E8-B8FE-06352F638B9B}.Release|x64.Build.0 = Release|Any CPU
 		{6BD3F6D8-B6FC-44E8-B8FE-06352F638B9B}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
 		{6BD3F6D8-B6FC-44E8-B8FE-06352F638B9B}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
+		{ADA9E802-8EFA-4E72-8E9A-6AA1E4181184}.Debug|x64.ActiveCfg = Debug|Any CPU
+		{ADA9E802-8EFA-4E72-8E9A-6AA1E4181184}.Debug|x64.Build.0 = Debug|Any CPU
+		{ADA9E802-8EFA-4E72-8E9A-6AA1E4181184}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
+		{ADA9E802-8EFA-4E72-8E9A-6AA1E4181184}.MinSizeRel|x64.Build.0 = Debug|Any CPU
+		{ADA9E802-8EFA-4E72-8E9A-6AA1E4181184}.Release|x64.ActiveCfg = Release|Any CPU
+		{ADA9E802-8EFA-4E72-8E9A-6AA1E4181184}.Release|x64.Build.0 = Release|Any CPU
+		{ADA9E802-8EFA-4E72-8E9A-6AA1E4181184}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
+		{ADA9E802-8EFA-4E72-8E9A-6AA1E4181184}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
 	EndGlobalSection
 	GlobalSection(SolutionProperties) = preSolution
 		HideSolutionNode = FALSE
@@ -218,5 +230,7 @@ Global
 		{39AEA075-6457-46C0-8821-1158BE6EFA57} = {C09ED521-D2DF-40BC-A76F-EB3CB870B2F6}
 		{9F6D7DE8-185B-4C1D-B809-3314D092DF28} = {C09ED521-D2DF-40BC-A76F-EB3CB870B2F6}
 		{6BD3F6D8-B6FC-44E8-B8FE-06352F638B9B} = {C09ED521-D2DF-40BC-A76F-EB3CB870B2F6}
+		{56ACFCD1-60C7-4E2C-B90D-B1F95ECE33A1} = {267F81EF-A8AB-49FD-9F56-D7EC3C1F7C7C}
+		{ADA9E802-8EFA-4E72-8E9A-6AA1E4181184} = {56ACFCD1-60C7-4E2C-B90D-B1F95ECE33A1}
 	EndGlobalSection
 EndGlobal

cmake/Protobuf.cmake

@@ -29,36 +29,43 @@ function(Protobuf_Generate_Cpp)
     list(APPEND ${_ARGS_CXX_HEADERS} "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.pb.h")
 
     add_custom_command(
-      OUTPUT "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.pb.cc"
-             "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.pb.h"
+      OUTPUT
+        "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.pb.cc"
+        "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.pb.h"
       COMMAND
-        ${Protobuf_PROTOC_EXECUTABLE} ARGS --cpp_out=${_ARGS_CXX_OUT_DIR} ${_proto_include_cli_args} ${ABS_FIL}
+        ${Protobuf_PROTOC_EXECUTABLE}
+      ARGS
+        --cpp_out=${_ARGS_CXX_OUT_DIR} ${_proto_include_cli_args} ${ABS_FIL}
       DEPENDS ${ABS_FIL}
       COMMENT "Running C++ protocol buffer compiler on ${FIL}"
-      VERBATIM)
+      VERBATIM
+    )
   endforeach()
 
-  set_source_files_properties(${${_ARGS_CXX_SOURCES}} ${${_ARGS_CXX_HEADERS}} PROPERTIES GENERATED TRUE)
+  set_source_files_properties(
+    ${${_ARGS_CXX_SOURCES}}
+    ${${_ARGS_CXX_HEADERS}}
+    PROPERTIES
+      GENERATED TRUE
+  )
 
   # remove some warnings from protobuf generated files
   if(WIN32)
-    set_source_files_properties(${${_ARGS_CXX_SOURCES}} PROPERTIES COMPILE_FLAGS
-                                                      "/wd4309 /wd4838")
+    set_source_files_properties(
+      ${${_ARGS_CXX_SOURCES}}
+      PROPERTIES
+        COMPILE_FLAGS "/wd4309 /wd4838"
+    )
   else()
     set_source_files_properties(
       ${${_ARGS_CXX_SOURCES}}
       PROPERTIES
-        COMPILE_FLAGS
-        "-Wno-unused-parameter -Wno-effc++ -Wno-pedantic -Wno-sign-compare -Wno-uninitialized"
+        COMPILE_FLAGS "-Wno-unused-parameter -Wno-effc++ -Wno-pedantic -Wno-sign-compare -Wno-uninitialized"
     )
   endif()
 
-  set(${_ARGS_CXX_SOURCES}
-      ${${_ARGS_CXX_SOURCES}}
-      PARENT_SCOPE)
-  set(${_ARGS_CXX_HEADERS}
-      ${${_ARGS_CXX_HEADERS}}
-      PARENT_SCOPE)
+  set(${_ARGS_CXX_SOURCES} ${${_ARGS_CXX_SOURCES}} PARENT_SCOPE)
+  set(${_ARGS_CXX_HEADERS} ${${_ARGS_CXX_HEADERS}} PARENT_SCOPE)
 endfunction()
 
 function(Service_Compiler_Generate_Cpp)
@@ -92,40 +99,51 @@ function(Service_Compiler_Generate_Cpp)
     list(APPEND ${_ARGS_CXX_HEADERS} "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.server.pb.h")
 
     set(TEMPLATES
-        ${CMAKE_SOURCE_DIR}/src/lib_spark/spark/api/service_compiler/templates/cpp/client.pb.cc
-        ${CMAKE_SOURCE_DIR}/src/lib_spark/spark/api/service_compiler/templates/cpp/client.pb.h
-        ${CMAKE_SOURCE_DIR}/src/lib_spark/spark/api/service_compiler/templates/cpp/server.pb.cc
-        ${CMAKE_SOURCE_DIR}/src/lib_spark/spark/api/service_compiler/templates/cpp/server.pb.h
+      ${CMAKE_SOURCE_DIR}/src/lib_spark/spark/api/service_compiler/templates/cpp/client.pb.cc
+      ${CMAKE_SOURCE_DIR}/src/lib_spark/spark/api/service_compiler/templates/cpp/client.pb.h
+      ${CMAKE_SOURCE_DIR}/src/lib_spark/spark/api/service_compiler/templates/cpp/server.pb.cc
+      ${CMAKE_SOURCE_DIR}/src/lib_spark/spark/api/service_compiler/templates/cpp/server.pb.h
     )
 
     set(_plugin_py ${CMAKE_SOURCE_DIR}/src/lib_spark/spark/api/service_compiler/plugin.py)
     if(WIN32)
       set(_plugin_cmd ${CMAKE_SOURCE_DIR}/src/lib_spark/spark/api/service_compiler/plugin.cmd)
       add_custom_command(
-        OUTPUT "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.pb.cc"
-               "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.pb.h"
-               "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.client.pb.cc"
-               "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.client.pb.h"
-               "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.server.pb.cc"
-               "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.server.pb.h"
+        OUTPUT
+          "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.pb.cc"
+          "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.pb.h"
+          "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.client.pb.cc"
+          "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.client.pb.h"
+          "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.server.pb.cc"
+          "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.server.pb.h"
         COMMAND SET PYTHONPATH=${SPARK_PYTHON}
         COMMAND ${CMAKE_SOURCE_DIR}/.venv/Scripts/activate.bat
         COMMAND
-          ${Protobuf_PROTOC_EXECUTABLE} ARGS --cpp_out=${_ARGS_CXX_OUT_DIR}
-          --plugin=protoc-gen-service=${_plugin_cmd}
-          --service_out=lang=cpp:${_ARGS_CXX_OUT_DIR} -I
-          ${PROTOBUF_INCLUDE_DIRS} -I ${_protobuf_src_dir} ${ABS_FIL}
-        DEPENDS ${ABS_FIL} ${TEMPLATES} ${_plugin_cmd} ${_plugin_py}
+          ${Protobuf_PROTOC_EXECUTABLE}
+          ARGS
+            --cpp_out=${_ARGS_CXX_OUT_DIR}
+            --plugin=protoc-gen-service=${_plugin_cmd}
+            --service_out=lang=cpp:${_ARGS_CXX_OUT_DIR}
+            -I ${PROTOBUF_INCLUDE_DIRS}
+            -I ${_protobuf_src_dir}
+            ${ABS_FIL}
+        DEPENDS
+          ${ABS_FIL}
+          ${TEMPLATES}
+          ${_plugin_cmd}
+          ${_plugin_py}
         COMMENT "Running C++ service compiler on ${FIL}"
-        VERBATIM)
+        VERBATIM
+      )
     else()
       add_custom_command(
-        OUTPUT "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.pb.cc"
-               "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.pb.h"
-               "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.client.pb.cc"
-               "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.client.pb.h"
-               "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.server.pb.cc"
-               "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.server.pb.h"
+        OUTPUT
+          "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.pb.cc"
+          "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.pb.h"
+          "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.client.pb.cc"
+          "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.client.pb.h"
+          "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.server.pb.cc"
+          "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.server.pb.h"
         COMMAND ${CMAKE_SOURCE_DIR}/tools/venv_exec.sh
         ARGS
           ${Protobuf_PROTOC_EXECUTABLE}
@@ -133,32 +151,32 @@ function(Service_Compiler_Generate_Cpp)
           --plugin=protoc-gen-service=${_plugin_py}
           --service_out=lang=cpp,make_file_name=${ABS_FIL}:${_ARGS_CXX_OUT_DIR}
           ${_proto_include_cli_args} ${ABS_FIL}
-        DEPENDS ${ABS_FIL} ${TEMPLATES} ${_plugin_py}
+        DEPENDS
+          ${ABS_FIL}
+          ${TEMPLATES}
+          ${_plugin_py}
         COMMENT "Running C++ service compiler on ${FIL}"
-        VERBATIM)
+        VERBATIM
+      )
     endif()
-
   endforeach()
 
   # remove some warnings from protobuf generated files
   if(WIN32)
-    set_source_files_properties(${${_ARGS_CXX_SOURCES}} PROPERTIES COMPILE_FLAGS
-                                                      "/wd4309 /wd4838")
+    set_source_files_properties(
+      ${${_ARGS_CXX_SOURCES}}
+      PROPERTIES
+        COMPILE_FLAGS "/wd4309 /wd4838"
+    )
   else()
     set_source_files_properties(
       ${${_ARGS_CXX_SOURCES}}
       PROPERTIES
-        COMPILE_FLAGS
-        "-Wno-unused-parameter -Wno-effc++ -Wno-pedantic -Wno-sign-compare -Wno-uninitialized"
+        COMPILE_FLAGS "-Wno-unused-parameter -Wno-effc++ -Wno-pedantic -Wno-sign-compare -Wno-uninitialized"
     )
   endif()
 
-  set(${_ARGS_CXX_SOURCES}
-      ${${_ARGS_CXX_SOURCES}}
-      PARENT_SCOPE)
-  set(${_ARGS_CXX_HEADERS}
-      ${${_ARGS_CXX_HEADERS}}
-      PARENT_SCOPE)
-
+  set(${_ARGS_CXX_SOURCES} ${${_ARGS_CXX_SOURCES}} PARENT_SCOPE)
+  set(${_ARGS_CXX_HEADERS} ${${_ARGS_CXX_HEADERS}} PARENT_SCOPE)
 endfunction()
 

compose.yaml

@@ -1,7 +1,7 @@
 services:
   sparkd:
     volumes:
-      - ${SPARKD__DATA_DIR:-sparkd_data}:/home/bts/sparkdata/data
+      - ${SPARKD__DATA_DIR:-sparkd_data}:/home/bts/.local/share/bts/sparkd/data
     build:
       dockerfile: src/app_sparkd/Dockerfile
       args:
@@ -12,19 +12,19 @@ services:
     shm_size: "${SPARKD__SHM_SIZE:-512mb}"
     command: >
       sh -c "
-        echo bts | sudo -S chown -R bts:bts /home/bts/sparkdata/data
-        if [ -e "/home/bts/sparkdata/data/logging.sparkd/log" ]; then
-          mv "/home/bts/sparkdata/data/logging.sparkd/log" "/home/bts/sparkdata/data/logging.sparkd/log_$(date '+%Y%m%d-%H%M')"
+        echo bts | sudo -S chown -R bts:bts /home/bts/.local/share/bts/sparkd/data
+        if [ -e "/home/bts/.local/share/bts/sparkd/data/logging.sparkd/log" ]; then
+          mv "/home/bts/.local/share/bts/sparkd/data/logging.sparkd/log" "/home/bts/.local/share/bts/sparkd/data/logging.sparkd/log_$(date '+%Y%m%d-%H%M')"
         fi
-        if [ -e "/home/bts/sparkdata/conf/.env.local" ]; then
-          echo 'Sourcing /home/bts/sparkdata/conf/.env.local'
+        if [ -e "/home/bts/.local/share/bts/sparkd/conf/.env.local" ]; then
+          echo 'Sourcing /home/bts/.local/share/bts/sparkd/conf/.env.local'
           set -a
-          . /home/bts/sparkdata/conf/.env.local
+          . /home/bts/.local/share/bts/sparkd/conf/.env.local
           set +a
         fi
-        if [ -d /home/bts/sparkdata/conf/plugins ]; then
-          echo 'Copying /home/bts/sparkdata/conf/plugins to runtime dir (/home/bts/bin)'
-          cp -r /home/bts/sparkdata/conf/plugins /home/bts/bin/
+        if [ -d /home/bts/.local/share/bts/sparkd/conf/plugins ]; then
+          echo 'Copying /home/bts/.local/share/bts/sparkd/conf/plugins to runtime dir (/home/bts/bin)'
+          cp -r /home/bts/.local/share/bts/sparkd/conf/plugins /home/bts/bin/
         fi
         ./sparkd run"
     env_file:
@@ -41,8 +41,8 @@ services:
 
   sparkd-symbols:
     volumes:
-      - ${SPARKD__DATA_DIR:-sparkd_data}:/home/bts/sparkdata/data
-      - ${SPARKD__RUNTIME_CONF_DIR:-sparkd_conf}:/home/bts/sparkdata/conf
+      - ${SPARKD__DATA_DIR:-sparkd_data}:/home/bts/.local/share/bts/sparkd/data
+      - ${SPARKD__RUNTIME_CONF_DIR:-sparkd_conf}:/home/bts/.local/share/bts/sparkd/conf
     build:
       dockerfile: src/app_sparkd/Dockerfile
       args:
@@ -53,21 +53,21 @@ services:
     shm_size: "${SPARKD__SHM_SIZE:-512mb}"
     command: >
       sh -c "
-        echo bts | sudo -S chown -R bts:bts /home/bts/sparkdata
-        if [ -e "/home/bts/sparkdata/data/logging.sparkd/log" ]; then
-          mv "/home/bts/sparkdata/data/logging.sparkd/log" "/home/bts/sparkdata/data/logging.sparkd/log_$(date '+%Y%m%d-%H%M')"
+        echo bts | sudo -S chown -R bts:bts /home/bts/.local/share/bts/sparkd
+        if [ -e "/home/bts/.local/share/bts/sparkd/data/logging.sparkd/log" ]; then
+          mv "/home/bts/.local/share/bts/sparkd/data/logging.sparkd/log" "/home/bts/.local/share/bts/sparkd/data/logging.sparkd/log_$(date '+%Y%m%d-%H%M')"
         fi
-        ls -alh /home/bts/sparkdata/conf/
-        cat /home/bts/sparkdata/conf/.env.local
-        if [ -e "/home/bts/sparkdata/conf/.env.local" ]; then
-          echo 'Sourcing /home/bts/sparkdata/conf/.env.local'
+        ls -alh /home/bts/.local/share/bts/sparkd/conf/
+        cat /home/bts/.local/share/bts/sparkd/conf/.env.local
+        if [ -e "/home/bts/.local/share/bts/sparkd/conf/.env.local" ]; then
+          echo 'Sourcing /home/bts/.local/share/bts/sparkd/conf/.env.local'
           set -a
-          . /home/bts/sparkdata/conf/.env.local
+          . /home/bts/.local/share/bts/sparkd/conf/.env.local
           set +a
         fi
-        if [ -d /home/bts/sparkdata/conf/plugins ]; then
-          echo 'Copying /home/bts/sparkdata/conf/plugins to runtime dir (/home/bts/bin)'
-          cp -r /home/bts/sparkdata/conf/plugins /home/bts/bin/
+        if [ -d /home/bts/.local/share/bts/sparkd/conf/plugins ]; then
+          echo 'Copying /home/bts/.local/share/bts/sparkd/conf/plugins to runtime dir (/home/bts/bin)'
+          cp -r /home/bts/.local/share/bts/sparkd/conf/plugins /home/bts/bin/
         fi
         ./sparkd run"
     env_file:
@@ -91,10 +91,10 @@ services:
       - type: volume
         read_only: true
         source: sparkd_data
-        target: /home/bts/sparkdata/data/logging.sparkd
+        target: /home/bts/.local/share/bts/sparkd/data/logging.sparkd
         volume:
-          subpath: logging.sparkd
-    command: ["tail", "-f", "/home/bts/sparkdata/data/logging.sparkd/log"]
+          subpath: logging
+    command: ["tail", "-f", "/home/bts/.local/share/bts/sparkd/data/logging.sparkd/log"]
 
   sparkd-ping-test:
     depends_on:
@@ -109,6 +109,89 @@ services:
         RUN apt-get update && apt-get -y install iputils-ping
     command: [ "ping", "sparkd" ]
 
+  pubsub:
+    volumes:
+      - ${PUBSUB__DATA_DIR:-pubsub_data}:/home/bts/.local/share/bts/PubSubServer/data
+    build:
+      dockerfile: src/DataStreamSuite/PubSubServer/Dockerfile
+      args:
+        - SOURCE_DIR=${PUBSUB__SOURCE_DIR}
+        - CONFIG_DIR=${PUBSUB__CONFIG_DIR}
+    ports:
+      - 22019:22019
+    command: >
+      sh -c "
+        echo bts | sudo -S chown -R bts:bts /home/bts/.local/share/bts/PubSubServer
+        if [ -e "/home/bts/.local/share/bts/PubSubServer/data/logging.PubSubServer/log" ]; then
+          mv "/home/bts/.local/share/bts/PubSubServer/data/logging.PubSubServer/log" "/home/bts/.local/share/bts/PubSubServer/data/logging.PubSubServer/log_$(date '+%Y%m%d-%H%M')"
+        fi
+        if [ -e "/home/bts/.local/share/bts/PubSubServer/conf/.env" ]; then
+          echo 'Sourcing /home/bts/.local/share/bts/PubSubServer/conf/.env'
+          set -a
+          . /home/bts/.local/share/bts/PubSubServer/conf/.env
+          set +a
+        fi
+        if [ -e "/home/bts/.local/share/bts/PubSubServer/conf/.env.local" ]; then
+          echo 'Sourcing /home/bts/.local/share/bts/PubSubServer/conf/.env.local'
+          set -a
+          . /home/bts/.local/share/bts/PubSubServer/conf/.env.local
+          set +a
+        fi
+        ./PubSubServer"
+    env_file:
+      - .env
+      - ${PUBSUB__CONFIG_DIR}/.env
+
+  md-publisher:
+    volumes:
+      - ${MD_PUBLISHER__DATA_DIR:-md_publisher_data}:/home/bts/.local/share/bts/DataStreamPublisher/data
+      - historic_md:/home/bts/.local/share/bts/databento
+    build:
+      dockerfile: src/DataStreamSuite/DataStreamPublisher/Dockerfile
+      args:
+        - SOURCE_DIR=${MD_PUBLISHER__SOURCE_DIR}
+        - CONFIG_DIR=${MD_PUBLISHER__CONFIG_DIR}
+    command: >
+      sh -c "
+        echo bts | sudo -S chown -R bts:bts /home/bts/.local/share/bts/DataStreamPublisher
+        if [ -e "/home/bts/.local/share/bts/DataStreamPublisher/data/logging.DataStreamPublisher/log" ]; then
+          mv "/home/bts/.local/share/bts/DataStreamPublisher/data/logging.DataStreamPublisher/log" "/home/bts/.local/share/bts/DataStreamPublisher/data/logging.DataStreamPublisher/log_$(date '+%Y%m%d-%H%M')"
+        fi
+        if [ -e "/home/bts/.local/share/bts/DataStreamPublisher/conf/.env" ]; then
+          echo 'Sourcing /home/bts/.local/share/bts/DataStreamPublisher/conf/.env'
+          set -a
+          . /home/bts/.local/share/bts/DataStreamPublisher/conf/.env
+          set +a
+        fi
+        if [ -e "/home/bts/.local/share/bts/DataStreamPublisher/conf/.env.local" ]; then
+          echo 'Sourcing /home/bts/.local/share/bts/DataStreamPublisher/conf/.env.local'
+          set -a
+          . /home/bts/.local/share/bts/DataStreamPublisher/conf/.env.local
+          set +a
+        fi
+      dockerize -wait tcp://pubsub:22019 -timeout 10s ./DataStreamPublisher"
+    env_file:
+      - .env
+      - ${MD_PUBLISHER__CONFIG_DIR}/.env
+
+  api-server:
+    depends_on:
+      - sparkd
+    ports:
+      - 8008:8008
+      - 9443:9443
+    build:
+      dockerfile: src/client/spark_api_server/Dockerfile
+    command: [ "dockerize", "-wait", "tcp://sparkd:8000", "-timeout", "10s", "./spark_api_server", "run" ]
+    environment:
+      - ASPNETCORE_ENVIRONMENT=Staging
+
 volumes:
   sparkd_data:
     name: "sparkd_${SPARKD__CONFIG_PORT:-8000}"
+  pubsub_data:
+    name: "pubsub_22019"
+  md_publisher_data:
+    name: "md_publisher_22019"
+  historic_md:
+    name: "historic_md"

global.json

@@ -1,6 +0,0 @@
-{
-  "sdk": {
-    "version": "6.0.200",
-    "rollForward": "latestMajor"
-  }
-}

spark_client.sln

@@ -1,143 +1,164 @@
-Microsoft Visual Studio Solution File, Format Version 12.00
-# Visual Studio Version 17
-VisualStudioVersion = 17.3.32922.545
-MinimumVisualStudioVersion = 15.0.26124.0
-Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "spark_api", "src\client\spark_api\spark_api.csproj", "{DC9A8975-8211-4FC9-A9BA-C8B9D26AD481}"
-EndProject
-Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Spark", "src\client\spark\Spark.csproj", "{9A4D4D07-1AB6-4B2B-B111-A4F34D6C267E}"
-EndProject
-Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Spark.Console", "src\client\Spark.Console\Spark.Console.csproj", "{387566C6-C6F6-412C-A30C-9855BBCB4816}"
-EndProject
-Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "spark_core", "src\client\spark_core\spark_core.csproj", "{6E449EB6-20AC-4FA6-B8D9-10F4BB7E1FEE}"
-EndProject
-Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "spark_sdk", "src\client\spark_sdk\spark_sdk.csproj", "{6963C314-83C4-4667-B0D0-C2F7AE208C29}"
-EndProject
-Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "spark_test", "src\client\spark_test\spark_test.csproj", "{ECD78CC5-138F-4809-A764-2358C2AC2E82}"
-EndProject
-Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "spark_websocket", "src\client\spark_websocket\spark_websocket.csproj", "{3201359D-C071-4089-9EDE-B04D759F1508}"
-EndProject
-Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "spark_api_server", "src\client\spark_api_server\spark_api_server.csproj", "{2B8C5D01-FF78-4BFA-B332-5E430586DAF9}"
-EndProject
-Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "DataStreamCore", "src\DataStreamSuite\DataStreamCore\DataStreamCore.csproj", "{4E4F491F-C688-43C4-88DA-AC92BC5FA1B8}"
-EndProject
-Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "DataStream", "src\DataStreamSuite\DataStream\DataStream.csproj", "{2AC54ADC-3246-427E-9CA9-FEA2DBCB9432}"
-EndProject
-Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "SharedStreams", "src\DataStreamSuite\SharedStreams\SharedStreams.csproj", "{4E96E5CC-7791-40B3-A8A1-C34FBB736B3B}"
-EndProject
-Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "PubSub", "src\DataStreamSuite\PubSub\PubSub.csproj", "{51C40BA9-F226-4A35-939D-1ED6F5FF7299}"
-EndProject
-Global
-	GlobalSection(SolutionConfigurationPlatforms) = preSolution
-		Debug|x64 = Debug|x64
-		MinSizeRel|x64 = MinSizeRel|x64
-		Release|x64 = Release|x64
-		RelWithDebInfo|x64 = RelWithDebInfo|x64
-	EndGlobalSection
-	GlobalSection(ProjectConfigurationPlatforms) = postSolution
-		{DC9A8975-8211-4FC9-A9BA-C8B9D26AD481}.Debug|x64.ActiveCfg = Debug|x64
-		{DC9A8975-8211-4FC9-A9BA-C8B9D26AD481}.Debug|x64.Build.0 = Debug|x64
-		{DC9A8975-8211-4FC9-A9BA-C8B9D26AD481}.MinSizeRel|x64.ActiveCfg = Release|x64
-		{DC9A8975-8211-4FC9-A9BA-C8B9D26AD481}.MinSizeRel|x64.Build.0 = Release|x64
-		{DC9A8975-8211-4FC9-A9BA-C8B9D26AD481}.Release|x64.ActiveCfg = Release|x64
-		{DC9A8975-8211-4FC9-A9BA-C8B9D26AD481}.Release|x64.Build.0 = Release|x64
-		{DC9A8975-8211-4FC9-A9BA-C8B9D26AD481}.RelWithDebInfo|x64.ActiveCfg = Release|x64
-		{DC9A8975-8211-4FC9-A9BA-C8B9D26AD481}.RelWithDebInfo|x64.Build.0 = Release|x64
-		{9A4D4D07-1AB6-4B2B-B111-A4F34D6C267E}.Debug|x64.ActiveCfg = Debug|x64
-		{9A4D4D07-1AB6-4B2B-B111-A4F34D6C267E}.Debug|x64.Build.0 = Debug|x64
-		{9A4D4D07-1AB6-4B2B-B111-A4F34D6C267E}.MinSizeRel|x64.ActiveCfg = Release|x64
-		{9A4D4D07-1AB6-4B2B-B111-A4F34D6C267E}.MinSizeRel|x64.Build.0 = Release|x64
-		{9A4D4D07-1AB6-4B2B-B111-A4F34D6C267E}.Release|x64.ActiveCfg = Release|x64
-		{9A4D4D07-1AB6-4B2B-B111-A4F34D6C267E}.Release|x64.Build.0 = Release|x64
-		{9A4D4D07-1AB6-4B2B-B111-A4F34D6C267E}.RelWithDebInfo|x64.ActiveCfg = Release|x64
-		{9A4D4D07-1AB6-4B2B-B111-A4F34D6C267E}.RelWithDebInfo|x64.Build.0 = Release|x64
-		{387566C6-C6F6-412C-A30C-9855BBCB4816}.Debug|x64.ActiveCfg = Debug|x64
-		{387566C6-C6F6-412C-A30C-9855BBCB4816}.Debug|x64.Build.0 = Debug|x64
-		{387566C6-C6F6-412C-A30C-9855BBCB4816}.MinSizeRel|x64.ActiveCfg = Debug|x64
-		{387566C6-C6F6-412C-A30C-9855BBCB4816}.MinSizeRel|x64.Build.0 = Debug|x64
-		{387566C6-C6F6-412C-A30C-9855BBCB4816}.Release|x64.ActiveCfg = Release|x64
-		{387566C6-C6F6-412C-A30C-9855BBCB4816}.Release|x64.Build.0 = Release|x64
-		{387566C6-C6F6-412C-A30C-9855BBCB4816}.RelWithDebInfo|x64.ActiveCfg = Release|x64
-		{387566C6-C6F6-412C-A30C-9855BBCB4816}.RelWithDebInfo|x64.Build.0 = Release|x64
-		{6E449EB6-20AC-4FA6-B8D9-10F4BB7E1FEE}.Debug|x64.ActiveCfg = Debug|x64
-		{6E449EB6-20AC-4FA6-B8D9-10F4BB7E1FEE}.Debug|x64.Build.0 = Debug|x64
-		{6E449EB6-20AC-4FA6-B8D9-10F4BB7E1FEE}.MinSizeRel|x64.ActiveCfg = Release|x64
-		{6E449EB6-20AC-4FA6-B8D9-10F4BB7E1FEE}.MinSizeRel|x64.Build.0 = Release|x64
-		{6E449EB6-20AC-4FA6-B8D9-10F4BB7E1FEE}.Release|x64.ActiveCfg = Release|x64
-		{6E449EB6-20AC-4FA6-B8D9-10F4BB7E1FEE}.Release|x64.Build.0 = Release|x64
-		{6E449EB6-20AC-4FA6-B8D9-10F4BB7E1FEE}.RelWithDebInfo|x64.ActiveCfg = Release|x64
-		{6E449EB6-20AC-4FA6-B8D9-10F4BB7E1FEE}.RelWithDebInfo|x64.Build.0 = Release|x64
-		{6963C314-83C4-4667-B0D0-C2F7AE208C29}.Debug|x64.ActiveCfg = Debug|x64
-		{6963C314-83C4-4667-B0D0-C2F7AE208C29}.Debug|x64.Build.0 = Debug|x64
-		{6963C314-83C4-4667-B0D0-C2F7AE208C29}.MinSizeRel|x64.ActiveCfg = Release|x64
-		{6963C314-83C4-4667-B0D0-C2F7AE208C29}.MinSizeRel|x64.Build.0 = Release|x64
-		{6963C314-83C4-4667-B0D0-C2F7AE208C29}.Release|x64.ActiveCfg = Release|x64
-		{6963C314-83C4-4667-B0D0-C2F7AE208C29}.Release|x64.Build.0 = Release|x64
-		{6963C314-83C4-4667-B0D0-C2F7AE208C29}.RelWithDebInfo|x64.ActiveCfg = Release|x64
-		{6963C314-83C4-4667-B0D0-C2F7AE208C29}.RelWithDebInfo|x64.Build.0 = Release|x64
-		{ECD78CC5-138F-4809-A764-2358C2AC2E82}.Debug|x64.ActiveCfg = Debug|x64
-		{ECD78CC5-138F-4809-A764-2358C2AC2E82}.Debug|x64.Build.0 = Debug|x64
-		{ECD78CC5-138F-4809-A764-2358C2AC2E82}.MinSizeRel|x64.ActiveCfg = Debug|x64
-		{ECD78CC5-138F-4809-A764-2358C2AC2E82}.MinSizeRel|x64.Build.0 = Debug|x64
-		{ECD78CC5-138F-4809-A764-2358C2AC2E82}.Release|x64.ActiveCfg = Release|x64
-		{ECD78CC5-138F-4809-A764-2358C2AC2E82}.Release|x64.Build.0 = Release|x64
-		{ECD78CC5-138F-4809-A764-2358C2AC2E82}.RelWithDebInfo|x64.ActiveCfg = Release|x64
-		{ECD78CC5-138F-4809-A764-2358C2AC2E82}.RelWithDebInfo|x64.Build.0 = Release|x64
-		{3201359D-C071-4089-9EDE-B04D759F1508}.Debug|x64.ActiveCfg = Debug|x64
-		{3201359D-C071-4089-9EDE-B04D759F1508}.Debug|x64.Build.0 = Debug|x64
-		{3201359D-C071-4089-9EDE-B04D759F1508}.MinSizeRel|x64.ActiveCfg = Debug|x64
-		{3201359D-C071-4089-9EDE-B04D759F1508}.MinSizeRel|x64.Build.0 = Debug|x64
-		{3201359D-C071-4089-9EDE-B04D759F1508}.Release|x64.ActiveCfg = Release|x64
-		{3201359D-C071-4089-9EDE-B04D759F1508}.Release|x64.Build.0 = Release|x64
-		{3201359D-C071-4089-9EDE-B04D759F1508}.RelWithDebInfo|x64.ActiveCfg = Release|x64
-		{3201359D-C071-4089-9EDE-B04D759F1508}.RelWithDebInfo|x64.Build.0 = Release|x64
-		{2B8C5D01-FF78-4BFA-B332-5E430586DAF9}.Debug|x64.ActiveCfg = Debug|x64
-		{2B8C5D01-FF78-4BFA-B332-5E430586DAF9}.Debug|x64.Build.0 = Debug|x64
-		{2B8C5D01-FF78-4BFA-B332-5E430586DAF9}.MinSizeRel|x64.ActiveCfg = Debug|x64
-		{2B8C5D01-FF78-4BFA-B332-5E430586DAF9}.MinSizeRel|x64.Build.0 = Debug|x64
-		{2B8C5D01-FF78-4BFA-B332-5E430586DAF9}.Release|x64.ActiveCfg = Release|x64
-		{2B8C5D01-FF78-4BFA-B332-5E430586DAF9}.Release|x64.Build.0 = Release|x64
-		{2B8C5D01-FF78-4BFA-B332-5E430586DAF9}.RelWithDebInfo|x64.ActiveCfg = Release|x64
-		{2B8C5D01-FF78-4BFA-B332-5E430586DAF9}.RelWithDebInfo|x64.Build.0 = Release|x64
-		{4E4F491F-C688-43C4-88DA-AC92BC5FA1B8}.Debug|x64.ActiveCfg = Debug|Any CPU
-		{4E4F491F-C688-43C4-88DA-AC92BC5FA1B8}.Debug|x64.Build.0 = Debug|Any CPU
-		{4E4F491F-C688-43C4-88DA-AC92BC5FA1B8}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
-		{4E4F491F-C688-43C4-88DA-AC92BC5FA1B8}.MinSizeRel|x64.Build.0 = Debug|Any CPU
-		{4E4F491F-C688-43C4-88DA-AC92BC5FA1B8}.Release|x64.ActiveCfg = Release|Any CPU
-		{4E4F491F-C688-43C4-88DA-AC92BC5FA1B8}.Release|x64.Build.0 = Release|Any CPU
-		{4E4F491F-C688-43C4-88DA-AC92BC5FA1B8}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
-		{4E4F491F-C688-43C4-88DA-AC92BC5FA1B8}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
-		{2AC54ADC-3246-427E-9CA9-FEA2DBCB9432}.Debug|x64.ActiveCfg = Debug|Any CPU
-		{2AC54ADC-3246-427E-9CA9-FEA2DBCB9432}.Debug|x64.Build.0 = Debug|Any CPU
-		{2AC54ADC-3246-427E-9CA9-FEA2DBCB9432}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
-		{2AC54ADC-3246-427E-9CA9-FEA2DBCB9432}.MinSizeRel|x64.Build.0 = Debug|Any CPU
-		{2AC54ADC-3246-427E-9CA9-FEA2DBCB9432}.Release|x64.ActiveCfg = Release|Any CPU
-		{2AC54ADC-3246-427E-9CA9-FEA2DBCB9432}.Release|x64.Build.0 = Release|Any CPU
-		{2AC54ADC-3246-427E-9CA9-FEA2DBCB9432}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
-		{2AC54ADC-3246-427E-9CA9-FEA2DBCB9432}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
-		{4E96E5CC-7791-40B3-A8A1-C34FBB736B3B}.Debug|x64.ActiveCfg = Debug|Any CPU
-		{4E96E5CC-7791-40B3-A8A1-C34FBB736B3B}.Debug|x64.Build.0 = Debug|Any CPU
-		{4E96E5CC-7791-40B3-A8A1-C34FBB736B3B}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
-		{4E96E5CC-7791-40B3-A8A1-C34FBB736B3B}.MinSizeRel|x64.Build.0 = Debug|Any CPU
-		{4E96E5CC-7791-40B3-A8A1-C34FBB736B3B}.Release|x64.ActiveCfg = Release|Any CPU
-		{4E96E5CC-7791-40B3-A8A1-C34FBB736B3B}.Release|x64.Build.0 = Release|Any CPU
-		{4E96E5CC-7791-40B3-A8A1-C34FBB736B3B}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
-		{4E96E5CC-7791-40B3-A8A1-C34FBB736B3B}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
-		{51C40BA9-F226-4A35-939D-1ED6F5FF7299}.Debug|x64.ActiveCfg = Debug|Any CPU
-		{51C40BA9-F226-4A35-939D-1ED6F5FF7299}.Debug|x64.Build.0 = Debug|Any CPU
-		{51C40BA9-F226-4A35-939D-1ED6F5FF7299}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
-		{51C40BA9-F226-4A35-939D-1ED6F5FF7299}.MinSizeRel|x64.Build.0 = Debug|Any CPU
-		{51C40BA9-F226-4A35-939D-1ED6F5FF7299}.Release|x64.ActiveCfg = Release|Any CPU
-		{51C40BA9-F226-4A35-939D-1ED6F5FF7299}.Release|x64.Build.0 = Release|Any CPU
-		{51C40BA9-F226-4A35-939D-1ED6F5FF7299}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
-		{51C40BA9-F226-4A35-939D-1ED6F5FF7299}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
-	EndGlobalSection
-	GlobalSection(SolutionProperties) = preSolution
-		HideSolutionNode = FALSE
-	EndGlobalSection
-	GlobalSection(ExtensibilityGlobals) = postSolution
-		SolutionGuid = {422D1620-0166-4C26-9A81-CA8A2EB1AB58}
-	EndGlobalSection
-	GlobalSection(Performance) = preSolution
-		HasPerformanceSessions = true
-	EndGlobalSection
-EndGlobal
+Microsoft Visual Studio Solution File, Format Version 12.00
+# Visual Studio Version 17
+VisualStudioVersion = 17.3.32922.545
+MinimumVisualStudioVersion = 15.0.26124.0
+Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "spark_api", "src\client\spark_api\spark_api.csproj", "{DC9A8975-8211-4FC9-A9BA-C8B9D26AD481}"
+EndProject
+Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Spark", "src\client\spark\Spark.csproj", "{9A4D4D07-1AB6-4B2B-B111-A4F34D6C267E}"
+EndProject
+Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Spark.Console", "src\client\Spark.Console\Spark.Console.csproj", "{387566C6-C6F6-412C-A30C-9855BBCB4816}"
+EndProject
+Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "spark_core", "src\client\spark_core\spark_core.csproj", "{6E449EB6-20AC-4FA6-B8D9-10F4BB7E1FEE}"
+EndProject
+Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "spark_sdk", "src\client\spark_sdk\spark_sdk.csproj", "{6963C314-83C4-4667-B0D0-C2F7AE208C29}"
+EndProject
+Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "spark_test", "src\client\spark_test\spark_test.csproj", "{ECD78CC5-138F-4809-A764-2358C2AC2E82}"
+EndProject
+Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "spark_websocket", "src\client\spark_websocket\spark_websocket.csproj", "{3201359D-C071-4089-9EDE-B04D759F1508}"
+EndProject
+Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "spark_api_server", "src\client\spark_api_server\spark_api_server.csproj", "{2B8C5D01-FF78-4BFA-B332-5E430586DAF9}"
+EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "DataStreamCore", "src\DataStreamSuite\DataStreamCore\DataStreamCore.csproj", "{4E4F491F-C688-43C4-88DA-AC92BC5FA1B8}"
+EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "DataStream", "src\DataStreamSuite\DataStream\DataStream.csproj", "{2AC54ADC-3246-427E-9CA9-FEA2DBCB9432}"
+EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "SharedStreams", "src\DataStreamSuite\SharedStreams\SharedStreams.csproj", "{4E96E5CC-7791-40B3-A8A1-C34FBB736B3B}"
+EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "PubSub", "src\DataStreamSuite\PubSub\PubSub.csproj", "{51C40BA9-F226-4A35-939D-1ED6F5FF7299}"
+EndProject
+Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "src", "src", "{4564E677-343A-46CB-AF02-AFC4E38BF94B}"
+EndProject
+Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "client", "client", "{56A8A70A-DA03-4CED-A144-EC10F1379FD8}"
+EndProject
+Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "DataStreamSuite", "DataStreamSuite", "{915EF238-CA0D-4E73-AF50-88CB3F29C7AD}"
+EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Bts.Extensions.Hosting", "src\DataStreamSuite\Bts.Extensions.Hosting\Bts.Extensions.Hosting.csproj", "{4E0E51AA-7B59-4808-B7CC-9EAEDF397460}"
+EndProject
+Global
+	GlobalSection(SolutionConfigurationPlatforms) = preSolution
+		Debug|x64 = Debug|x64
+		MinSizeRel|x64 = MinSizeRel|x64
+		Release|x64 = Release|x64
+		RelWithDebInfo|x64 = RelWithDebInfo|x64
+	EndGlobalSection
+	GlobalSection(ProjectConfigurationPlatforms) = postSolution
+		{DC9A8975-8211-4FC9-A9BA-C8B9D26AD481}.Debug|x64.ActiveCfg = Debug|x64
+		{DC9A8975-8211-4FC9-A9BA-C8B9D26AD481}.Debug|x64.Build.0 = Debug|x64
+		{DC9A8975-8211-4FC9-A9BA-C8B9D26AD481}.MinSizeRel|x64.ActiveCfg = Release|x64
+		{DC9A8975-8211-4FC9-A9BA-C8B9D26AD481}.MinSizeRel|x64.Build.0 = Release|x64
+		{DC9A8975-8211-4FC9-A9BA-C8B9D26AD481}.Release|x64.ActiveCfg = Release|x64
+		{DC9A8975-8211-4FC9-A9BA-C8B9D26AD481}.Release|x64.Build.0 = Release|x64
+		{DC9A8975-8211-4FC9-A9BA-C8B9D26AD481}.RelWithDebInfo|x64.ActiveCfg = Release|x64
+		{DC9A8975-8211-4FC9-A9BA-C8B9D26AD481}.RelWithDebInfo|x64.Build.0 = Release|x64
+		{9A4D4D07-1AB6-4B2B-B111-A4F34D6C267E}.Debug|x64.ActiveCfg = Debug|x64
+		{9A4D4D07-1AB6-4B2B-B111-A4F34D6C267E}.Debug|x64.Build.0 = Debug|x64
+		{9A4D4D07-1AB6-4B2B-B111-A4F34D6C267E}.MinSizeRel|x64.ActiveCfg = Release|x64
+		{9A4D4D07-1AB6-4B2B-B111-A4F34D6C267E}.MinSizeRel|x64.Build.0 = Release|x64
+		{9A4D4D07-1AB6-4B2B-B111-A4F34D6C267E}.Release|x64.ActiveCfg = Release|x64
+		{9A4D4D07-1AB6-4B2B-B111-A4F34D6C267E}.Release|x64.Build.0 = Release|x64
+		{9A4D4D07-1AB6-4B2B-B111-A4F34D6C267E}.RelWithDebInfo|x64.ActiveCfg = Release|x64
+		{9A4D4D07-1AB6-4B2B-B111-A4F34D6C267E}.RelWithDebInfo|x64.Build.0 = Release|x64
+		{387566C6-C6F6-412C-A30C-9855BBCB4816}.Debug|x64.ActiveCfg = Debug|x64
+		{387566C6-C6F6-412C-A30C-9855BBCB4816}.Debug|x64.Build.0 = Debug|x64
+		{387566C6-C6F6-412C-A30C-9855BBCB4816}.MinSizeRel|x64.ActiveCfg = Debug|x64
+		{387566C6-C6F6-412C-A30C-9855BBCB4816}.MinSizeRel|x64.Build.0 = Debug|x64
+		{387566C6-C6F6-412C-A30C-9855BBCB4816}.Release|x64.ActiveCfg = Release|x64
+		{387566C6-C6F6-412C-A30C-9855BBCB4816}.Release|x64.Build.0 = Release|x64
+		{387566C6-C6F6-412C-A30C-9855BBCB4816}.RelWithDebInfo|x64.ActiveCfg = Release|x64
+		{387566C6-C6F6-412C-A30C-9855BBCB4816}.RelWithDebInfo|x64.Build.0 = Release|x64
+		{6E449EB6-20AC-4FA6-B8D9-10F4BB7E1FEE}.Debug|x64.ActiveCfg = Debug|x64
+		{6E449EB6-20AC-4FA6-B8D9-10F4BB7E1FEE}.Debug|x64.Build.0 = Debug|x64
+		{6E449EB6-20AC-4FA6-B8D9-10F4BB7E1FEE}.MinSizeRel|x64.ActiveCfg = Release|x64
+		{6E449EB6-20AC-4FA6-B8D9-10F4BB7E1FEE}.MinSizeRel|x64.Build.0 = Release|x64
+		{6E449EB6-20AC-4FA6-B8D9-10F4BB7E1FEE}.Release|x64.ActiveCfg = Release|x64
+		{6E449EB6-20AC-4FA6-B8D9-10F4BB7E1FEE}.Release|x64.Build.0 = Release|x64
+		{6E449EB6-20AC-4FA6-B8D9-10F4BB7E1FEE}.RelWithDebInfo|x64.ActiveCfg = Release|x64
+		{6E449EB6-20AC-4FA6-B8D9-10F4BB7E1FEE}.RelWithDebInfo|x64.Build.0 = Release|x64
+		{6963C314-83C4-4667-B0D0-C2F7AE208C29}.Debug|x64.ActiveCfg = Debug|x64
+		{6963C314-83C4-4667-B0D0-C2F7AE208C29}.Debug|x64.Build.0 = Debug|x64
+		{6963C314-83C4-4667-B0D0-C2F7AE208C29}.MinSizeRel|x64.ActiveCfg = Release|x64
+		{6963C314-83C4-4667-B0D0-C2F7AE208C29}.MinSizeRel|x64.Build.0 = Release|x64
+		{6963C314-83C4-4667-B0D0-C2F7AE208C29}.Release|x64.ActiveCfg = Release|x64
+		{6963C314-83C4-4667-B0D0-C2F7AE208C29}.Release|x64.Build.0 = Release|x64
+		{6963C314-83C4-4667-B0D0-C2F7AE208C29}.RelWithDebInfo|x64.ActiveCfg = Release|x64
+		{6963C314-83C4-4667-B0D0-C2F7AE208C29}.RelWithDebInfo|x64.Build.0 = Release|x64
+		{ECD78CC5-138F-4809-A764-2358C2AC2E82}.Debug|x64.ActiveCfg = Debug|x64
+		{ECD78CC5-138F-4809-A764-2358C2AC2E82}.Debug|x64.Build.0 = Debug|x64
+		{ECD78CC5-138F-4809-A764-2358C2AC2E82}.MinSizeRel|x64.ActiveCfg = Debug|x64
+		{ECD78CC5-138F-4809-A764-2358C2AC2E82}.MinSizeRel|x64.Build.0 = Debug|x64
+		{ECD78CC5-138F-4809-A764-2358C2AC2E82}.Release|x64.ActiveCfg = Release|x64
+		{ECD78CC5-138F-4809-A764-2358C2AC2E82}.Release|x64.Build.0 = Release|x64
+		{ECD78CC5-138F-4809-A764-2358C2AC2E82}.RelWithDebInfo|x64.ActiveCfg = Release|x64
+		{ECD78CC5-138F-4809-A764-2358C2AC2E82}.RelWithDebInfo|x64.Build.0 = Release|x64
+		{3201359D-C071-4089-9EDE-B04D759F1508}.Debug|x64.ActiveCfg = Debug|x64
+		{3201359D-C071-4089-9EDE-B04D759F1508}.Debug|x64.Build.0 = Debug|x64
+		{3201359D-C071-4089-9EDE-B04D759F1508}.MinSizeRel|x64.ActiveCfg = Debug|x64
+		{3201359D-C071-4089-9EDE-B04D759F1508}.MinSizeRel|x64.Build.0 = Debug|x64
+		{3201359D-C071-4089-9EDE-B04D759F1508}.Release|x64.ActiveCfg = Release|x64
+		{3201359D-C071-4089-9EDE-B04D759F1508}.Release|x64.Build.0 = Release|x64
+		{3201359D-C071-4089-9EDE-B04D759F1508}.RelWithDebInfo|x64.ActiveCfg = Release|x64
+		{3201359D-C071-4089-9EDE-B04D759F1508}.RelWithDebInfo|x64.Build.0 = Release|x64
+		{2B8C5D01-FF78-4BFA-B332-5E430586DAF9}.Debug|x64.ActiveCfg = Debug|x64
+		{2B8C5D01-FF78-4BFA-B332-5E430586DAF9}.Debug|x64.Build.0 = Debug|x64
+		{2B8C5D01-FF78-4BFA-B332-5E430586DAF9}.MinSizeRel|x64.ActiveCfg = Debug|x64
+		{2B8C5D01-FF78-4BFA-B332-5E430586DAF9}.MinSizeRel|x64.Build.0 = Debug|x64
+		{2B8C5D01-FF78-4BFA-B332-5E430586DAF9}.Release|x64.ActiveCfg = Release|x64
+		{2B8C5D01-FF78-4BFA-B332-5E430586DAF9}.Release|x64.Build.0 = Release|x64
+		{2B8C5D01-FF78-4BFA-B332-5E430586DAF9}.RelWithDebInfo|x64.ActiveCfg = Release|x64
+		{2B8C5D01-FF78-4BFA-B332-5E430586DAF9}.RelWithDebInfo|x64.Build.0 = Release|x64
+		{4E4F491F-C688-43C4-88DA-AC92BC5FA1B8}.Debug|x64.ActiveCfg = Debug|Any CPU
+		{4E4F491F-C688-43C4-88DA-AC92BC5FA1B8}.Debug|x64.Build.0 = Debug|Any CPU
+		{4E4F491F-C688-43C4-88DA-AC92BC5FA1B8}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
+		{4E4F491F-C688-43C4-88DA-AC92BC5FA1B8}.MinSizeRel|x64.Build.0 = Debug|Any CPU
+		{4E4F491F-C688-43C4-88DA-AC92BC5FA1B8}.Release|x64.ActiveCfg = Release|Any CPU
+		{4E4F491F-C688-43C4-88DA-AC92BC5FA1B8}.Release|x64.Build.0 = Release|Any CPU
+		{4E4F491F-C688-43C4-88DA-AC92BC5FA1B8}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
+		{4E4F491F-C688-43C4-88DA-AC92BC5FA1B8}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
+		{2AC54ADC-3246-427E-9CA9-FEA2DBCB9432}.Debug|x64.ActiveCfg = Debug|Any CPU
+		{2AC54ADC-3246-427E-9CA9-FEA2DBCB9432}.Debug|x64.Build.0 = Debug|Any CPU
+		{2AC54ADC-3246-427E-9CA9-FEA2DBCB9432}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
+		{2AC54ADC-3246-427E-9CA9-FEA2DBCB9432}.MinSizeRel|x64.Build.0 = Debug|Any CPU
+		{2AC54ADC-3246-427E-9CA9-FEA2DBCB9432}.Release|x64.ActiveCfg = Release|Any CPU
+		{2AC54ADC-3246-427E-9CA9-FEA2DBCB9432}.Release|x64.Build.0 = Release|Any CPU
+		{2AC54ADC-3246-427E-9CA9-FEA2DBCB9432}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
+		{2AC54ADC-3246-427E-9CA9-FEA2DBCB9432}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
+		{4E96E5CC-7791-40B3-A8A1-C34FBB736B3B}.Debug|x64.ActiveCfg = Debug|Any CPU
+		{4E96E5CC-7791-40B3-A8A1-C34FBB736B3B}.Debug|x64.Build.0 = Debug|Any CPU
+		{4E96E5CC-7791-40B3-A8A1-C34FBB736B3B}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
+		{4E96E5CC-7791-40B3-A8A1-C34FBB736B3B}.MinSizeRel|x64.Build.0 = Debug|Any CPU
+		{4E96E5CC-7791-40B3-A8A1-C34FBB736B3B}.Release|x64.ActiveCfg = Release|Any CPU
+		{4E96E5CC-7791-40B3-A8A1-C34FBB736B3B}.Release|x64.Build.0 = Release|Any CPU
+		{4E96E5CC-7791-40B3-A8A1-C34FBB736B3B}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
+		{4E96E5CC-7791-40B3-A8A1-C34FBB736B3B}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
+		{51C40BA9-F226-4A35-939D-1ED6F5FF7299}.Debug|x64.ActiveCfg = Debug|Any CPU
+		{51C40BA9-F226-4A35-939D-1ED6F5FF7299}.Debug|x64.Build.0 = Debug|Any CPU
+		{51C40BA9-F226-4A35-939D-1ED6F5FF7299}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
+		{51C40BA9-F226-4A35-939D-1ED6F5FF7299}.MinSizeRel|x64.Build.0 = Debug|Any CPU
+		{51C40BA9-F226-4A35-939D-1ED6F5FF7299}.Release|x64.ActiveCfg = Release|Any CPU
+		{51C40BA9-F226-4A35-939D-1ED6F5FF7299}.Release|x64.Build.0 = Release|Any CPU
+		{51C40BA9-F226-4A35-939D-1ED6F5FF7299}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
+		{51C40BA9-F226-4A35-939D-1ED6F5FF7299}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
+		{4E0E51AA-7B59-4808-B7CC-9EAEDF397460}.Debug|x64.ActiveCfg = Debug|Any CPU
+		{4E0E51AA-7B59-4808-B7CC-9EAEDF397460}.Debug|x64.Build.0 = Debug|Any CPU
+		{4E0E51AA-7B59-4808-B7CC-9EAEDF397460}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
+		{4E0E51AA-7B59-4808-B7CC-9EAEDF397460}.MinSizeRel|x64.Build.0 = Debug|Any CPU
+		{4E0E51AA-7B59-4808-B7CC-9EAEDF397460}.Release|x64.ActiveCfg = Release|Any CPU
+		{4E0E51AA-7B59-4808-B7CC-9EAEDF397460}.Release|x64.Build.0 = Release|Any CPU
+		{4E0E51AA-7B59-4808-B7CC-9EAEDF397460}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
+		{4E0E51AA-7B59-4808-B7CC-9EAEDF397460}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
+	EndGlobalSection
+	GlobalSection(SolutionProperties) = preSolution
+		HideSolutionNode = FALSE
+	EndGlobalSection
+	GlobalSection(ExtensibilityGlobals) = postSolution
+		SolutionGuid = {422D1620-0166-4C26-9A81-CA8A2EB1AB58}
+	EndGlobalSection
+	GlobalSection(Performance) = preSolution
+		HasPerformanceSessions = true
+	EndGlobalSection
+	GlobalSection(NestedProjects) = preSolution
+		{56A8A70A-DA03-4CED-A144-EC10F1379FD8} = {4564E677-343A-46CB-AF02-AFC4E38BF94B}
+		{915EF238-CA0D-4E73-AF50-88CB3F29C7AD} = {4564E677-343A-46CB-AF02-AFC4E38BF94B}
+		{4E0E51AA-7B59-4808-B7CC-9EAEDF397460} = {915EF238-CA0D-4E73-AF50-88CB3F29C7AD}
+	EndGlobalSection
+EndGlobal

src/DataStreamSuite/ArithmeticExpressions/ArithmeticExpressionGrammar.y

@@ -8,7 +8,6 @@
    public int Integer;
    public double Real;
    public string Identifier;
-   public DateTime DateTime; 
    public ImmutableList<IArithmeticExpression> Arguments;
    public ImmutableList<Assignment> Assignments;
    public IArithmeticExpression Argument;

src/DataStreamSuite/BlueRegistry/DoubleRegistryHandler.cs

@@ -1,4 +1,6 @@
 using System;
+using System.Runtime.Versioning;
+
 using Microsoft.Win32;
 
 namespace BCG
@@ -10,6 +12,7 @@ namespace BCG
          rk.SetValue (name, value.ToString ());
       }
 
+      [SupportedOSPlatform("windows")]
       public static double getValue (RegistryKey rk, string name)
       {
          object v = rk.GetValue (name);

src/DataStreamSuite/BooleanExpressions/BooleanExpression.cs

@@ -1,19 +1,11 @@
 namespace Bts.BooleanExpressions;
 
-using bts;
-
 class BooleanExpression : IBooleanExpression {
    readonly Func<IDictionary<string, object?>, bool> evaluate;
 
    public BooleanExpression(Func<IDictionary<string, object?>, bool> evaluate) =>
       this.evaluate = evaluate;
 
-   public bool Evaluate(IDictionary<string, object?> env) {
-      try {
-         return evaluate(env);
-      } catch (Exception ex) {
-         Log.assert(false, $"BooleanExpression.Evaluate failed: {ex}");
-         throw;
-      }
-   }
+   public bool Evaluate(IDictionary<string, object?> env) =>
+      evaluate(env);
 }

src/DataStreamSuite/BooleanExpressions/BooleanExpressionTokens.lex

@@ -47,11 +47,6 @@ IsOfType                 '/'
 
 %%
 
-{StringConstant} {
-   yylval.String = yytext.Substring(1, yytext.Length - 2).Replace(@"\'", "'");
-   return (int)Token.STRING_CONSTANT;
-} 
-
 {BooleanConstant} {
    yylval.Boolean = yytext.ToLower() switch {
       "true" => true,
@@ -62,6 +57,11 @@ IsOfType                 '/'
    return (int)Token.BOOLEAN_CONSTANT;
 }
 
+{StringConstant} {
+   yylval.String = yytext.Substring(1, yytext.Length - 2).Replace(@"\'", "'");
+   return (int)Token.STRING_CONSTANT;
+} 
+
 {NullConstant} {
    yylval.Null = DBNull.Value;
    return (int)Token.NULL_CONSTANT;

src/DataStreamSuite/Bts.Extensions.Hosting/Bts.Extensions.Hosting.csproj

@@ -0,0 +1,13 @@
+<Project Sdk="Microsoft.NET.Sdk">
+
+  <PropertyGroup>
+    <TargetFramework>net8.0</TargetFramework>
+    <ImplicitUsings>enable</ImplicitUsings>
+    <Nullable>enable</Nullable>
+  </PropertyGroup>
+
+  <ItemGroup>
+    <PackageReference Include="Microsoft.Extensions.Hosting" Version="10.0.0" />
+  </ItemGroup>
+
+</Project>

src/DataStreamSuite/Bts.Extensions.Hosting/HostBuilderExtensions.cs

@@ -0,0 +1,71 @@
+namespace Bts.Extensions.Hosting;
+
+using Microsoft.Extensions.Configuration;
+using Microsoft.Extensions.Hosting;
+using Microsoft.Extensions.Logging;
+
+using System.Diagnostics;
+
+public static class HostBuilderExtensions {
+   public static IHostBuilder CreateBtsBuilder(string[] args) {
+      // Obtain calling assembly name here, rather than in the lambda where it is actually needed,
+      // because want the name of the assembly that called this function, not the assembly that will eventually invoke
+      // the lambda...that will be the Microsoft.Extensions.Hosting assembly.
+      // Get the name of the assembly that called this function. That will act as the "appsettings"
+      // json file prefix
+      var exeName = GetCallingAssemblyName();
+      return Host.CreateDefaultBuilder(args)
+         .ConfigureAppConfiguration((context, configBuilder) =>
+            ConfigureApp(configBuilder, context.HostingEnvironment.EnvironmentName, exeName));
+   }
+
+   public static void ConfigureApp(IConfigurationBuilder configBuilder, string? environment = null, string? prefix = null) {
+      var baseFileName = prefix is null ? "appsettings" : $"{prefix}.appsettings";
+      AddJsonFile($"{baseFileName}.json");
+      if (environment is not null) {
+         AddJsonFile($"{baseFileName}.{environment}.json");
+      }
+
+      var cfg = configBuilder.Build().GetSection("config")?.Value;
+      if (!SetBasePath(cfg)) {
+         AddJsonFile(cfg);
+      }
+
+      bool SetBasePath(string? path) {
+         if (!Directory.Exists(path)) {
+            return false;
+         }
+         configBuilder.SetBasePath(path);
+         return true;
+      }
+
+      bool AddJsonFile(string? path) {
+         if (!File.Exists(path)) {
+            return false;
+         }
+         configBuilder.AddJsonFile(path);
+         return true;
+      }
+   }
+
+   public static void ConfigureLogging(ILoggingBuilder loggingBuilder, ILoggerProvider loggerProvider) =>
+      loggingBuilder.AddFilter("Microsoft", LogLevel.Warning)
+         .AddFilter("System", LogLevel.Warning)
+         .AddFilter("NToastNotify", LogLevel.Warning)
+         .AddProvider(loggerProvider)
+         .AddSimpleConsole(opts => {
+            opts.SingleLine = true;
+            opts.TimestampFormat = "yyyy-MM-dd HH:mm:ss.ffffff|";
+         })
+         .AddDebug();
+
+   public static string GetCallingAssemblyName() =>
+      new StackTrace()
+         .GetFrame(2)
+         ?.GetMethod()
+         ?.Module
+         ?.Assembly
+         ?.GetName()
+         ?.Name
+         ?? throw new InvalidOperationException("Failed to determine name of calling assembly (calling method is null)");
+}

src/DataStreamSuite/Bts.Net/Bts.Net.csproj

@@ -0,0 +1,9 @@
+<Project Sdk="Microsoft.NET.Sdk">
+
+  <PropertyGroup>
+    <TargetFramework>net8.0</TargetFramework>
+    <ImplicitUsings>enable</ImplicitUsings>
+    <Nullable>enable</Nullable>
+  </PropertyGroup>
+
+</Project>

src/DataStreamSuite/Bts.Net/HostComparer.cs

@@ -0,0 +1,28 @@
+namespace Bts.Net;
+
+using System.Linq;
+using System.Net;
+
+/// <summary>
+/// Provides facilities to compare hosts for equality.
+/// <para>
+/// This class doesn't implement IEqualityComparer because a consistent hash code cannot be
+/// guarenteed for equivalent host strings. The equals impl relies on Dns.GetHostAddresses, which
+/// is non-determinstic (implementations that depends on mutable properties face a similar problem).
+/// </para>
+/// </summary>
+public class HostComparer {
+   public static bool Equals(string? x, string? y) {
+      if (x is null) {
+         return y is null;
+      }
+      if (y is null) {
+         return false;
+      }
+      return Dns.GetHostAddresses(x).Intersect(
+         Dns.GetHostAddresses(y), IpAddressComparer.Instance).Any();
+   }
+
+   public static bool Equals(IPHostEntry x, IPHostEntry y) =>
+      x.AddressList.Intersect(y.AddressList, IpAddressComparer.Instance).Any();
+}

src/DataStreamSuite/Bts.Net/IpAddressComparer.cs

@@ -0,0 +1,30 @@
+namespace Bts.Net;
+
+using System.Diagnostics.CodeAnalysis;
+using System.Net;
+
+/// <summary>
+/// Compares ip addresses. Unlike the default comparer, v4 and equivalent v6 addresses are
+/// considered equal.
+/// </summary>
+public class IpAddressComparer : IEqualityComparer<IPAddress> {
+   public static IEqualityComparer<IPAddress> Instance { get; } = new IpAddressComparer();
+
+   public bool Equals(IPAddress? x, IPAddress? y) {
+      if (x is null) {
+         return y is null;
+      }
+      if (y is null) {
+         return false;
+      }
+      return x.Equals(y) ||
+         (x.IsIPv4MappedToIPv6 && x.MapToIPv4().Equals(y)) ||
+         (y.IsIPv4MappedToIPv6 && y.MapToIPv4().Equals(x));
+   }
+
+   public int GetHashCode([DisallowNull] IPAddress address) =>
+      address.IsIPv4MappedToIPv6
+         // Return consistent hash code for v4 and equivalent v6
+         ? address.MapToIPv4().GetHashCode()
+         : address.GetHashCode();
+}

src/DataStreamSuite/Bts.Net/IpHostEntryJsonConverter.cs

@@ -0,0 +1,33 @@
+namespace Bts.Net;
+
+using System;
+using System.Net;
+using System.Text.Json;
+using System.Text.Json.Serialization;
+
+public class IpHostEntryJsonConverter : JsonConverter<IPHostEntry> {
+   public override IPHostEntry? Read(ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options) {
+      throw new NotImplementedException();
+   }
+
+   public override void Write(Utf8JsonWriter writer, IPHostEntry value, JsonSerializerOptions options) {
+      writer.WriteStartObject();
+      // HostName
+      writer.WriteString(nameof(value.HostName), value.HostName);
+      // Aliases
+      writer.WritePropertyName("Aliases");
+      writer.WriteStartArray();
+      foreach (var alias in value.Aliases) {
+         writer.WriteStringValue(alias);
+      }
+      writer.WriteEndArray();
+      // Addresses
+      writer.WritePropertyName("Addresses");
+      writer.WriteStartArray();
+      foreach (var address in value.AddressList) {
+         writer.WriteStringValue(address.ToString());
+      }
+      writer.WriteEndArray();
+      writer.WriteEndObject();
+   }
+}

src/DataStreamSuite/Bts.Net/IpHostEntryJsonSerializer.cs

@@ -0,0 +1,39 @@
+namespace Bts.Net;
+
+using System;
+using System.IO;
+using System.Net;
+using System.Text;
+using System.Text.Json;
+
+/// <summary>
+/// This class seriailzes IpHostEntry to json (string). A common buffer is used to reduce memory
+/// allocations on repeated calls to <see cref="Serialize"/>. For that reason, this class is NOT
+/// thread safe.
+/// </summary>
+public class IpHostEntryJsonSerializer {
+   readonly MemoryStream buffer = new();
+   readonly IpHostEntryJsonConverter converter;
+
+   public IpHostEntryJsonSerializer(IpHostEntryJsonConverter? converter = null) =>
+      this.converter = converter ?? new() { };
+
+   public string? Serialize(IPHostEntry dp, JsonSerializerOptions? opts = null) {
+      opts ??= JsonSerializerOptions.Default;
+      buffer.Position = 0;
+      buffer.SetLength(0);
+      using (var jsonWriter = new Utf8JsonWriter(buffer)) {
+         converter.Write(jsonWriter, dp, opts);
+      }
+      var bytes = buffer.GetBuffer();
+      var span = new ArraySegment<byte>(bytes, 0, (int)buffer.Length);
+      return Encoding.UTF8.GetString(span);
+   }
+
+   public void Serialize(IPHostEntry dp, Utf8JsonWriter jsonWriter, JsonSerializerOptions? opts = null) {
+      opts ??= JsonSerializerOptions.Default;
+      buffer.Position = 0;
+      buffer.SetLength(0);
+      converter.Write(jsonWriter, dp, opts);
+   }
+}

src/DataStreamSuite/Bts.Net/IpUtils.cs

@@ -0,0 +1,94 @@
+namespace Bts.Net;
+
+using System.Diagnostics.CodeAnalysis;
+using System.Net;
+using System.Text.RegularExpressions;
+
+public static partial class IpUtils {
+   static readonly Regex ipv4Regex = new(string.Join('.', @"(((0|1)\d)|(2\d)|\d)?\d"));
+
+   // public static void ParseUri(string str, out string? scheme, out string? host, out string? port, out string? path, out string? queryParams) {
+   //    if (Uri.TryCreate(str, UriKind.Absolute, out var uri)) {
+   //    }
+   // }
+
+   // public static bool AreEndpointsEqual(Uri? x, Uri? y) {
+   //    if (x is null) {
+   //       return y is null;
+   //    }
+   //    if (y is null) {
+   //       return false;
+   //    }
+   //    if (x.IsAbsoluteUri != y.IsAbsoluteUri) {
+   //       return false;
+   //    }
+   //    if (x.IsAbsoluteUri) {
+   //       if (x.Scheme != y.Scheme) {
+   //          return false;
+   //       }
+   //    }
+   //    string? xHost = null;
+   // }
+
+   // static string ResolveHostIpv4(Uri uri) {
+   //    // switch (uri.HostNameType) {
+   //    //    case UriHostNameType.IPv4:
+   //    //       return uri.Host;
+   //    //    case UriHostNameType.Dns:
+   //    //       break;
+   //    //    case UriHostNameType.IPv6:
+   //    //       var addresses = Dns.GetHostAddresses(uri.Host);
+   //    //       foreach (var address in addresses) {
+   //    //          if (address.IsIPv4MappedToIPv6) {
+   //    //             return address.MapToIPv4().Address;
+   //    //          }
+   //    //       }
+   //    //       break;
+   //    // }
+   //    var hostAddresses = Dns.GetHostAddresses(uri.Host);
+   //    foreach (IPAddress address in hostAddresses) {
+   //       if (address.Equals(remoteEndPoint.Address) ||
+   //           (address.IsIPv4MappedToIPv6 && address.MapToIPv4().Equals(remoteEndPoint.Address)) ||
+   //           (remoteEndPoint.Address.IsIPv4MappedToIPv6 && remoteEndPoint.Address.MapToIPv4().Equals(address))) {
+   //          isMatch = true;
+   //          break;
+   //       }
+   //    }
+   // }
+
+   // public static bool AreEndpointsEqual(string? x, string? y) {
+   //    if (x == y) {
+   //       return true;
+   //    }
+   //    if (x is null) {
+   //       return y is null;
+   //    }
+   //    if (y is null) {
+   //       return false;
+   //    }
+   //    var xUri = new Uri(x);
+   //    var yUri = new Uri(y);
+   //    // return xUri.Equals(yUri);
+   // }
+
+   public static bool IsIpv4(string host) {
+      host = removeSchemaFromUrl(host);
+      host = removePortFromUrl(host);
+      return ipv4Regex.IsMatch(host);
+   }
+
+   static string removeSchemaFromUrl(string url) {
+      var i = url.IndexOf("://");
+      return i < 0 ? url : url.Substring(i + 3);
+   }
+
+   static string removePortFromUrl(string url) {
+      var match = MyRegex().Match(url);
+      return match.Success
+         ? string.Concat(url.AsSpan(0, match.Index), url.AsSpan(match.Index + match.Length))
+         : url;
+   }
+
+   [GeneratedRegex(":\\d{1,4}")]
+   private static partial Regex MyRegex();
+}

src/DataStreamSuite/ColumnMapExpressions/ColumnMapExpressions.csproj

@@ -7,14 +7,11 @@
   </PropertyGroup>
 
   <ItemGroup>
-    <PackageReference Include="Google.Protobuf" Version="3.24.3" />
     <PackageReference Include="YaccLexTools" Version="1.2.2" />
   </ItemGroup>
 
   <ItemGroup>
-    <ProjectReference Include="..\DataStreamCore\DataStreamCore.csproj" />
     <ProjectReference Include="..\ExpressionCore\ExpressionCore.csproj" />
-    <ProjectReference Include="..\..\client\spark_core\spark_core.csproj" />
   </ItemGroup>
 
   <ItemGroup Label="ColumnMapExpression">

src/DataStreamSuite/DataStream/BlueOrb/BlueAddIndexColumnTable.cs

@@ -0,0 +1,118 @@
+namespace Bts.DataStream;
+
+using BCG;
+
+using bts.collections;
+using bts.utils;
+
+using Bts.DataStreamCore;
+
+using Spark.Core.Collections;
+
+using System;
+using System.Collections;
+using System.Collections.Generic;
+using System.Linq;
+
+public class BlueAddIndexColumnTable : IBlueRowCollection {
+   readonly IBlueTableSchema schema;
+   readonly IList<object?>[] columnValues;
+   readonly int count;
+
+   BlueAddIndexColumnTable(IBlueTableSchema schema, IList<object?>[] columnValues, int? count = null) {
+      this.schema = schema;
+      this.columnValues = columnValues;
+      this.count = count ?? columnValues[0].Count;
+   }
+
+   IBlueTableSchema IBlueRowCollection.schema => schema;
+
+   public int Count => count;
+
+   public IBlueRow this[int id] =>
+      id > 0 && id <= count
+         ? GetRow(id - 1)
+         : throw new KeyNotFoundException();
+
+   public static BlueAddIndexColumnTable Create(string tableName, ColumnInfo[] columnInfos, IList<object?>[] columnValues) {
+      if (columnInfos.Any(c => c.colName == "id")) {
+         throw new ArgumentException("'id' column already exists", nameof(columnInfos));
+      }
+      if (columnInfos.Length != columnValues.Length) {
+         throw new ArgumentException("columnValues length differs from columnInfos length", nameof(columnValues));
+      }
+      int count = columnValues.FirstOrDefault()?.Count ?? 0;
+      var origCols = columnInfos;
+      var origColValues = columnValues;
+      columnInfos = new ColumnInfo[origCols.Length + 1];
+      columnInfos[0] = new ColumnInfo("id", 0, typeof(int));
+      columnValues = new IList<object?>[origCols.Length + 1];
+      columnValues[0] = new StepList<int>(1, 1, count).WrapSelector(id => (object?)id, null);
+      for (int i = 1; i < columnInfos.Length; i++) {
+         columnInfos[i] = new ColumnInfo(origCols[i - 1].colName, i, origCols[i - 1].colType);
+         columnValues[i] = origColValues[i - 1];
+      }
+      var schema = new BlueTableSchema(columnInfos, tableName);
+      return new(schema, columnValues, count);
+   }
+
+   public static BlueAddIndexColumnTable Create(IBlueTableSchema schema, IList<object?>[] columnValues) {
+      int count = columnValues.FirstOrDefault()?.Count ?? 0;
+      int? idColIdx = schema.TryResolveIndices("id");
+      var idColValues = new StepList<int>(1, 1, count).WrapSelector(id => (object?)id, null);
+      var columnInfos = schema.columnInfo;
+      if (idColIdx.HasValue) {
+         // Replace existing id column. columnValues may or may not contain the old ids (if its
+         // length is one less column than that of columnInfos, it doesn't contain them).
+         switch (columnValues.Length - schema.columnInfo.Length) {
+            case 0:
+               columnValues[idColIdx.Value] = idColValues;
+               break;
+            case -1:
+               columnValues = ArrayHelper.InsertExact(columnValues, idColIdx.Value, true, idColValues);
+               break;
+            default:
+               throw new ArgumentException("ColumnInfos length disagrees with columnValues length");
+         }
+      } else {
+         if (columnValues.Length != columnInfos.Length) {
+            throw new ArgumentException("ColumnInfos length disagrees with columnValues length");
+         }
+         var origColInfos = columnInfos;
+         columnInfos = new ColumnInfo[columnInfos.Length + 1];
+         columnInfos[0] = new("id", 0, typeof(int));
+         for (int i = 1; i < columnInfos.Length; i++) {
+            var c = origColInfos[i - 1];
+            columnInfos[i] = new(c.colName, c.colNo + 1, c.colType);
+         }
+         columnValues = ArrayHelper.InsertExact(columnValues, 0, true, idColValues);
+      }
+      return new(new BlueTableSchema(schema.tableName, columnInfos), columnValues);
+   }
+
+   public bool containsID(int id) =>
+      id > 0 && id <= count;
+
+   public IList<object?> GetColumn(int index) =>
+      columnValues[index];
+
+   public IBlueRow GetRow(int index) =>
+      new BlueRowColumnWrapper(schema, index, columnValues);
+
+   public int IndexOf(IBlueRow item) {
+      int id = item.Id();
+      return id > 0 && id <= count && BlueRowEqualityComparer.Equals(GetRow(id - 1), item)
+         ? id - 1
+         : -1;
+   }
+
+   public IEnumerator<IBlueRow> GetEnumerator() {
+      int count = Count;
+      for (int i = 0; i < count; i++) {
+         yield return GetRow(i);
+      }
+   }
+
+   IEnumerator IEnumerable.GetEnumerator() =>
+      GetEnumerator();
+}

src/DataStreamSuite/DataStream/BlueOrb/BlueIndexedColumnTable.cs

@@ -0,0 +1,56 @@
+namespace Bts.DataStream;
+
+using BCG;
+
+using Bts.DataStreamCore;
+
+using mbts.utils;
+
+using System.Collections;
+using System.Collections.Generic;
+using System.Linq;
+
+public class BlueIndexedColumnTable : IBlueRowCollection {
+   readonly IBlueTableSchema schema;
+   readonly IList<object?>[] columnValues;
+   readonly Dictionary<int, int> rowIdToIndex;
+
+   public BlueIndexedColumnTable(IBlueTableSchema schema, IList<object?>[] columnValues) {
+      this.schema = schema;
+      this.columnValues = columnValues;
+      int idColumnIndex = schema.ResolveIndices("id");
+      rowIdToIndex = columnValues[idColumnIndex]
+         .Select((idObj, idx) => new KeyValuePair<int, int>(ExceptionAssert.Argument.Is<int>(idObj), idx))
+         .ToDictionary();
+   }
+
+   public IBlueRow this[int id] =>
+      rowIdToIndex.TryGetValue(id, out var idx)
+         ? GetRow(idx)
+         : throw new KeyNotFoundException();
+
+   IBlueTableSchema IBlueRowCollection.schema => schema;
+
+   public int Count => rowIdToIndex.Count;
+
+   public bool containsID(int id) =>
+      rowIdToIndex.ContainsKey(id);
+
+   public IEnumerator<IBlueRow> GetEnumerator() {
+      int count = Count;
+      for (int i = 0; i < count; i++) {
+         yield return GetRow(i);
+      }
+   }
+
+   public IBlueRow GetRow(int index) =>
+      new BlueRowColumnWrapper(schema, index, columnValues);
+
+   public int IndexOf(IBlueRow item) =>
+      rowIdToIndex.TryGetValue(item.Id(), out var idx) && BlueRowEqualityComparer.Equals(GetRow(idx), item)
+         ? idx
+         : -1;
+
+   IEnumerator IEnumerable.GetEnumerator() =>
+      GetEnumerator();
+}

src/DataStreamSuite/DataStream/BlueOrb/BlueRowColumnWrapper.cs

@@ -0,0 +1,21 @@
+namespace Bts.DataStream;
+
+using BCG;
+
+using System.Collections.Generic;
+
+public class BlueRowColumnWrapper : IBlueRow {
+   readonly int index;
+   readonly IList<object?>[] columns;
+
+   public BlueRowColumnWrapper(IBlueTableSchema schema, int index, IList<object?>[] columns) {
+      this.schema = schema;
+      this.index = index;
+      this.columns = columns;
+   }
+
+   public object? this[int index] =>
+      columns[index][this.index];
+
+   public IBlueTableSchema schema { get; }
+}

src/DataStreamSuite/DataStream/BlueOrb/CountStream.cs

@@ -14,9 +14,6 @@ namespace BCG {
          myVS = vs;
       }
 
-      public CountStream(string tableName)
-         : this(new TableStream(tableName)) { }
-
       protected override void myOnFirstSubscriber() {
          myVS.onChange += new ChangeDelegate(myHandleChanges);
       }

src/DataStreamSuite/DataStream/BlueOrb/MillisecondStream.cs

@@ -43,7 +43,7 @@ namespace bts.BlueOrb {
             _last_fire_time = DateTime.Now;
             IBlueRow nr = new BlueRow(_schema, _id++, _last_fire_time,
             _last_fire_time.ToString("yyyy-MM-dd HH:mm:ss.fff"));
-            _table.add_row(nr);
+            _table = _table.Add(nr);
             var dp = BlueDataUtility.build_dp_inc(_table, myLastValue as DataPacket);
             myFireEvent(dp);
          }

src/DataStreamSuite/DataStream/BlueOrb/Parquet/IParquetColumnReader.cs

@@ -0,0 +1,29 @@
+namespace Bts.DataStream.Parquet;
+
+using bts.collections;
+
+using System;
+using System.Collections.Generic;
+
+public interface IParquetColumnReader : IDisposable {
+   IList<object?> Read();
+   IList<object?> Read(int? start, int? stop);
+}
+
+public interface IParquetColumnReader<TLogical, TPhysical> : IParquetColumnReader {
+   new IList<TLogical?> Read();
+   new IList<TLogical?> Read(int? start, int? stop);
+
+   IList<object?> IParquetColumnReader.Read() =>
+      Read().WrapSelector(v => (object?)v, null);
+
+   IList<object?> IParquetColumnReader.Read(int? start, int? stop) =>
+      Read(start, stop).WrapSelector(v => (object?)v, null);
+}
+
+public interface IParquetColumnReader<T> : IParquetColumnReader<T, T> {
+   new IList<T?> Read();
+
+   IList<object?> IParquetColumnReader.Read() =>
+      Read().WrapSelector(v => (object?)v, null);
+}

src/DataStreamSuite/DataStream/BlueOrb/Parquet/ParquetAdapter.cs

@@ -0,0 +1,507 @@
+namespace Bts.DataStream.Parquet;
+
+using bts;
+using bts.collections;
+
+using mbts.utils;
+
+using ParquetSharp;
+
+using Spark.Core.Collections;
+
+using System;
+using System.Collections.Generic;
+
+public static class ParquetAdapter {
+   public static IParquetColumnReader GetColumnReader(this RowGroupReader rowGroupReader, int index, Type? colType = null) {
+      int rowCount = checked((int)rowGroupReader.MetaData.NumRows);
+      var colReader = rowGroupReader.Column(index);
+      return getColumnReader(colReader, rowCount, colType: colType);
+   }
+
+   public static IList<object?>[] ReadColumns(this ParquetFileReader fileReader) {
+      var schema = fileReader.FileMetaData.Schema.ToBlueSchema("parquetReader");
+      var colInfos = schema.columnInfo;
+      var colTypes = Array.ConvertAll(colInfos, c => c.colType);
+      var colValues = new IList<object?>[fileReader.FileMetaData.NumColumns];
+      int rowCount = checked((int)fileReader.FileMetaData.NumRows);
+      for (int i = 0; i < colValues.Length; i++) {
+         colValues[i] = new List<object?>(rowCount);
+      }
+      int rowGroupCount = fileReader.FileMetaData.NumRowGroups;
+      for (int rowGroup = 0; rowGroup < rowGroupCount; ++rowGroup) {
+         using var rowGroupReader = fileReader.RowGroup(rowGroup);
+         rowGroupReader.ReadColumnsTo(ref colValues, colTypes);
+      }
+      return colValues;
+   }
+
+   public static IList<object?>[] ReadColumns(this ParquetFileReader fileReader, int? start, int? stop) {
+      if (!start.HasValue && !stop.HasValue) {
+         return ReadColumns(fileReader);
+      }
+      var schema = fileReader.FileMetaData.ToBlueSchema("parquetReader");
+      var colInfos = schema.columnInfo;
+      var colTypes = Array.ConvertAll(colInfos, c => c.colType);
+      var colValues = new IList<object?>[fileReader.FileMetaData.NumColumns];
+      int rowCount = checked((int)fileReader.FileMetaData.NumRows);
+      start ??= 0;
+      stop ??= rowCount;
+      var sliceLength = stop.Value - start.Value;
+      for (int i = 0; i < colValues.Length; i++) {
+         colValues[i] = new List<object?>(sliceLength);
+      }
+      int numRowGroups = fileReader.FileMetaData.NumRowGroups;
+      int? numRowsPerGroup = null;
+      int rowIdx = 0;
+      for (int rowGroup = 0; rowGroup < numRowGroups; ++rowGroup) {
+         if (stop.Value <= rowIdx) {
+            break;
+         }
+         using var rowGroupReader = fileReader.RowGroup(rowGroup);
+         int numRowsInGroup = checked((int)rowGroupReader.MetaData.NumRows);
+         numRowsPerGroup ??= numRowsInGroup;
+         if (rowGroup < numRowGroups - 1) {
+            Log.assert(numRowsInGroup == numRowsPerGroup.Value);
+         } else {
+            Log.assert(numRowsInGroup <= numRowsPerGroup.Value);
+         }
+         int startIdx = start.Value - rowIdx;
+         int stopIdx = stop.Value - rowIdx;
+         rowIdx += numRowsInGroup;
+         if (startIdx >= numRowsInGroup) {
+            continue;
+         }
+         startIdx = Math.Max(0, startIdx);
+         stopIdx = Math.Min(stopIdx, numRowsInGroup);
+         int n = rowGroupReader.ReadColumnsTo(ref colValues, colTypes, startIdx, stopIdx);
+         Log.assert(n == (stopIdx - startIdx));
+      }
+      return colValues;
+   }
+
+   public static int ReadColumnsTo(this RowGroupReader rowGroupReader, ref IList<object?>[] columnValues, Type[] colTypes) {
+      int rowCount = checked((int)rowGroupReader.MetaData.NumRows);
+      for (int colIdx = rowGroupReader.MetaData.NumColumns - 1; colIdx > -1; colIdx--) {
+         using var colReader = rowGroupReader.GetColumnReader(colIdx, colTypes[colIdx]);
+         columnValues[colIdx].AddRange(colReader.Read());
+      }
+      return rowCount;
+   }
+
+   public static int ReadColumnsTo(this RowGroupReader rowGroupReader, ref IList<object?>[] columnValues, Type[] colTypes, int? start, int? stop) {
+      if (!start.HasValue && !stop.HasValue) {
+         return ReadColumnsTo(rowGroupReader, ref columnValues, colTypes);
+      }
+      int numRows = checked((int)rowGroupReader.MetaData.NumRows);
+      if (start.HasValue && start.Value >= numRows) {
+         return 0;
+      }
+      int? readRowCount = null;
+      for (int colIdx = rowGroupReader.MetaData.NumColumns - 1; colIdx > -1; colIdx--) {
+         using var colReader = rowGroupReader.GetColumnReader(colIdx, colTypes[colIdx]);
+         var colVals = colReader.Read(start, stop);
+         readRowCount ??= colVals.Count;
+         Log.assert(readRowCount == colVals.Count);
+         columnValues[colIdx].AddRange(colVals);
+      }
+      return readRowCount ?? throw new ArgumentException("Row group has no columns", nameof(rowGroupReader));
+   }
+
+   static IParquetColumnReader getColumnReader(ColumnReader reader, int bufferSize = 4096, Type? colType = null) {
+      var desc = reader.ColumnDescriptor;
+      var logicalType = desc.LogicalType;
+      return logicalType.Type switch {
+         LogicalTypeEnum.Int => IntReader(),
+         LogicalTypeEnum.String => new ParquetColumnReader<string?>(reader, bufferSize),
+         LogicalTypeEnum.Date => new ParquetColumnReader<DateTime?>(reader, bufferSize),
+         LogicalTypeEnum.Decimal => DecimalReader(),
+         LogicalTypeEnum.Timestamp => TimestampReader(),
+         LogicalTypeEnum.None => DefaultReader(),
+         _ => throw new NotImplementedException(),
+      };
+
+      IParquetColumnReader DefaultReader() {
+         return desc.PhysicalType switch {
+            PhysicalType.Boolean => new ParquetColumnReader<bool?>(reader, bufferSize),
+            PhysicalType.Int32 => new ParquetColumnReader<int?>(reader, bufferSize),
+            PhysicalType.Int64 => new ParquetColumnReader<long?>(reader, bufferSize),
+            PhysicalType.Float => new ParquetColumnReader<float?>(reader, bufferSize),
+            PhysicalType.Double => new ParquetColumnReader<double?>(reader, bufferSize),
+            _ => throw new NotImplementedException(),
+         };
+      }
+
+      IParquetColumnReader TimestampReader() {
+         var tlt = ExceptionAssert.of_type<TimestampLogicalType>(logicalType);
+         return new ParquetColumnReader<DateTimeNanos?, DateTime?>(reader, dtn => dtn?.DateTime, bufferSize);
+      }
+
+      IParquetColumnReader IntReader() {
+         var ilt = ExceptionAssert.of_type<IntLogicalType>(logicalType);
+         if (!ilt.IsSigned) {
+            if (colType is null) {
+               return ilt.BitWidth switch {
+                  8 => new ParquetColumnReader<byte?>(reader, bufferSize),
+                  16 => new ParquetColumnReader<ushort?>(reader, bufferSize),
+                  32 => new ParquetColumnReader<uint?>(reader, bufferSize),
+                  64 => new ParquetColumnReader<ulong?>(reader, bufferSize),
+                  _ => throw new NotImplementedException(),
+               };
+            } else if (colType == typeof(byte)) {
+               return ilt.BitWidth switch {
+                  8 => new ParquetColumnReader<byte?>(reader, bufferSize),
+                  16 => new ParquetColumnReader<ushort?, byte?>(reader, v => v.HasValue ? checked((byte)v.Value) : null, bufferSize),
+                  32 => new ParquetColumnReader<uint?, ushort?>(reader, v => v.HasValue ? checked((byte)v.Value) : null, bufferSize),
+                  64 => new ParquetColumnReader<ulong?, ushort?>(reader, v => v.HasValue ? checked((byte)v.Value) : null, bufferSize),
+                  _ => throw new NotImplementedException(),
+               };
+            } else if (colType == typeof(ushort)) {
+               return ilt.BitWidth switch {
+                  8 => new ParquetColumnReader<byte?, ushort?>(reader, v => v.HasValue ? v.Value : null, bufferSize),
+                  16 => new ParquetColumnReader<ushort?>(reader, bufferSize),
+                  32 => new ParquetColumnReader<uint?, ushort?>(reader, v => v.HasValue ? checked((ushort)v.Value) : null, bufferSize),
+                  64 => new ParquetColumnReader<ulong?, ushort?>(reader, v => v.HasValue ? checked((ushort)v.Value) : null, bufferSize),
+                  _ => throw new NotImplementedException(),
+               };
+            } else if (colType == typeof(uint)) {
+               return ilt.BitWidth switch {
+                  8 => new ParquetColumnReader<byte?, uint?>(reader, v => v.HasValue ? v.Value : null, bufferSize),
+                  16 => new ParquetColumnReader<ushort?, uint?>(reader, v => v.HasValue ? v.Value : null, bufferSize),
+                  32 => new ParquetColumnReader<uint?>(reader, bufferSize),
+                  64 => new ParquetColumnReader<ulong?, uint?>(reader, v => v.HasValue ? checked((uint)v.Value) : null, bufferSize),
+                  _ => throw new NotImplementedException(),
+               };
+            } else if (colType == typeof(ulong)) {
+               return ilt.BitWidth switch {
+                  8 => new ParquetColumnReader<byte?, ulong?>(reader, v => v.HasValue ? v.Value : null, bufferSize),
+                  16 => new ParquetColumnReader<ushort?, ulong?>(reader, v => v.HasValue ? v.Value : null, bufferSize),
+                  32 => new ParquetColumnReader<uint?, ulong?>(reader, v => v.HasValue ? v.Value : null, bufferSize),
+                  64 => new ParquetColumnReader<ulong?>(reader, bufferSize),
+                  _ => throw new NotImplementedException(),
+               };
+            }
+            throw new NotImplementedException($"Expected signed int column type, but got {colType}");
+         } else {
+            if (colType is null) {
+               return ilt.BitWidth switch {
+                  8 => new ParquetColumnReader<sbyte?>(reader, bufferSize),
+                  16 => new ParquetColumnReader<short?>(reader, bufferSize),
+                  32 => new ParquetColumnReader<int?>(reader, bufferSize),
+                  64 => new ParquetColumnReader<long?>(reader, bufferSize),
+                  _ => throw new NotImplementedException(),
+               };
+            } else if (colType == typeof(sbyte)) {
+               return ilt.BitWidth switch {
+                  8 => new ParquetColumnReader<sbyte?>(reader, bufferSize),
+                  16 => new ParquetColumnReader<short?, sbyte?>(reader, v => v.HasValue ? checked((sbyte)v.Value) : null, bufferSize),
+                  32 => new ParquetColumnReader<int?, sbyte?>(reader, v => v.HasValue ? checked((sbyte)v.Value) : null, bufferSize),
+                  64 => new ParquetColumnReader<long?, sbyte?>(reader, v => v.HasValue ? checked((sbyte)v.Value) : null, bufferSize),
+                  _ => throw new NotImplementedException(),
+               };
+            } else if (colType == typeof(short)) {
+               return ilt.BitWidth switch {
+                  8 => new ParquetColumnReader<char?, short?>(reader, v => v.HasValue ? checked((short)v.Value) : null, bufferSize),
+                  16 => new ParquetColumnReader<short?>(reader, bufferSize),
+                  32 => new ParquetColumnReader<int?, short?>(reader, v => v.HasValue ? checked((short)v.Value) : null, bufferSize),
+                  64 => new ParquetColumnReader<long?, short?>(reader, v => v.HasValue ? checked((short)v.Value) : null, bufferSize),
+                  _ => throw new NotImplementedException(),
+               };
+            } else if (colType == typeof(int)) {
+               return ilt.BitWidth switch {
+                  8 => new ParquetColumnReader<char?, int?>(reader, v => v.HasValue ? v.Value : null, bufferSize),
+                  16 => new ParquetColumnReader<short?, int?>(reader, v => v.HasValue ? v.Value : null, bufferSize),
+                  32 => new ParquetColumnReader<int?>(reader, bufferSize),
+                  64 => new ParquetColumnReader<long?, int?>(reader, v => v.HasValue ? checked((int)v.Value) : null, bufferSize),
+                  _ => throw new NotImplementedException(),
+               };
+            } else if (colType == typeof(long)) {
+               return ilt.BitWidth switch {
+                  8 => new ParquetColumnReader<char?, long?>(reader, v => v.HasValue ? v.Value : null, bufferSize),
+                  16 => new ParquetColumnReader<short?, long?>(reader, v => v.HasValue ? v.Value : null, bufferSize),
+                  32 => new ParquetColumnReader<int?, long?>(reader, v => v.HasValue ? v.Value : null, bufferSize),
+                  64 => new ParquetColumnReader<long?>(reader, bufferSize),
+                  _ => throw new NotImplementedException(),
+               };
+            }
+            throw new NotImplementedException($"Expected unsigned int column type, but got {colType}");
+         }
+      }
+
+      IParquetColumnReader DecimalReader() {
+         var dlt = ExceptionAssert.of_type<DecimalLogicalType>(logicalType);
+         return new ParquetColumnReader<double?>(reader, bufferSize);
+      }
+   }
+
+   public static IColumnWriter GetColumnWriter(TypeCode typeCode, ColumnWriter writer, int bufferSize = 4096) {
+      return typeCode switch {
+         TypeCode.Boolean => new ColumnWriter<bool?>(writer, bufferSize),
+         TypeCode.SByte => new ColumnWriter<sbyte?>(writer, bufferSize, GetInt8Converter((IntLogicalType)writer.ColumnDescriptor.LogicalType)),
+         TypeCode.Int16 => new ColumnWriter<short?>(writer, bufferSize, GetInt16Converter((IntLogicalType)writer.ColumnDescriptor.LogicalType)),
+         TypeCode.Int32 => new ColumnWriter<int?>(writer, bufferSize, GetInt32Converter((IntLogicalType)writer.ColumnDescriptor.LogicalType)),
+         TypeCode.Int64 => new ColumnWriter<long?>(writer, bufferSize, GetInt64Converter((IntLogicalType)writer.ColumnDescriptor.LogicalType)),
+         TypeCode.Byte => new ColumnWriter<byte?>(writer, bufferSize, GetUInt8Converter((IntLogicalType)writer.ColumnDescriptor.LogicalType)),
+         TypeCode.UInt16 => new ColumnWriter<ushort?>(writer, bufferSize, GetUInt16Converter((IntLogicalType)writer.ColumnDescriptor.LogicalType)),
+         TypeCode.UInt32 => new ColumnWriter<uint?>(writer, bufferSize, GetUInt32Converter((IntLogicalType)writer.ColumnDescriptor.LogicalType)),
+         TypeCode.UInt64 => new ColumnWriter<ulong?>(writer, bufferSize, GetUInt64Converter((IntLogicalType)writer.ColumnDescriptor.LogicalType)),
+         TypeCode.Single => new ColumnWriter<float?>(writer, bufferSize),
+         TypeCode.Double => new ColumnWriter<double?>(writer, bufferSize),
+         TypeCode.DateTime => new ColumnWriter<DateTimeNanos?>(writer, bufferSize, ConvertDateTimeObj),
+         TypeCode.String => new ColumnWriter<string?>(writer, bufferSize),
+         _ => throw new NotSupportedException($"Unsupported column TypeCode {typeCode}"),
+      };
+
+      static DateTimeNanos? ConvertDateTimeObj(object? obj) {
+         if (obj is DateTime dt) {
+            return new(dt);
+         }
+         if (obj is DateTimeNanos dtn) {
+            return dtn;
+         }
+         throw new InvalidCastException($"Unable to convert object of type {obj?.GetType()} to DateTimeNanos");
+      }
+   }
+
+   public static Func<object?, sbyte?> GetInt8Converter(IntLogicalType logicalType) =>
+      GetInt8Converter(logicalType.IsSigned, logicalType.BitWidth);
+
+   public static Func<object?, sbyte?> GetInt8Converter(bool isSigned, int bitWidth) =>
+      isSigned
+         ? bitWidth switch {
+            8 => obj => obj as sbyte?,
+            16 => obj => obj is short v ? checked((sbyte)v) : null,
+            32 => obj => obj is int v ? checked((sbyte)v) : null,
+            64 => obj => obj is long v ? checked((sbyte)v) : null,
+            _ => throw new NotImplementedException(),
+         }
+         : bitWidth switch {
+            8 => obj => obj is byte v ? checked((sbyte)v) : null,
+            16 => obj => obj is ushort v ? checked((sbyte)v) : null,
+            32 => obj => obj is uint v ? checked((sbyte)v) : null,
+            64 => obj => obj is ulong v ? checked((sbyte)v) : null,
+            _ => throw new NotImplementedException(),
+         };
+
+   public static Func<object?, short?> GetInt16Converter(IntLogicalType logicalType) =>
+      GetInt16Converter(logicalType.IsSigned, logicalType.BitWidth);
+
+   public static Func<object?, short?> GetInt16Converter(bool isSigned, int bitWidth) =>
+      isSigned
+         ? bitWidth switch {
+            8 => obj => obj is sbyte v ? v : null,
+            16 => obj => obj as short?,
+            32 => obj => obj is int v ? checked((short)v) : null,
+            64 => obj => obj is long v ? checked((short)v) : null,
+            _ => throw new NotImplementedException(),
+         }
+         : bitWidth switch {
+            8 => obj => obj is byte v ? v : null,
+            16 => obj => obj is ushort v ? checked((short)v) : null,
+            32 => obj => obj is uint v ? checked((short)v) : null,
+            64 => obj => obj is ulong v ? checked((short)v) : null,
+            _ => throw new NotImplementedException(),
+         };
+
+   public static Func<object?, int?> GetInt32Converter(IntLogicalType logicalType) =>
+      GetInt32Converter(logicalType.IsSigned, logicalType.BitWidth);
+
+   public static Func<object?, int?> GetInt32Converter(bool isSigned, int bitWidth) =>
+      isSigned
+         ? bitWidth switch {
+            8 => obj => obj is sbyte v ? v : null,
+            16 => obj => obj is short v ? v : null,
+            32 => obj => obj as int?,
+            64 => obj => obj is long v ? checked((int)v) : null,
+            _ => throw new NotImplementedException(),
+         }
+         : bitWidth switch {
+            8 => obj => obj is byte v ? v : null,
+            16 => obj => obj is ushort v ? v : null,
+            32 => obj => obj is uint v ? checked((int)v) : null,
+            64 => obj => obj is ulong v ? checked((int)v) : null,
+            _ => throw new NotImplementedException(),
+         };
+
+   public static Func<object?, long?> GetInt64Converter(IntLogicalType logicalType) =>
+      GetInt64Converter(logicalType.IsSigned, logicalType.BitWidth);
+
+   public static Func<object?, long?> GetInt64Converter(bool isSigned, int bitWidth) =>
+      isSigned
+         ? bitWidth switch {
+            8 => obj => obj is sbyte v ? v : null,
+            16 => obj => obj is short v ? v : null,
+            32 => obj => obj is int v ? v : null,
+            64 => obj => obj as long?,
+            _ => throw new NotImplementedException(),
+         }
+         : bitWidth switch {
+            8 => obj => obj is byte v ? v : null,
+            16 => obj => obj is ushort v ? v : null,
+            32 => obj => obj is uint v ? v : null,
+            64 => obj => obj is ulong v ? checked((long)v) : null,
+            _ => throw new NotImplementedException(),
+         };
+
+   public static Func<object?, byte?> GetUInt8Converter(IntLogicalType logicalType) =>
+      GetUInt8Converter(logicalType.IsSigned, logicalType.BitWidth);
+
+   public static Func<object?, byte?> GetUInt8Converter(bool isSigned, int bitWidth) =>
+      isSigned
+         ? bitWidth switch {
+            8 => obj => obj is sbyte v ? checked((byte)v) : null,
+            16 => obj => obj is short v ? checked((byte)v) : null,
+            32 => obj => obj is int v ? checked((byte)v) : null,
+            64 => obj => obj is long v ? checked((byte)v) : null,
+            _ => throw new NotImplementedException(),
+         }
+         : bitWidth switch {
+            8 => obj => obj as byte?,
+            16 => obj => obj is ushort v ? checked((byte)v) : null,
+            32 => obj => obj is uint v ? checked((byte)v) : null,
+            64 => obj => obj is ulong v ? checked((byte)v) : null,
+            _ => throw new NotImplementedException(),
+         };
+
+   public static Func<object?, ushort?> GetUInt16Converter(IntLogicalType logicalType) =>
+      GetUInt16Converter(logicalType.IsSigned, logicalType.BitWidth);
+
+   public static Func<object?, ushort?> GetUInt16Converter(bool isSigned, int bitWidth) =>
+      isSigned
+         ? bitWidth switch {
+            8 => obj => obj is sbyte v ? checked((ushort)v) : null,
+            16 => obj => obj is short v ? checked((ushort)v) : null,
+            32 => obj => obj is int v ? checked((ushort)v) : null,
+            64 => obj => obj is long v ? checked((ushort)v) : null,
+            _ => throw new NotImplementedException(),
+         }
+         : bitWidth switch {
+            8 => obj => obj is byte v ? v : null,
+            16 => obj => obj is ushort v ? v : null,
+            32 => obj => obj is uint v ? checked((ushort)v) : null,
+            64 => obj => obj is ulong v ? checked((ushort)v) : null,
+            _ => throw new NotImplementedException(),
+         };
+
+   public static Func<object?, uint?> GetUInt32Converter(IntLogicalType logicalType) =>
+      GetUInt32Converter(logicalType.IsSigned, logicalType.BitWidth);
+
+   public static Func<object?, uint?> GetUInt32Converter(bool isSigned, int bitWidth) =>
+      isSigned
+         ? bitWidth switch {
+            8 => obj => obj is sbyte v ? checked((uint)v) : null,
+            16 => obj => obj is short v ? checked((uint)v) : null,
+            32 => obj => obj is int v ? checked((uint)v) : null,
+            64 => obj => obj is long v ? checked((uint)v) : null,
+            _ => throw new NotImplementedException(),
+         }
+         : bitWidth switch {
+            8 => obj => obj is byte v ? v : null,
+            16 => obj => obj is ushort v ? v : null,
+            32 => obj => obj is uint v ? v : null,
+            64 => obj => obj is ulong v ? checked((uint)v) : null,
+            _ => throw new NotImplementedException(),
+         };
+
+   public static Func<object?, ulong?> GetUInt64Converter(IntLogicalType logicalType) =>
+      GetUInt64Converter(logicalType.IsSigned, logicalType.BitWidth);
+
+   public static Func<object?, ulong?> GetUInt64Converter(bool isSigned, int bitWidth) =>
+      isSigned
+         ? bitWidth switch {
+            8 => obj => obj is sbyte v ? checked((ulong)v) : null,
+            16 => obj => obj is short v ? checked((ulong)v) : null,
+            32 => obj => obj is int v ? checked((ulong)v) : null,
+            64 => obj => obj is long v ? checked((ulong)v) : null,
+            _ => throw new NotImplementedException(),
+         }
+         : bitWidth switch {
+            8 => obj => obj is byte v ? v : null,
+            16 => obj => obj is ushort v ? v : null,
+            32 => obj => obj is uint v ? v : null,
+            64 => obj => obj is ulong v ? v : null,
+            _ => throw new NotImplementedException(),
+         };
+
+   public static string ColumnEnumTypeKey(string colName) =>
+      $"{colName}.EnumType";
+
+   public static KeyValuePair<string, string> ColumnEnumTypeEntry(string colName, Type enumType) {
+      if (enumType.IsNullable()) {
+         enumType = enumType.GetGenericArguments()[0];
+      }
+      if (!enumType.IsEnum) {
+         throw new ArgumentException($"Must be enum type (actually {enumType})", nameof(enumType));
+      }
+      return new(ColumnEnumTypeKey(colName), enumType.ToString());
+   }
+
+   public static Type? GetColumnEnumType(IReadOnlyDictionary<string, string>? keyValueMetadata, string columnName) {
+      if (keyValueMetadata is not null) {
+         var key = ColumnEnumTypeKey(columnName);
+         if (keyValueMetadata.TryGetValue(key, out var typeName)) {
+            return Type.GetType(typeName);
+         }
+      }
+      return null;
+   }
+
+   public interface IColumnWriter : IDisposable {
+      void Write(IReadOnlyCollection<object?> values);
+   }
+
+   public interface IColumnWriter<T> : IColumnWriter {
+      void Write(IReadOnlyCollection<T?> values);
+   }
+
+   class ColumnWriter<T> : IColumnWriter<T> {
+      readonly T?[] buffer;
+      readonly LogicalColumnWriter<T> writer;
+      readonly Func<object?, T?> convert;
+
+      public ColumnWriter(ColumnWriter writer, int bufferSize)
+         : this(writer, new T?[bufferSize]) { }
+
+      public ColumnWriter(ColumnWriter writer, int bufferSize, Func<object?, T?> convert)
+         : this(writer, new T?[bufferSize], convert) { }
+
+      public ColumnWriter(ColumnWriter writer, T?[] buffer)
+         : this(writer, buffer, obj => (T?)obj) { }
+
+      public ColumnWriter(ColumnWriter writer, T?[] buffer, Func<object?, T?> convert) {
+         this.buffer = buffer;
+         this.writer = writer.LogicalWriter<T>();
+         this.convert = convert;
+      }
+
+      public void Write(IReadOnlyCollection<T?> values) {
+         if (buffer.Length == 0) {
+            throw new InvalidOperationException("Cannot write because buffer length is 0");
+         }
+         var count = values.Count;
+         var nPages = count / buffer.Length;
+         using var en = values.GetEnumerator();
+         for (int pg = 0; pg < nPages; pg++) {
+            for (int i = 0; i < buffer.Length; i++) {
+               Log.assert(en.MoveNext(), "Enumeration failed before reaching expected count");
+               buffer[i] = en.Current;
+            }
+            writer.WriteBatch(buffer!);
+         }
+         var remainder = count - nPages * buffer.Length;
+         if (remainder != 0) {
+            for (int i = 0; i < remainder; i++) {
+               Log.assert(en.MoveNext(), "Enumeration failed before reaching expected count");
+               buffer[i] = en.Current;
+            }
+            writer.WriteBatch(buffer!, 0, remainder);
+         }
+      }
+
+      public void Dispose() =>
+         writer.Dispose();
+
+      void IColumnWriter.Write(IReadOnlyCollection<object?> values) =>
+         Write(ReadOnlySelectorCollection.Create(values, convert));
+   }
+}

src/DataStreamSuite/DataStream/BlueOrb/Parquet/ParquetColumnReader.cs

@@ -0,0 +1,111 @@
+namespace Bts.DataStream.Parquet;
+
+using bts;
+using bts.collections;
+
+using ParquetSharp;
+
+using System;
+using System.Collections.Generic;
+
+class ParquetColumnReader<T> : IParquetColumnReader<T> {
+   protected readonly LogicalColumnReader<T> reader;
+   protected readonly T?[] buffer;
+
+   public ParquetColumnReader(ColumnReader reader, int bufferSize)
+      : this(reader, new T?[bufferSize]) { }
+
+   protected ParquetColumnReader(ColumnReader reader, T?[] buffer) {
+      this.buffer = buffer;
+      this.reader = reader.LogicalReader<T>();
+   }
+
+   ~ParquetColumnReader() =>
+      Dispose();
+
+   public IList<T?> Read() {
+      if (buffer.Length == 0) {
+         throw new InvalidOperationException("Cannot write because buffer length is 0");
+      }
+      int n = reader.ReadBatch(buffer!);
+      return buffer.WrapSubRange(0, n);
+   }
+
+   public IList<T?> Read(int? start, int? stop) {
+      if (buffer.Length == 0) {
+         throw new InvalidOperationException("Cannot write because buffer length is 0");
+      }
+      start ??= 0;
+      stop ??= buffer.Length;
+      int length = stop.Value - start.Value;
+      if (length <= 0) {
+         return new List<T?>(0);
+      }
+      if (start.Value > 0) {
+         var nSkipped = checked((int)reader.Skip(start.Value));
+         Log.assert(nSkipped >= 0 && nSkipped <= start.Value);
+         if (nSkipped < start.Value) {
+            return new List<T?>(0);
+         }
+      }
+      int n = reader.ReadBatch(buffer!, 0, length);
+      return buffer.WrapSubRange(0, n);
+   }
+
+   public void Dispose() {
+      GC.SuppressFinalize(this);
+      reader.Dispose();
+   }
+}
+
+class ParquetColumnReader<TIn, TOut> : IParquetColumnReader<TOut> {
+   readonly Func<TIn, TOut?> convert;
+   readonly LogicalColumnReader<TIn> reader;
+   readonly TIn[] buffer;
+
+   public ParquetColumnReader(ColumnReader reader, Func<TIn, TOut?> convert, int bufferSize)
+      : this(reader, convert, new TIn[bufferSize]) { }
+
+   public ParquetColumnReader(ColumnReader reader, Func<TIn, TOut?> convert, TIn[] buffer) {
+      this.convert = convert;
+      this.reader = reader.LogicalReader<TIn>();
+      this.buffer = buffer;
+   }
+
+   ~ParquetColumnReader() =>
+      Dispose();
+
+   public IList<TOut?> Read() {
+      if (buffer.Length == 0) {
+         throw new InvalidOperationException("Cannot write because buffer length is 0");
+      }
+      int n = reader.ReadBatch(buffer!);
+      return buffer.WrapSelector(convert, null).WrapSubRange(0, n);
+   }
+
+   public IList<TOut?> Read(int? start, int? stop) {
+      if (buffer.Length == 0) {
+         throw new InvalidOperationException("Cannot write because buffer length is 0");
+      }
+      start ??= 0;
+      stop ??= buffer.Length;
+      int length = stop.Value - start.Value;
+      if (length <= 0) {
+         return new List<TOut?>(0);
+      }
+      if (start.Value > 0) {
+         var nSkipped = checked((int)reader.Skip(start.Value));
+         Log.assert(nSkipped >= 0 && nSkipped <= start.Value);
+         if (nSkipped < start.Value) {
+            return new List<TOut?>(0);
+         }
+      }
+      int n = reader.ReadBatch(buffer!, 0, length);
+      return buffer.WrapSelector(convert, null).WrapSubRange(0, n);
+   }
+
+   public void Dispose() {
+      GC.SuppressFinalize(this);
+      reader.Dispose();
+   }
+}

src/DataStreamSuite/DataStream/BlueOrb/Parquet/ParquetExtensions.cs

@@ -0,0 +1,159 @@
+namespace Bts.DataStream.Parquet;
+
+using BCG;
+
+using bts;
+using bts.collections;
+
+using Microsoft.Scripting.Utils;
+
+using ParquetSharp;
+
+using System;
+using System.Collections.Generic;
+using System.Linq;
+
+public static class ParquetExtensions {
+   public static int RowCount(this FileMetaData fileMetaData) =>
+      checked((int)fileMetaData.NumRows);
+
+   public static IBlueTableSchema ToBlueSchema(this SchemaDescriptor schemaDescriptor, string tableName = "parquet", IReadOnlyDictionary<string, string>? keyValueMetadata = null) {
+      var colInfos = new ColumnInfo[schemaDescriptor.NumColumns];
+      for (int i = 0; i < colInfos.Length; i++) {
+         colInfos[i] = schemaDescriptor.BlueColumn(i, keyValueMetadata);
+      }
+      return new BlueTableSchema(tableName, colInfos);
+   }
+
+   public static IBlueTableSchema ToBlueSchema(this FileMetaData fileMetaData, string tableName = "parquet") =>
+      ToBlueSchema(fileMetaData.Schema, tableName, fileMetaData.KeyValueMetadata);
+
+   public static ColumnInfo BlueColumn(this SchemaDescriptor schemaDescriptor, int index, IReadOnlyDictionary<string, string>? keyValueMetadata = null) {
+      var c = schemaDescriptor.Column(index);
+      var lt = c.LogicalType;
+      if (lt.Type == LogicalTypeEnum.Enum) {
+         var enumType = ParquetAdapter.GetColumnEnumType(keyValueMetadata, c.Name);
+         if (enumType is not null) {
+            return new(c.Name, index, enumType);
+         }
+      }
+      return new(c.Name, index, getTypeCode(c.LogicalType, c.PhysicalType));
+   }
+
+   public static IBlueRowCollection ReadBlueTable(this ParquetFileReader fileReader, int? start = null, int? stop = null) {
+      var schema = fileReader.FileMetaData.ToBlueSchema("parquetReader");
+      var colInfos = schema.columnInfo;
+      IBlueRowCollection? rv;
+      if (!schema.columnInfo.Any(c => c.colName == "id")) {
+         rv = BlueAddIndexColumnTable.Create(schema, fileReader.ReadColumns(start, stop));
+      } else {
+         rv = new BlueIndexedColumnTable(schema, fileReader.ReadColumns(start, stop));
+      }
+      Log.assert(!schema.columnInfo.Any(c => c is null));
+      return rv;
+   }
+
+   public static (Column[] Column, IReadOnlyDictionary<string, string> Meta) ToParquetSchema(this IBlueTableSchema schema) {
+      var blueColumns = schema.columnInfo;
+      var parquetColumns = new Column[blueColumns.Length];
+      var parquetMeta = new Dictionary<string, string>();
+      KeyValuePair<string, string>[] parquetColMeta;
+      for (int i = 0; i < blueColumns.Length; i++) {
+         (parquetColumns[i], parquetColMeta) = ToParquetColumn(blueColumns[i]);
+         parquetMeta.AddRange(parquetColMeta);
+      }
+      return (parquetColumns, parquetMeta);
+   }
+
+   public static (Column Column, KeyValuePair<string, string>[] Meta) ToParquetColumn(this ColumnInfo columnInfo) {
+      var colType = columnInfo.colType;
+      bool isNullable = colType.IsNullable();
+      if (isNullable) {
+         colType = colType.GetGenericArguments()[0];
+      }
+      KeyValuePair<string, string>[] meta;
+      TypeCode typeCode;
+      var colName = columnInfo.colName;
+      if (colType.IsEnum) {
+         typeCode = colType.GetEnumUnderlyingType().GetTypeCode();
+         meta = new[] { ParquetAdapter.ColumnEnumTypeEntry(columnInfo.colName, colType) };
+      } else {
+         typeCode = colType.GetTypeCode();
+         meta = Array.Empty<KeyValuePair<string, string>>();
+      }
+      return (ToParquetColumn(colName, typeCode), meta);
+
+      static Column ToParquetColumn(string colName, TypeCode typeCode) =>
+         typeCode switch {
+            TypeCode.Boolean => new Column<bool?>(colName),
+            TypeCode.Byte => new Column<byte?>(colName),
+            TypeCode.Int16 => new Column<short?>(colName),
+            TypeCode.Int32 => new Column<int?>(colName),
+            TypeCode.Int64 => new Column<long?>(colName),
+            TypeCode.UInt16 => new Column<ushort?>(colName),
+            TypeCode.UInt32 => new Column<uint?>(colName),
+            TypeCode.UInt64 => new Column<ulong?>(colName),
+            TypeCode.Single => new Column<float?>(colName),
+            TypeCode.Double => new Column<double?>(colName),
+            TypeCode.DateTime => new Column<DateTimeNanos?>(colName),
+            TypeCode.String => new Column<string?>(colName),
+            _ => throw new NotSupportedException($"Unsupported column TypeCode {typeCode}"),
+         };
+   }
+
+   static TypeCode getTypeCode(LogicalType logicalType, PhysicalType physicalType) {
+      return logicalType.Type switch {
+         // LogicalTypeEnum.Int => TypeCode.Int32,
+         // LogicalTypeEnum.Decimal => TypeCode.Double,
+         LogicalTypeEnum.Date or LogicalTypeEnum.Time or LogicalTypeEnum.Timestamp => TypeCode.DateTime,
+         LogicalTypeEnum.String => TypeCode.String,
+         LogicalTypeEnum.Int => FromInt(),
+         LogicalTypeEnum.Decimal => FromDecimal(),
+         _ => FromPhysicalType(),
+      };
+
+      TypeCode FromPhysicalType() =>
+         physicalType switch {
+            PhysicalType.Boolean => TypeCode.Boolean,
+            PhysicalType.Int32 => TypeCode.Int32,
+            PhysicalType.Int64 => TypeCode.Int64,
+            PhysicalType.Float => TypeCode.Single,
+            PhysicalType.Double => TypeCode.Double,
+            _ => throw new NotImplementedException($"TypeCode not implemented for {logicalType} (logical) {physicalType} (physical)"),
+         };
+
+      TypeCode FromInt() {
+         var ilt = (IntLogicalType)logicalType;
+         var bitWidth = ilt.BitWidth;
+         return ilt.IsSigned ? FromSignedInt() : FromUnsignedInt();
+
+         TypeCode FromSignedInt() =>
+            bitWidth switch {
+               8 => TypeCode.SByte,
+               16 => TypeCode.Int16,
+               32 => TypeCode.Int32,
+               64 => TypeCode.Int64,
+               _ => throw new NotSupportedException($"Bit width not supported for signed int ({bitWidth})"),
+            };
+
+         TypeCode FromUnsignedInt() =>
+            bitWidth switch {
+               8 => TypeCode.Byte,
+               16 => TypeCode.UInt16,
+               32 => TypeCode.UInt32,
+               64 => TypeCode.UInt64,
+               _ => throw new NotSupportedException($"Bit width not supported for unsigned int ({bitWidth})"),
+            };
+      }
+
+      TypeCode FromDecimal() {
+         var dlt = (DecimalLogicalType)logicalType;
+         var scale = dlt.Scale;
+         var precision = dlt.Precision;
+         return scale switch {
+            0 => throw new NotSupportedException($"Scale not supported for decimal ({scale})"),
+            _ => TypeCode.Double,
+         };
+      }
+   }
+}

src/DataStreamSuite/DataStream/BlueOrb/Parquet/ParquetGroupTable.cs

@@ -0,0 +1,234 @@
+namespace Bts.DataStream.Parquet;
+
+using BCG;
+
+using bts;
+using bts.collections;
+
+using Bts.DataStreamCore;
+
+using ParquetSharp;
+
+using Spark.Core.Collections;
+
+using System;
+using System.Collections;
+using System.Collections.Generic;
+using System.Diagnostics.CodeAnalysis;
+
+public class ParquetGroupTable : IBlueRowCollection, IDisposable {
+   readonly RowGroupReader reader;
+   readonly IBlueTableSchema schema;
+   readonly bool leaveOpen;
+   readonly int fullCount;
+   readonly object syncRoot = new();
+   readonly int? start;
+   readonly int? stop;
+   readonly int startIndex;
+   readonly int stopIndex;
+   readonly int idColumnIndex;
+   readonly int idColumnIndexInFile;
+   readonly Func<int, int>? rowIdFactory;
+   DataCache? dataCacheBacking;
+
+   public ParquetGroupTable(
+      RowGroupReader reader,
+      IBlueTableSchema schema,
+      int? start = null,
+      int? stop = null,
+      int idColumnIndex = -1,
+      int idColumnIndexInFile = -1,
+      Func<int, int>? rowIdFactory = null,
+      bool leaveOpen = false
+   ) {
+      this.reader = reader;
+      this.schema = schema;
+      this.start = start;
+      this.stop = stop;
+      this.idColumnIndex = idColumnIndex;
+      this.idColumnIndexInFile = idColumnIndexInFile;
+      this.rowIdFactory = rowIdFactory;
+      this.leaveOpen = leaveOpen;
+      fullCount = checked((int)reader.MetaData.NumRows);
+      Log.assert(fullCount >= 0);
+      startIndex = SliceList.GetStartIndex(start, fullCount, true);
+      stopIndex = SliceList.GetStopIndex(stop, fullCount, true);
+      stopIndex = Math.Max(stopIndex, startIndex);
+      Log.assert(
+         startIndex >= 0
+         && startIndex <= fullCount
+         && stopIndex >= 0
+         && stopIndex <= fullCount
+         && stopIndex >= startIndex);
+   }
+
+   ~ParquetGroupTable() =>
+      Dispose();
+
+   public int Count => stopIndex - startIndex;
+
+   IBlueTableSchema IBlueRowCollection.schema => schema;
+
+   DataCache dataCache =>
+      EnsureInit();
+
+   public IBlueRow this[int id] => dataCache[id];
+
+   public bool containsID(int id) =>
+      dataCache.RowIdToIndex.ContainsKey(id);
+
+   public IBlueRow GetRow(int index) =>
+      dataCache.Rows[index];
+
+   public int IndexOf(IBlueRow item) {
+      int id = item.Id();
+      return dataCache.RowIdToIndex.TryGetValue(id, out var idx) && BlueRowEqualityComparer.Equals(dataCache.Rows[idx], item)
+         ? idx
+         : -1;
+   }
+
+   public bool TryGetValue(int id, [NotNullWhen(true)] out IBlueRow? row) {
+      if (dataCache.RowIdToIndex.TryGetValue(id, out var idx)) {
+         row = dataCache.Rows[idx];
+         return true;
+      }
+      row = null;
+      return false;
+   }
+
+   public IEnumerator<IBlueRow> GetEnumerator() {
+      for (int i = 0; i < dataCache.Rows.Length; i++) {
+         yield return dataCache.Rows[i];
+      }
+   }
+
+   public void Dispose() {
+      GC.SuppressFinalize(this);
+      if (!leaveOpen) {
+         reader.Dispose();
+      }
+   }
+
+   [MemberNotNull(nameof(dataCacheBacking))]
+   internal DataCache EnsureInit() {
+      if (dataCacheBacking.HasValue) {
+         return dataCacheBacking.Value;
+      }
+      lock (syncRoot) {
+         if (dataCacheBacking.HasValue) {
+            return dataCacheBacking.Value;
+         }
+         var rows = new IBlueRow[Count];
+         int numColumns = reader.MetaData.NumColumns;
+         var blueColumns = schema.columnInfo;
+         Log.assert((blueColumns.Length - numColumns) switch {
+            0 or 1 => true,
+            _ => false,
+         });
+         var columnValues = allocateColumnValues(
+            reader, blueColumns.Length, Count, idColumnIndex, idColumnIndexInFile, startIndex, stopIndex, rowIdFactory);
+         var rowIdToIndex = new Dictionary<int, int>();
+         for (int i = 0; i < rows.Length; i++) {
+            rows[i] = new BlueRowColumnWrapper(schema, i, columnValues);
+         }
+         dataCacheBacking = new(rows, rowIdToIndex, columnValues);
+      }
+      return dataCacheBacking.Value;
+   }
+
+   static IList<object?>[] allocateColumnValues(
+      RowGroupReader reader,
+      int numColumns,
+      int numRows,
+      int idColIdx,
+      int idColIdxInFile,
+      int startIndex,
+      int stopIndex,
+      Func<int, int>? idFactory
+   ) {
+      if (idColIdx < 0 || idColIdx >= numColumns) {
+         throw new ArgumentException("Id column index out of bounds", nameof(idColIdx));
+      }
+      var columnValues = new IList<object?>[numColumns];
+      if (idColIdxInFile == -1) {
+         // Generate ids (there is no id column in the file)
+         ArgumentNullException.ThrowIfNull(idFactory);
+         for (int i = 0; i < numColumns; i++) {
+            if (i < idColIdx) {
+               ReadColumn(i, i);
+            } else if (i == idColIdx) {
+               columnValues[idColIdx] = new StepList<int>(0, 1, numRows).WrapSelector(idx => (object?)idFactory(idx), null);
+            } else {
+               ReadColumn(i - 1, i);
+            }
+         }
+      } else if (idFactory is not null) {
+         // Generate ids (skip id column in the file)
+         for (int i = 0; i < numColumns; i++) {
+            if (i == idColIdx) {
+               columnValues[idColIdx] = new StepList<int>(0, 1, numRows).WrapSelector(idx => (object?)idFactory(idx), null);
+            } else {
+               if (i > idColIdx && i < idColIdxInFile) {
+                  ReadColumn(i - 1, i);
+               } else if (i > idColIdxInFile && i < idColIdx) {
+                  ReadColumn(i + 1, i);
+               } else {
+                  ReadColumn(i, i);
+               }
+            }
+         }
+      } else {
+         // Get ids from the file (output table id column index may differ from file)
+         // Case 1) File id col index > output
+         //    src cols: a, b, c, d, (id), e, f, ...
+         //    dst cols: a, b, (id), c, d, e, f, ...
+         //
+         // Case 2) File id col index == output
+         //    src cols: a, b, (id), c, d, e, f, ...
+         //    dst cols: a, b, (id), c, d, e, f, ...
+         //
+         // Case 3) File id col index < output
+         //    src cols: a, b, (id), c, d, e, f, ...
+         //    dst cols: a, b, c, d, (id), e, f, ...
+
+         for (int i = 0; i < numColumns; i++) {
+            if (i == idColIdx) {
+               ReadColumn(idColIdxInFile, idColIdx);
+            } else if (i > idColIdx && i < idColIdxInFile) {
+               ReadColumn(i - 1, i);
+            } else if (i > idColIdxInFile && i < idColIdx) {
+               ReadColumn(i, i - 1);
+            } else {
+               ReadColumn(i, i);
+            }
+         }
+      }
+      return columnValues;
+
+      void ReadColumn(int srcIdx, int dstIdx) {
+         using var colReader = reader.GetColumnReader(srcIdx);
+         columnValues[dstIdx] = colReader.Read(startIndex, stopIndex);
+      }
+   }
+
+   IEnumerator IEnumerable.GetEnumerator() =>
+      GetEnumerator();
+
+   internal readonly struct DataCache {
+      public readonly IBlueRow[] Rows;
+      public readonly Dictionary<int, int> RowIdToIndex;
+      public readonly IList<object?>[] ColumnValues;
+
+      public DataCache(IBlueRow[] rows, Dictionary<int, int> rowIdToIndex, IList<object?>[] columnValues) {
+         Rows = rows;
+         RowIdToIndex = rowIdToIndex;
+         ColumnValues = columnValues;
+      }
+
+      public IBlueRow this[int id] =>
+         Rows[RowIdToIndex[id]];
+
+      public IBlueRow GetRow(int index) =>
+         Rows[index];
+   }
+}

src/DataStreamSuite/DataStream/BlueOrb/Parquet/ParquetReaderStream.cs

@@ -0,0 +1,65 @@
+namespace Bts.DataStream.Parquet;
+
+using BCG;
+
+using bts;
+using bts.collections;
+
+using Bts.DataStreamCore;
+using Bts.PubSub;
+
+using Spark.Core.Disposables;
+
+using System;
+using System.Threading;
+
+[DataStreamUserRespresentation(false)]
+public class ParquetReaderStream : ViewStream {
+   readonly string file;
+   readonly int? start;
+   readonly int? stop;
+   readonly IWorkScheduler workScheduler;
+   IBlueRowCollection? table;
+   IDisposable? workerSub;
+
+   public ParquetReaderStream(string file, IWorkScheduler workScheduler) {
+      this.file = file;
+      this.workScheduler = workScheduler;
+   }
+
+   public ParquetReaderStream(string file, int? start, int? stop, IWorkScheduler workScheduler) {
+      this.file = file;
+      this.start = start;
+      this.stop = stop;
+      this.workScheduler = workScheduler;
+   }
+
+   protected override void myOnFirstSubscriber() {
+      Start();
+      var worker = new AnonymousWorker(DoWork);
+      workScheduler.Add(worker);
+      workerSub = Disposable.Create(() => workScheduler.Remove(worker));
+
+      async void Start() {
+         try {
+            Volatile.Write(ref table, await ParquetTableReader.ReadAsync(file, start, stop));
+         } catch (Exception ex) {
+            Log.DataFlow.error($"[parquet_reader_stream_err.json]{{ \"err\": \"{ex}\" }}");
+         }
+      }
+
+      void DoWork() {
+         var table = Volatile.Read(ref this.table);
+         if (table is null) {
+            return;
+         }
+         Disposable.dispose_thread_safe(ref workerSub);
+         if (table.schema.columnInfo.Length > 1) {
+            myFireEvent(new DataPacket(table, table, new BlueTableImmutable(table.schema)));
+         }
+      }
+   }
+
+   protected override void myOnLastSubscriber() =>
+      Disposable.dispose_thread_safe(ref workerSub);
+}

src/DataStreamSuite/DataStream/BlueOrb/Parquet/ParquetRow.cs

@@ -0,0 +1,26 @@
+// namespace Bts.DataStream.Parquet;
+
+// using BCG;
+
+// using global::Parquet.Data;
+
+// public class ParquetRow : IBlueRow {
+//    readonly int id;
+//    readonly int rowIndex;
+//    readonly DataColumn[] data;
+
+//    public ParquetRow(int id, int rowIndex, DataColumn[] data, IBlueTableSchema schema) {
+//       this.id = id;
+//       this.rowIndex = rowIndex;
+//       this.data = data;
+//       Schema = schema;
+//    }
+
+//    public object? this[int index] =>
+//       index == 0
+//          ? id
+//          : data[index - 1].Data.GetValue(rowIndex);
+
+//    public IBlueTableSchema Schema { get; }
+//    IBlueTableSchema IBlueRow.schema => Schema;
+// }

src/DataStreamSuite/DataStream/BlueOrb/Parquet/ParquetTable.cs

@@ -0,0 +1,145 @@
+namespace Bts.DataStream.Parquet;
+
+using BCG;
+
+using bts;
+using bts.collections;
+
+using Bts.DataStreamCore;
+
+using ParquetSharp;
+
+using Spark.Core.Collections;
+
+using System;
+using System.Collections;
+using System.Collections.Generic;
+using System.Linq;
+using System.Threading.Tasks;
+
+public class ParquetTable : IBlueRowCollection, IDisposable {
+   readonly SortedList<int, ParquetGroupTable> groups = new();
+   readonly int fullCount;
+   readonly IBlueTableSchema schema;
+   readonly ParquetFileReader? reader;
+   readonly int startIndex;
+   readonly int stopIndex;
+
+   public ParquetTable(ParquetFileReader reader, string tableName = "parquet", int? start = null, int? stop = null, bool leaveOpen = false) {
+      schema = reader.FileMetaData.ToBlueSchema(tableName);
+      int idColIdx;
+      if (schema.TryResolveIndices("id", out int idColIdxInFile)) {
+         idColIdx = idColIdxInFile;
+      } else {
+         var origCols = schema.columnInfo;
+         var cols = new ColumnInfo[origCols.Length + 1];
+         idColIdx = 0;
+         cols[0] = new("id", 0, typeof(int));
+         for (int i = 0; i < origCols.Length; i++) {
+            var c = origCols[i];
+            cols[i + 1] = new(c.colName, i + 1, c.colType);
+         }
+         schema = new BlueTableSchema(cols, schema.tableName);
+      }
+      fullCount = checked((int)reader.FileMetaData.NumRows);
+      Log.assert(fullCount >= 0);
+      startIndex = SliceList.GetStartIndex(start, fullCount, true);
+      stopIndex = SliceList.GetStopIndex(stop, fullCount, true);
+      stopIndex = Math.Max(stopIndex, startIndex);
+      Log.assert(startIndex >= 0 && startIndex <= fullCount && stopIndex >= 0 && stopIndex <= fullCount && stopIndex >= startIndex);
+      int numRowGroups = reader.FileMetaData.NumRowGroups;
+      int rowGroupLastIdx = 0;
+      object syncRoot = new();
+      for (int i = 0; i < numRowGroups && rowGroupLastIdx < Count; i++) {
+         int? grpStart = start.HasValue ? startIndex - rowGroupLastIdx : null;
+         int? grpStop = stop.HasValue ? stopIndex - rowGroupLastIdx : null;
+         int offset = rowGroupLastIdx;
+         var grp = new ParquetGroupTable(reader.RowGroup(i), schema, grpStart, grpStop, idColIdx, idColIdxInFile, GetAutoId);
+         if (grp.Count == 0) {
+            continue;
+         }
+         rowGroupLastIdx += grp.Count;
+         groups.Add(rowGroupLastIdx, grp);
+
+         int GetAutoId(int index) =>
+            index + offset;
+      }
+      Parallel.For(0, groups.Count, i => groups.GetValueAtIndex(i).EnsureInit());
+      if (!leaveOpen) {
+         this.reader = reader;
+      }
+   }
+
+   ~ParquetTable() =>
+      Dispose();
+
+   public IBlueTableSchema Schema => schema;
+
+   public int Count => stopIndex - startIndex;
+
+   IBlueTableSchema IBlueRowCollection.schema => schema;
+
+   public IBlueRow this[int index] {
+      get {
+         foreach (var grp in groups.Values) {
+            if (grp.TryGetValue(index, out var r)) {
+               return r;
+            }
+         }
+         throw new KeyNotFoundException();
+      }
+   }
+
+   public static ParquetTable FromFile(string path, int? start = null, int? stop = null, string tableName = "parquet") =>
+      new(new(path), tableName, start, stop, false);
+
+   public IBlueRow GetRow(int index) {
+      if (index < 0 || index >= Count) {
+         throw new IndexOutOfRangeException();
+      }
+      int rowGroupIdx = groups.Keys.BinarySearch(index);
+      if (rowGroupIdx < 0) {
+         rowGroupIdx = ~rowGroupIdx;
+      }
+      Log.assert(rowGroupIdx <= groups.Count);
+      var grp = groups[rowGroupIdx];
+      int idxWithinGrp = index;
+      if (rowGroupIdx > 0) {
+         idxWithinGrp -= groups.Keys[rowGroupIdx - 1];
+      }
+      return grp[idxWithinGrp];
+   }
+
+   public bool containsID(int id) =>
+      groups.Values.Any(rgl => rgl.containsID(id));
+
+   public int IndexOf(IBlueRow item) {
+      int idx;
+      int grpFirstIdx = 0;
+      foreach (var (grpLastIdx, grp) in groups) {
+         idx = grp.IndexOf(item);
+         if (idx >= 0) {
+            return idx + grpFirstIdx;
+         }
+         grpFirstIdx = grpLastIdx;
+      }
+      return -1;
+   }
+
+   public IEnumerator<IBlueRow> GetEnumerator() {
+      foreach (var rgl in groups.Values) {
+         foreach (var r in rgl) {
+            yield return r;
+         }
+      }
+   }
+
+   public void Dispose() {
+      GC.SuppressFinalize(this);
+      groups.Values.DisposeAll();
+      reader?.Dispose();
+   }
+
+   IEnumerator IEnumerable.GetEnumerator() =>
+      GetEnumerator();
+}

src/DataStreamSuite/DataStream/BlueOrb/Parquet/ParquetTableReader.cs

@@ -0,0 +1,32 @@
+#nullable enable
+namespace Bts.DataStream.Parquet;
+
+using BCG;
+
+using Bts.DataStreamCore;
+
+using ParquetSharp;
+
+using System.IO;
+using System.Threading.Tasks;
+
+public class ParquetTableReader {
+   public static Task<IBlueRowCollection> ReadAsync(Stream stream, bool leaveOpen = false, int? start = null, int? stop = null) =>
+      Task.Run(() => Read(stream, leaveOpen, start, stop));
+
+   public static Task<IBlueRowCollection> ReadAsync(string path, int? start = null, int? stop = null) =>
+      Task.Run(() => Read(path, start, stop));
+
+   public static IBlueRowCollection Read(Stream stream, bool leaveOpen = false, int? start = null, int? stop = null) {
+      using var fileReader = new ParquetFileReader(stream, leaveOpen);
+      var rv = fileReader.ReadBlueTable(start, stop);
+      rv.schema.Validate();
+      return rv;
+   }
+
+   public static IBlueRowCollection Read(string path, int? start = null, int? stop = null) {
+      IBlueRowCollection rv = ParquetTable.FromFile(path, start, stop);
+      rv.schema.Validate();
+      return rv;
+   }
+}

src/DataStreamSuite/DataStream/BlueOrb/Parquet/ParquetTableWriter.cs

@@ -0,0 +1,38 @@
+namespace Bts.DataStream.Parquet;
+
+using BCG;
+
+using Bts.DataStreamCore;
+
+using bts;
+
+using ParquetSharp;
+
+using System.IO;
+using System.Threading.Tasks;
+
+public static class ParquetTableWriter {
+   public static Task WriteAsync(IBlueRowCollection blueRows, Stream outStream, bool leaveOpen = false) =>
+      Task.Run(() => Write(blueRows, outStream, leaveOpen));
+
+   public static void Write(IBlueRowCollection blueRows, Stream outStream, bool leaveOpen = false) {
+      var (parquetColumns, parquetMeta) = blueRows.schema.ToParquetSchema();
+      using var schema = Column.CreateSchemaNode(parquetColumns);
+      using var writerProps = createWriterProperties();
+      using var writer = new ParquetFileWriter(outStream, schema, writerProps, keyValueMetadata: parquetMeta, leaveOpen: leaveOpen);
+      using var rowGroup = writer.AppendRowGroup();
+      for (int i = 0; i < blueRows.schema.columnInfo.Length; i++) {
+         var col = blueRows.schema.columnInfo[i];
+         Log.assert(col.colNo == i);
+         using var colWriter = ParquetAdapter.GetColumnWriter(col.type_code(), rowGroup.NextColumn());
+         colWriter.Write(blueRows.GetColumn(i));
+      }
+      writer.Close();
+   }
+
+   static WriterProperties createWriterProperties(Compression compression = Compression.Snappy) {
+      using var builder = new WriterPropertiesBuilder();
+      builder.Compression(compression);
+      return builder.Build();
+   }
+}

src/DataStreamSuite/DataStream/BlueOrb/Parquet/ParquetWriterStream.cs

@@ -0,0 +1,87 @@
+namespace Bts.DataStream.Parquet;
+
+using BCG;
+
+using bts.collections;
+
+using Bts.DataStreamCore;
+using Bts.PubSub;
+
+using mbts.utils;
+
+using Spark.Core.Disposables;
+using Spark.Core.Threading;
+
+using System;
+using System.IO;
+
+[DataStreamUserRespresentation(false)]
+public class ParquetWriterStream : ViewStream {
+   readonly string file;
+   readonly ViewStream inputStream;
+   readonly IWorkScheduler workScheduler;
+   readonly AtomicGate gate = new();
+   IDataPacket? pendingOutput;
+   IDisposable? dispose;
+
+   public ParquetWriterStream(ViewStream inputStream, string file, IWorkScheduler workScheduler) {
+      this.inputStream = inputStream;
+      this.file = file;
+      this.workScheduler = workScheduler;
+   }
+
+   protected override void myOnFirstSubscriber() {
+      BlueTableImmutable.DataPacketBuilder? outputBuilder = null;
+      bool isFirstUpdate = false;
+      var worker = new AnonymousWorker(DoWork);
+      inputStream.onChange += OnInputStreamChange;
+      workScheduler.Add(worker);
+      dispose = Disposable.Create(Dispose);
+
+      void Dispose() {
+         workScheduler.Remove(worker);
+         inputStream.onChange -= OnInputStreamChange;
+      }
+
+      void OnInputStreamChange(object? dpObj) {
+         var dp = ExceptionAssert.Argument.Is<IDataPacket>(dpObj);
+         using (gate.BeginTransaction()) {
+            outputBuilder ??= new BlueTableImmutable(dp.Schema).ToDataPacketBuilder();
+            foreach (IBlueRow r in dp.addedRows) {
+               outputBuilder.AddOrUpdate(r);
+            }
+            foreach (IBlueRow r in dp.deletedRows) {
+               outputBuilder.Remove(r.Id());
+            }
+         }
+      }
+
+      async void DoWork() {
+         using (gate.BeginTransaction()) {
+            if (outputBuilder is null) {
+               return;
+            }
+            if (pendingOutput is not null) {
+               myFireEvent(pendingOutput);
+               pendingOutput = null;
+            }
+            var dp = outputBuilder.ToDataPacket();
+            if (isFirstUpdate) {
+               isFirstUpdate = false;
+            } else if (!dp.HasChanges()) {
+               return;
+            }
+            {
+               using var outStream = File.OpenWrite(file);
+               await ParquetTableWriter.WriteAsync(dp.newRows, outStream);
+            }
+            // Fire the update at the beginning of the next doWork call so it happens on the
+            // 'work' thread.
+            pendingOutput = dp;
+         }
+      }
+   }
+
+   protected override void myOnLastSubscriber() =>
+      Disposable.dispose_thread_safe(ref dispose);
+}

src/DataStreamSuite/DataStream/BlueOrb/TimeSeriesStream.cs

@@ -1,25 +1,39 @@
-using System;
-using System.Data.Entity.Core.Mapping;
-
 using BCG;
 
+using bts.utils;
+
+using Bts.DataStreamCore;
+
+using mbts.utils;
+
+using Spark.Core.Collections;
+using Spark.Core.Threading;
+
+using System;
+using System.Collections.Generic;
+using System.Collections.Immutable;
+using System.Linq;
+
 namespace bts.BlueOrb {
    // Keeps adding changing rows to the stream, hence preserving the history
    // and creating a time series, for example this can be used to make bbo updates
    // into a time series of prices/qty
    [DataStreamUserRespresentation("TimeSeriesStream")]
    public class TimeSeriesStream : ViewStream {
-      private CircularBufferTable _table;
-      private ViewStream _stream;
+      readonly ViewStream _stream;
+      readonly string timeColumn;
+      readonly AtomicGate gate = new();
+      CircularBufferTable? _table;
       private int _id = 0;
       private int _size = 0;
 
       public TimeSeriesStream(ViewStream stream)
          : this(stream, Int32.MaxValue) { }
 
-      public TimeSeriesStream(ViewStream stream, int size) {
+      public TimeSeriesStream(ViewStream stream, int size, string timeColumn = "ts_event") {
          _stream = stream;
          _size = size;
+         this.timeColumn = timeColumn;
       }
 
       protected override void myOnFirstSubscriber() {
@@ -31,27 +45,37 @@ namespace bts.BlueOrb {
 
 
       private void _onstream_change(object o) {
-         DataPacket dp = o as DataPacket;
+         var dp = ExceptionAssert.Argument.Is<IDataPacket>(o);
          DateTime now = DateTime.Now;
+         using var _ = gate.BeginTransaction();
          if (_table == null) {
-            IBlueTableSchema schema = new BlueTableSchema(dp.newRows.schema.columnInfo, dp.newRows.schema.tableName);
-            schema = schema.add_columns(Tuple.Create(typeof(DateTime), "timestamp"));
-            _table = new CircularBufferTable(schema, _size);
+            var colInfos = dp.newRows.schema.columnInfo;
+            if (colInfos.TryGetFirst(c => c.colName == timeColumn, out var currTimeCol, out var currTimeColIdx)) {
+               colInfos = colInfos.ToArray();
+               colInfos[currTimeColIdx] = new(timeColumn, currTimeCol.colNo, typeof(DateTime));
+            } else {
+               colInfos = ArrayHelper.add(colInfos, new(timeColumn, colInfos.Length, typeof(DateTime)));
+            }
+            _table = new CircularBufferTable(new BlueTableSchema(colInfos, dp.newRows.schema.tableName), _size);
          }
+         var addedRowsBd = new BlueTableImmutable(_table.schema).ToDataPacketBuilder();
+         var deletedRowsBd = new BlueTableImmutable(_table.schema).ToDataPacketBuilder();
          foreach (IBlueRow r in dp.addedRows) {
             MutableBlueRow ar = new MutableBlueRow(_table.schema, r);
-            ar.setValue("timestamp", now);
-            ar.setValue("id", _id++);
-            _table.add_row(ar);
+            ar.setValue(timeColumn, now);
+            int id = _id++;
+            ar.setValue("id", id);
+            _table = _table.Add(ar, out var oldRow);
+            addedRowsBd.AddOrUpdate(ar);
+            if (oldRow is not null) {
+               deletedRowsBd.AddOrUpdate(oldRow);
+            }
          }
-         var dp_out =  BlueDataUtility.build_dp(_table, myLastValue as DataPacket);
+         var addedRows = addedRowsBd.ToImmutable();
+         var deletedRows = deletedRowsBd.ToImmutable();
+         var dp_out = new DataPacket(_table, addedRows, deletedRows);
          myFireEvent(dp_out);
       }
    }
-
-
-
 }
 
-
-

src/DataStreamSuite/DataStream/BlueOrb/TimeseriesIndex.cs

@@ -0,0 +1,197 @@
+namespace Bts.DataStream;
+
+using bts;
+
+using Spark.Core.Collections;
+
+using System;
+using System.Collections.Generic;
+using System.Collections.Immutable;
+
+/// <summary>
+/// Maps a timeseries of keys to unique int32 ids.
+/// </summary>
+public class TimeseriesIndex<T> where T : notnull {
+   static readonly IComparer<KeyValuePair<DateTime, int>> idTimeseriesComp =
+      KeyValuePairComparer.KeyComparer<DateTime, int>();
+
+   readonly ImmutableDictionary<T, ImmutableList<KeyValuePair<DateTime, int>>> nameToId;
+   readonly ImmutableDictionary<int, TimeseriesKey<T>> idToName;
+   readonly int idCounter;
+
+   public TimeseriesIndex()
+      : this(ImmutableDictionary<T, ImmutableList<KeyValuePair<DateTime, int>>>.Empty, ImmutableDictionary<int, TimeseriesKey<T>>.Empty, 0) { }
+
+   TimeseriesIndex(ImmutableDictionary<T, ImmutableList<KeyValuePair<DateTime, int>>> nameToId, ImmutableDictionary<int, TimeseriesKey<T>> idToName, int idCounter) {
+      this.nameToId = nameToId;
+      this.idToName = idToName;
+      this.idCounter = idCounter;
+   }
+
+   public int Count => idToName.Count;
+
+   public ImmutableDictionary<T, ImmutableList<KeyValuePair<DateTime, int>>> NameToId => nameToId;
+   public ImmutableDictionary<int, TimeseriesKey<T>> IdToName => idToName;
+
+   public bool ContainsId(int id) =>
+      idToName.ContainsKey(id);
+
+   public int GetNearest(TimeseriesKey<T> key) {
+      if (!nameToId.TryGetValue(key.Name, out var ids) || ids.Count == 0) {
+         throw new KeyNotFoundException();
+      }
+      int idx = ids.BinarySearch(new(key.Timestamp, -1), idTimeseriesComp);
+      if (idx < 0) {
+         idx = ~idx;
+         if (idx == 0) {
+            throw new KeyNotFoundException();
+         }
+         idx--;
+      }
+      return ids[idx].Value;
+   }
+
+   public bool TryGetNearest(TimeseriesKey<T> key, out int result) {
+      if (!nameToId.TryGetValue(key.Name, out var ids) || ids.Count == 0) {
+         result = -1;
+         return false;
+      }
+      int idx = ids.BinarySearch(new(key.Timestamp, -1), idTimeseriesComp);
+      if (idx < 0) {
+         idx = ~idx;
+         if (idx == 0) {
+            result = -1;
+            return false;
+         }
+         idx--;
+      }
+      result = ids[idx].Value;
+      return true;
+   }
+
+   public TimeseriesIndex<T> GetNearestOrAdd(TimeseriesKey<T> key, out int result) {
+      int idx;
+      if (!nameToId.TryGetValue(key.Name, out var ids) || ids.Count == 0) {
+         ids = ImmutableList<KeyValuePair<DateTime, int>>.Empty;
+         idx = 0;
+      } else {
+         idx = ids.BinarySearch(new(key.Timestamp, -1), idTimeseriesComp);
+         if (idx < 0) {
+            idx = ~idx;
+         }
+      }
+      if (idx == 0) {
+         result = idCounter + 1;
+         ids = ids.Insert(0, new(key.Timestamp, result));
+         return new(
+            nameToId.SetItem(key.Name, ids),
+            idToName.SetItem(result, key),
+            result);
+      }
+      result = ids[idx].Value;
+      return this;
+   }
+
+   public TimeseriesIndex<T> GetOrAdd(TimeseriesKey<T> key, out int result) {
+      int idx;
+      if (!nameToId.TryGetValue(key.Name, out var ids) || ids.Count == 0) {
+         ids = ImmutableList<KeyValuePair<DateTime, int>>.Empty;
+         idx = 0;
+      } else {
+         idx = ids.BinarySearch(new(key.Timestamp, -1), idTimeseriesComp);
+         if (idx < 0) {
+            idx = ~idx;
+         } else {
+            result = ids[idx].Value;
+            return this;
+         }
+      }
+      result = idCounter + 1;
+      ids = ids.Insert(idx, new(key.Timestamp, result));
+      return new(
+         nameToId.SetItem(key.Name, ids),
+         idToName.SetItem(result, key),
+         result);
+   }
+
+   public IList<KeyValuePair<DateTime, int>> GetIds(T name) =>
+      nameToId[name];
+
+   public Builder ToBuilder() =>
+      new(this);
+
+   public class Builder {
+      readonly ImmutableDictionary<T, ImmutableList<KeyValuePair<DateTime, int>>>.Builder nameToId;
+      readonly ImmutableDictionary<int, TimeseriesKey<T>>.Builder idToName;
+      int idCounter;
+
+      public Builder(TimeseriesIndex<T> source)
+         : this(source.nameToId, source.idToName, source.idCounter) { }
+
+      public Builder(ImmutableDictionary<T, ImmutableList<KeyValuePair<DateTime, int>>> nameToId, ImmutableDictionary<int, TimeseriesKey<T>> idToName, int idCounter) {
+         this.nameToId = nameToId.ToBuilder();
+         this.idToName = idToName.ToBuilder();
+         this.idCounter = idCounter;
+      }
+
+      public int GetNearest(TimeseriesKey<T> key) =>
+         getOrAdd(key, false, false);
+
+      public int GetNearestOrAdd(TimeseriesKey<T> key) =>
+         getOrAdd(key, false, true);
+
+      public int GetOrAdd(TimeseriesKey<T> key) =>
+         getOrAdd(key, true, true);
+
+      public int GetLatest(T key) =>
+         nameToId.TryGetValue(key, out var ids)
+            ? ids[^1].Value
+            : throw new KeyNotFoundException();
+
+      public bool TryGetLatest(T key, out int result) {
+         if (nameToId.TryGetValue(key, out var ids)) {
+            result = ids[^1].Value;
+            return true;
+         }
+         result = 0;
+         return false;
+      }
+
+      public ImmutableList<KeyValuePair<DateTime, int>> GetIds(T name) =>
+         nameToId.TryGetValue(name, out var ids)
+            ? ids
+            : ImmutableList<KeyValuePair<DateTime, int>>.Empty;
+
+      public TimeseriesIndex<T> ToImmutable() =>
+         new(nameToId.ToImmutable(), idToName.ToImmutable(), idCounter);
+
+      int getOrAdd(TimeseriesKey<T> key, bool requireExactTimestamp, bool canAdd) {
+         var ids = nameToId.GetOrAdd(key.Name, _ => ImmutableList<KeyValuePair<DateTime, int>>.Empty);
+         int idx = ids.BinarySearch(new(key.Timestamp, 0), idTimeseriesComp);
+         int id;
+         if (idx < 0) {
+            Log.Gen.info($"Adding key to timeseries index ({key.Timestamp}, {key.Name})");
+            // We didn't get an exact match. If requireExactTimestamp is false and a record with an
+            // earlier Timestamp exists, use that. Otherwise, add a new record with this Timestamp.
+            idx = ~idx;
+            if (requireExactTimestamp || idx == 0) {
+               if (!canAdd) {
+                  throw new KeyNotFoundException();
+               }
+               id = ++idCounter;
+               ids = ids.Insert(idx, new(key.Timestamp, id));
+               nameToId[key.Name] = ids;
+               idToName.Add(id, key);
+            } else {
+               if (idx == ids.Count) {
+                  --idx;
+               }
+               id = ids[idx].Value;
+            }
+         } else {
+            id = ids[idx].Value;
+         }
+         return id;
+      }
+   }
+}

src/DataStreamSuite/DataStream/BlueOrb/TimeseriesKey.cs

@@ -0,0 +1,22 @@
+namespace Bts.DataStream;
+
+using Spark.Core.Collections;
+
+using System;
+using System.Collections.Generic;
+
+public readonly record struct TimeseriesKey<T>(T Name, DateTime Timestamp) : IDualComparable<TimeseriesKey<T>> {
+   public int CompareTo(TimeseriesKey<T> other) {
+      var rv = Timestamp.CompareTo(other.Timestamp);
+      if (rv == 0) {
+         rv = Comparer<T>.Default.Compare(Name, other.Name);
+      }
+      return rv;
+   }
+
+   public static implicit operator (T Name, DateTime Timestamp)(TimeseriesKey<T> key) =>
+      (key.Name, key.Timestamp);
+
+   public static implicit operator TimeseriesKey<T>((T Name, DateTime Timestamp) tup) =>
+      new(tup.Name, tup.Timestamp);
+}

src/DataStreamSuite/DataStream/DataStream.csproj

@@ -17,12 +17,11 @@
   </ItemGroup>
 
   <ItemGroup>
-    <PackageReference Include="HtmlAgilityPack" Version="1.11.57" />
     <PackageReference Include="IronPython" Version="3.4.2" />
     <PackageReference Include="Microsoft.Data.Analysis" Version="0.21.1" />
     <PackageReference Include="MongoDB.Bson" Version="2.23.1" />
     <PackageReference Include="MongoDB.Driver" Version="2.23.1" />
-    <PackageReference Include="Parquet.Net" Version="5.0.1" />
+    <PackageReference Include="ParquetSharp" Version="20.0.0" />
     <PackageReference Include="System.Data.SQLite" Version="1.0.118" />
     <PackageReference Include="MathNet.Numerics" Version="5.0.0" />
   </ItemGroup>

src/DataStreamSuite/DataStream/DataStreamExtensions.cs

@@ -2,6 +2,7 @@ namespace Bts.DataStream;
 
 using BCG;
 
+using bts;
 using bts.api.core;
 
 using Bts.DataStreamCore;
@@ -15,27 +16,13 @@ using Spark.Core.Threading;
 using System;
 using System.Linq;
 using System.Reactive.Linq;
+using System.Threading;
+using System.Threading.Tasks;
 
 public static class DataStreamExtensions {
-   public static ViewStream AsViewStream(this IObservable<IDataPacket> source) =>
-      source as ViewStream ?? new ObservableViewStream(source);
-
    public static ViewStream AsViewStream<T>(this IObservable<IDataPacket<T>> source, string? name = null, ILogger? logger = null) where T : IBlueRow =>
       source as ViewStream ?? new ObservableViewStream(source.Select(dp => new DataPacket(dp.newRows, dp.addedRows, dp.deletedRows)), name, logger);
 
-   public static IObservable<DataPacket> AsObservable(this ViewStream source) {
-      return source as IObservable<DataPacket> ?? Observable.Create<DataPacket>(Subscribe);
-
-      IDisposable Subscribe(IObserver<DataPacket> observer) {
-         source.onChange += OnChange;
-         return Disposable.Create(() => source.onChange -= OnChange);
-
-         void OnChange(object? newValue) =>
-            observer.OnNext(newValue as DataPacket ?? throw new ArgumentException(
-               $"Expected DataPacket, got {newValue?.GetType()}", nameof(newValue)));
-      }
-   }
-
    // public static IObservable<IDataPacket<T>> SelectDataPacket<T>(this IObservable<IDataPacket> source, Func<IBlueRow, T> selector) where T : IBlueRow =>
    //    source.Select(dp => {
    //       dp.get_new_added_deleted(out var newRows, out var addedRows, out var deletedRows);
@@ -134,46 +121,6 @@ public static class DataStreamExtensions {
    public static ViewStream AsViewStream<T>(this IObservable<TableUpdate<T, int>> source, IBlueTableSchema schema) where T : IBlueRow =>
       source.AsDataPackets(schema).AsViewStream();
 
-   public static int ResolveIndices(this IBlueTableSchema schema, string name) {
-      var indices = ResolveIndices(schema, new[] { name });
-      return indices[0];
-   }
-
-   public static (int, int) ResolveIndices(this IBlueTableSchema schema, string name1, string name2) {
-      var indices = ResolveIndices(schema, new[] { name1, name2 });
-      return (indices[0], indices[1]);
-   }
-
-   public static (int, int, int) ResolveIndices(this IBlueTableSchema schema, string name1, string name2, string name3) {
-      var indices = ResolveIndices(schema, new[] { name1, name2, name3 });
-      return (indices[0], indices[1], indices[2]);
-   }
-
-   public static (int, int, int, int) ResolveIndices(this IBlueTableSchema schema, string name1, string name2, string name3, string name4) {
-      var indices = ResolveIndices(schema, new[] { name1, name2, name3, name4 });
-      return (indices[0], indices[1], indices[2], indices[3]);
-   }
-
-   public static int[] ResolveIndices(this IBlueTableSchema schema, string[] columnNames) {
-      var columns = schema.columnInfo;
-      var indices = new int[columnNames.Length];
-      int nResolved = 0;
-      for (int i = 0; i < columnNames.Length; i++) {
-         int origNResolved = nResolved;
-         for (int j = 0; j < columns.Length; j++) {
-            if (columns[j].colName == columnNames[i]) {
-               indices[i] = j;
-               nResolved++;
-               break;
-            }
-         }
-         if (nResolved == origNResolved) {
-            throw new ArgumentException($"No column named '{columnNames[i]}'", nameof(schema));
-         }
-      }
-      return indices;
-   }
-
    public static IObservable<IDataPacket> OnWorkScheduler(this IObservable<IDataPacket> source, IWorkScheduler scheduler) =>
       Observable.Create<IDataPacket>(o => {
          var inputGate = new AtomicGate();
@@ -198,7 +145,7 @@ public static class DataStreamExtensions {
                foreach (IBlueRow r in value.addedRows) {
                   bd.AddOrUpdate(r);
                }
-               pendingDpOut = bd.ToDataPacket(out lastTable);
+               pendingDpOut = bd.ToDataPacket(out lastTable, false);
             }
          }
 
@@ -214,5 +161,118 @@ public static class DataStreamExtensions {
          }
       });
 
+   public static IObservable<IDataPacket> NonIncrementalAsyncTransform(this IObservable<IDataPacket> source, Func<IBlueRowCollection, Task<IBlueRowCollection?>> transformAsync, IWorkScheduler workScheduler) {
+      const int STATE_IDLE = 0;
+      const int STATE_BUSY = 1;
+      const int STATE_DISPOSED = 2;
+      return Observable.Create<IDataPacket>(Subscribe);
+      IDisposable Subscribe(IObserver<IDataPacket> o) {
+         var syncRoot = new object();
+         IBlueRowCollection? lastTableIn = null;
+         BlueTableImmutable? lastTableOut = null;
+         int nConcurrentDoWorks = 0;
+         int state = STATE_IDLE;
+         IDisposable? dispose = null;
+         dispose = Disposable.Create(source.Subscribe(OnNext), workScheduler.Subscribe(DoWork));
+         return Disposable.Create(Dispose);
 
+         void Dispose() {
+            lock (syncRoot) {
+               switch (state) {
+                  case STATE_DISPOSED:
+                     return;
+                  case STATE_BUSY:
+                     state = STATE_DISPOSED;
+                     return;
+               }
+               state = STATE_DISPOSED;
+            }
+            Disposable.dispose(ref dispose);
+         }
+
+         void OnNext(IDataPacket value) =>
+            Volatile.Write(ref lastTableIn, value.newRows);
+
+         async void DoWork() {
+            lock (syncRoot) {
+               nConcurrentDoWorks++;
+               switch (state) {
+                  case STATE_DISPOSED:
+                  case STATE_BUSY:
+                     return;
+                  case STATE_IDLE:
+                     break;
+                  default:
+                     Log.assert(false, $"Unexpected state ({StateName(state)}) when DoWork called");
+                     return;
+               }
+               state = STATE_BUSY;
+            }
+            var dp = await PublishTable();
+            bool isDisposed = false;
+            lock (syncRoot) {
+               switch (state) {
+                  case STATE_DISPOSED:
+                     isDisposed = true;
+                     break;
+                  case STATE_BUSY:
+                     break;
+                  default:
+                     Log.assert(false, $"Unexpected state ({StateName(state)}) after building output table (async)");
+                     return;
+               }
+            }
+            if (isDisposed) {
+               Disposable.dispose(ref dispose);
+               return;
+            }
+            if (dp is not null) {
+               o.OnNext(dp);
+            }
+            lock (syncRoot) {
+               nConcurrentDoWorks = 0;
+               switch (state) {
+                  case STATE_DISPOSED:
+                     isDisposed = true;
+                     break;
+                  case STATE_BUSY:
+                     state = STATE_IDLE;
+                     break;
+                  default:
+                     Log.assert(false, $"Unexpected state ({StateName(state)}) after firing output table");
+                     return;
+               }
+            }
+            if (isDisposed) {
+               Disposable.dispose(ref dispose);
+            }
+
+            async Task<DataPacket?> PublishTable() {
+               var tableIn = Interlocked.Exchange(ref lastTableIn, null);
+               if (tableIn is null) {
+                  return null;
+               }
+               var tableOut = await transformAsync(tableIn);
+               if (tableOut is null) {
+                  return null;
+               }
+               lastTableOut ??= new(tableOut.schema);
+               var bd = lastTableOut.ToDataPacketBuilder();
+               bd.Clear();
+               foreach (IBlueRow r in tableOut) {
+                  bd.AddOrUpdate(r, BlueRowEqualityComparer.Instance);
+               }
+               return bd.ToDataPacket(out lastTableOut);
+            }
+         }
+
+         static string StateName(int state) =>
+            state switch {
+               STATE_IDLE => "IDLE",
+               STATE_BUSY => "BUSY",
+               STATE_DISPOSED => "DISPOSED",
+               _ => $"STATE_{state}",
+            };
+      }
+   }
 }

src/DataStreamSuite/DataStream/Misc/DSLStream.cs

@@ -3,24 +3,24 @@ namespace bts.BlueOrb {
    using BCG;
    using System.Collections.Generic;
    using Bts.DataStreamExpressions;
+   using Bts.DataStreamCore;
 
    // All this does is take in a dsl expression, parse it and fire the data that come out of running it.
    // So for example, if an expresison is passed 
    [DataStreamUserRespresentation("dsl")]
    public class DSLStream : ViewStream {
       private readonly string _dsl;
-      public DSLStream(string dsl) {
+      public DSLStream(string dsl, IDataStreamFactory streamFactory) {
          Log.Gen.info($"DSLStream.cons dsl: {dsl}");
          _dsl = dsl;
-         ViewStream rs = build_stream(dsl);
-         rs = rs ?? new LiteralStream("error", $"msg:invalid dsl \"{dsl}\"");
+         var rs = build_stream(streamFactory, dsl) ?? new LiteralStream("error", $"msg:invalid dsl");
          mySetRealSource(rs);
       }
 
-      static ViewStream? build_stream(string stream_expression) {
+      static ViewStream? build_stream(IDataStreamFactory streamFactory, string stream_expression) {
          try {
             var exp = DataStreamExpressionsInterpreter.Evaluate(stream_expression);
-            var rv = exp.getView(new Dictionary<string, StatementExpression>(0));
+            var rv = exp.getView(new Dictionary<string, StatementExpression>(0), streamFactory);
             return rv;
 
          } catch (Exception ex) {

src/DataStreamSuite/DataStream/Misc/TCPConnectionTestStream.cs

@@ -40,7 +40,7 @@ namespace BCG {
         protected override void myOnFirstSubscriber() {
             MasterWorker.getInstance(false).addWorker(this, _send_period);
             var server_rci = new RemoteConnectionInfo(_ip, _port);
-            bts.AsyncTCPConnector_APM.connect_to(server_rci, this);
+            bts.AsyncTCPConnector_AsyncWrapper.connect_to(server_rci, this);
         }
 
         protected override void myOnLastSubscriber() {
@@ -76,4 +76,4 @@ namespace BCG {
 
 
     }
-}
+}

src/DataStreamSuite/DataStream/Offloading/CloudStreamManager.cs

@@ -1,95 +0,0 @@
-using System;
-using System.Collections.Generic;
-using System.IO;
-using System.Text;
-using System.Runtime.InteropServices;
-using Bts.DataStreamExpressions;
-using BCG.Utilities.General;
-using bts;
-
-namespace BCG {
-   // listens to the requests posted on "cloud_streams" request table, and 
-   // if unfulfilled starts a ViewPubsliher
-   public class RemoteQueryManager : IWorker {
-      public class StreamPubUnit {
-         public string name;
-         public string stream_expression;
-         public ViewStream stream;
-         public bool persist;
-         public ViewPublisher view_publisher;
-
-         public StreamPubUnit(string name_, string stream_expression_, ViewStream stream_, bool persist_, ViewPublisher view_publisher_) {
-            name = name_;
-            stream_expression = stream_expression_;
-            stream = stream_;
-            persist = persist_;
-            view_publisher = view_publisher_;
-         }
-      }
-      static ViewStream? build_stream(string stream_expression)
-      {
-         try
-         {
-            var exp = DataStreamExpressionsInterpreter.Evaluate(stream_expression);
-            var rv = exp.getView(new Dictionary<string, StatementExpression>(0));
-            return rv;
-
-         } catch (Exception ex) {
-            bts.Log.assert(false, $"Could not build stream: {stream_expression} ex: {ex}");
-            return null;
-         }
-      }
-      private IDictionary<string, StreamPubUnit>
-         _stream_pub_map = new Dictionary<string, StreamPubUnit>();
-      private ViewStream _stream = new TableStream("cloud_streams");
-      private PubTableBroker _broker = new PubTableBroker();
-      private ManagedSubscriber _sub = ManagedSubscriber.instance;
-
-
-      public RemoteQueryManager() {
-         _stream.onChange += _on_str;
-         _sub.subscribe("RemoteRequest", (string tag, byte[] data) => {
-            RemoteRequest req = RemoteRequest.construct_from_bytes(data);
-            _spawn_remote_stream(req.published_table_name, req.stream_expression, false);
-         });
-      }
-      private void _on_str(object obj)
-      {
-         if (obj is not DataPacket dp) {
-            return;
-         }
-         foreach (IBlueRow r in dp.newRows)
-         {
-            string name = r.get<string>("name");
-            bool persist = r.get<bool>("persist");
-            string stream_exp = r.get<string>("stream_exp");
-            if (!_stream_pub_map.ContainsKey(name)) {
-               _spawn_remote_stream(name, stream_exp, persist);
-            }
-         }
-      }
-
-      private void _spawn_remote_stream(string name, string stream_expression, bool persist) {
-         bts.Log.Gen.info($"CloudStreamManager.assign name: {name} stream_expression: {stream_expression}");
-         ViewStream stream = build_stream(stream_expression);
-         _expose_to_ws(name, stream);
-         if (persist) {
-            string tempDir = Path.GetTempPath();
-        Directory.CreateDirectory(tempDir); // make sure it exists
-
-        stream = new PersistorStream(stream, name, tempDir, 10_000_000); // 10 MB
-         }
-         ViewPublisher view_pub = new ViewPublisher(name, _broker, stream);
-         StreamPubUnit u = new StreamPubUnit(name, stream_expression, stream, persist, view_pub);
-         _stream_pub_map.Add(name, u);
-      }
-
-      private void _expose_to_ws(string name, ViewStream stream) {
-         // TODO: john
-      }
-
-      public void doWork() {
-
-      }
-   }
-}

src/DataStreamSuite/DataStream/Offloading/RemoteQueryManager.cs

@@ -0,0 +1,168 @@
+using bts;
+
+using Bts.DataStreamCore;
+using Bts.DataStreamExpressions;
+using Bts.PubSub;
+
+using Microsoft.Extensions.Hosting;
+
+using Spark.Core.Disposables;
+
+using System;
+using System.Collections.Generic;
+using System.IO;
+using System.Linq;
+using System.Reactive.Linq;
+using System.Threading;
+using System.Threading.Tasks;
+
+namespace BCG {
+   // listens to the requests posted on "cloud_streams" request table, and 
+   // if unfulfilled starts a ViewPubsliher
+   public class RemoteQueryManager : IHostedService {
+      public class StreamPubUnit {
+         public string name;
+         public string stream_expression;
+         public ViewStream stream;
+         public bool persist;
+         public ViewPublisher view_publisher;
+
+         public StreamPubUnit(string name_, string stream_expression_, ViewStream stream_, bool persist_, ViewPublisher view_publisher_) {
+            name = name_;
+            stream_expression = stream_expression_;
+            stream = stream_;
+            persist = persist_;
+            view_publisher = view_publisher_;
+         }
+      }
+      static ViewStream? build_stream(IDataStreamFactory streamFactory, string stream_expression)
+      {
+         try
+         {
+            var exp = DataStreamExpressionsInterpreter.Evaluate(stream_expression);
+            var rv = exp.getView(new Dictionary<string, StatementExpression>(0), streamFactory);
+            return rv;
+
+         } catch (Exception ex) {
+            bts.Log.assert(false, $"Could not build stream: {stream_expression} ex: {ex}");
+            return null;
+         }
+      }
+      private IDictionary<string, StreamPubUnit>
+         _stream_pub_map = new Dictionary<string, StreamPubUnit>();
+      readonly IPubTableBroker _broker;
+      readonly IManagedSubscriber _sub;
+      readonly IDataStreamFactory streamFactory;
+      readonly ITableStreamFactory tableStreamFactory;
+      readonly object syncRoot = new();
+      IDisposable? cloudStreamsSub;
+      IDisposable? remoteRequestsSub;
+      State state;
+
+      public RemoteQueryManager(IManagedSubscriber managedSubscriber, IPubTableBroker pubTableBroker, ITableStreamFactory tableStreamFactory, IEnumerable<IDataStreamFactory> streamFactories) {
+         _sub = managedSubscriber;
+         _broker = pubTableBroker;
+         this.tableStreamFactory = tableStreamFactory;
+         var streamFactoryList = streamFactories.ToList();
+         streamFactory = streamFactoryList.Count == 1
+            ? streamFactoryList[0]
+            : new CompositeDataStreamFactory(streamFactoryList);
+      }
+
+      enum State {
+         Idle = 0,
+         Running = 1,
+         Disposed = 2,
+         WaitingToDispose = 3,
+      }
+
+      public Task StartAsync(CancellationToken cancellationToken) {
+         lock (syncRoot) {
+            switch (state) {
+               case State.Idle:
+                  break;
+               case State.WaitingToDispose:
+               case State.Disposed:
+                  throw new ObjectDisposedException(nameof(RemoteQueryManager));
+               default:
+                  return Task.CompletedTask;
+            }
+            state = State.Running;
+         }
+         if (cloudStreamsSub is not null) {
+            throw new InvalidOperationException("cloudStreamsSub already created");
+         }
+         if (remoteRequestsSub is not null) {
+            throw new InvalidOperationException("remoteRequestsSub already created");
+         }
+         var cloudStream = tableStreamFactory.Create("cloud_streams")
+            ?? throw new Exception("Failed to create TableStream 'cloud_stream'");
+         cloudStreamsSub = cloudStream.AsObservable().Subscribe(_on_str);
+         remoteRequestsSub = _sub.Observe("RemoteRequest").Subscribe(data => {
+            RemoteRequest req = RemoteRequest.construct_from_bytes(data);
+            _spawn_remote_stream(req.published_table_name, req.stream_expression, false);
+         });
+         lock (syncRoot) {
+            if (state == State.WaitingToDispose) {
+               state = State.Disposed;
+            }
+         }
+         if (state == State.Disposed) {
+            Disposable.dispose(ref cloudStreamsSub);
+            Disposable.dispose(ref remoteRequestsSub);
+         }
+         return Task.CompletedTask;
+      }
+
+      public Task StopAsync(CancellationToken cancellationToken) {
+         lock (syncRoot) {
+            switch (state) {
+               case State.Running:
+                  break;
+               default:
+                  return Task.CompletedTask;
+            }
+            state = State.Disposed;
+         }
+         Disposable.dispose(ref cloudStreamsSub);
+         Disposable.dispose(ref remoteRequestsSub);
+         return Task.CompletedTask;
+      }
+
+      private void _on_str(IDataPacket dp)
+      {
+         foreach (IBlueRow r in dp.newRows)
+         {
+            string name = r.get<string>("name");
+            bool persist = r.get<bool>("persist");
+            string stream_exp = r.get<string>("stream_exp");
+            if (!_stream_pub_map.ContainsKey(name)) {
+               _spawn_remote_stream(name, stream_exp, persist);
+            }
+         }
+      }
+
+      private void _spawn_remote_stream(string name, string stream_expression, bool persist) {
+         bts.Log.Gen.info($"CloudStreamManager.assign name: {name} stream_expression: {stream_expression}");
+         var stream = build_stream(streamFactory, stream_expression);
+         if (stream is null) {
+            Log.DataFlow.error($"Failed to spawn remote stream. name: {name}; expression: {stream_expression}");
+            return;
+         }
+         _expose_to_ws(name, stream);
+         if (persist) {
+            string tempDir = Path.GetTempPath();
+        Directory.CreateDirectory(tempDir); // make sure it exists
+
+        stream = new PersistorStream(stream, name, tempDir, 10_000_000); // 10 MB
+         }
+         ViewPublisher view_pub = new ViewPublisher(name, _broker, stream);
+         StreamPubUnit u = new StreamPubUnit(name, stream_expression, stream, persist, view_pub);
+         _stream_pub_map.Add(name, u);
+      }
+
+      private void _expose_to_ws(string name, ViewStream stream) {
+         // TODO: john
+      }
+   }
+}

src/DataStreamSuite/DataStream/Offloading/RemoteQueryStream.cs

@@ -2,17 +2,30 @@ using BCG;
 
 using bts;
 
+using Bts.DataStreamCore;
+
+using Spark.Core.Disposables;
+
 using System;
 
 namespace BCG {
-   // this is a stream so we can run it from the DSQA (since dsqa can only load streams)
-   [DataStreamUserRespresentation("remoteQuery")]
+   /// <summary>
+   /// This is a stream so we can run it from the DSQA (since dsqa can only load streams).
+   /// <para>
+   /// Note: It's hidden because this stream is now created by an IDataStreamFactory
+   /// (see RemoteQueryStreamFactory.cs for details)
+   /// </para>
+   /// </summary>
+   [DataStreamUserRespresentation(false)]
    public class RemoteQueryStream : ViewStream {
       private readonly string _name, _stream_exp;
-      private PublisherCore _publisher_core = new PublisherCore();
-      private ViewStream _source_str;
+      readonly ITableStreamFactory tableStreamFactory;
+      readonly IPublisherCore _publisher_core;
+      IDisposable? subscription;
 
-      public RemoteQueryStream(params string[] lines) {
+      public RemoteQueryStream(ITableStreamFactory tableStreamFactory, IPublisherCore publisherCore, params string[] lines) {
+         this.tableStreamFactory = tableStreamFactory;
+         _publisher_core = publisherCore;
          _stream_exp = get_expression(lines);
          _name = Guid.NewGuid().ToString();
          _name = $"remotereq_{DateTimeOffset.UtcNow.ToUnixTimeMilliseconds().ToString()}";
@@ -21,17 +34,17 @@ namespace BCG {
       }
 
       protected override void myOnFirstSubscriber() {
+         Log.isnull(subscription);
          RemoteRequest rem_req = new RemoteRequest(_stream_exp, _name);
          byte[] rem_req_bytes = rem_req.serialize();
          _publisher_core.publish("System", "RemoteRequest", rem_req_bytes);
-         _source_str = new TableStream(_name);
-         _source_str.onChange += new ChangeDelegate(_handle_data);
+         var sourceStream = tableStreamFactory.Create(_name);
+         sourceStream.onChange += _handle_data;
+         subscription = Disposable.Create(() => sourceStream.onChange -= _handle_data);
       }
 
-      protected override void myOnLastSubscriber() {
-         //maybe there should be a way to turn remote requests off ?
-         _source_str.onChange -= new ChangeDelegate(_handle_data);
-      }
+      protected override void myOnLastSubscriber() =>
+         Disposable.dispose(ref subscription);
 
       static string get_expression(params string[] lines) {
          for (int i = 0; i < lines.Length; i++) {

src/DataStreamSuite/DataStream/Primitives/Count/CountStream.cs

@@ -2,7 +2,7 @@ using System;
 
 namespace bts.BlueOrb {
    using BCG;
-   [DataStreamUserRespresentation("CountStream")]
+   [DataStreamUserRespresentation("oldCount")]
    public class CountStream : ViewStream {
       private DataPacket myLastFiredDataPacket;
       private IBlueTableSchema mySchema = new BlueTableSchema("count",
@@ -15,9 +15,6 @@ namespace bts.BlueOrb {
          myVS = vs;
       }
 
-      public CountStream(string tableName)
-         : this(new TableStream(tableName)) { }
-
       protected override void myOnFirstSubscriber() {
          myVS.onChange += new ChangeDelegate(myHandleChanges);
       }

src/DataStreamSuite/DataStream/Primitives/Filter/FilterStream.cs

@@ -1,4 +1,7 @@
+using bts;
+
 using Bts.DataStream;
+using Bts.DataStream.Primitives.Filter;
 
 using mbts.config;
 
@@ -21,6 +24,7 @@ namespace BCG.StreamWorld.Addition {
       public FilterStream(ViewStream vs, IStreamFilter filter) {
          myBaseStream = vs;
          myFilter = filter;
+         vs.ErrorChanged += OnError;
          if (filter is IDynamicStreamFilter)
             (filter as IDynamicStreamFilter).onFilterChange += new FilterChangeDelegate(myHandleFilterChange);
       }
@@ -36,31 +40,45 @@ namespace BCG.StreamWorld.Addition {
          myStateTable.clear();
       }
 
+      bool filterRow(IBlueRow r) {
+         try {
+            return myFilter.filter(r);
+         } catch (Exception ex) {
+            Log.Gen.error($"FilterStream delegate threw exception: {ex}");
+            OnError(new StreamFilterException(StreamId, StreamName, null, ex, r));
+         }
+         return false;
+      }
+
       private void myHandleChanges(object newValue) {
-         DataPacket dp = newValue as DataPacket;
-         if (myStateTable == null) {
-            bts.Log.assert(myBaseTable == null);
-            myStateTable = new BlueTable(dp.schema);
-            myBaseTable = new BlueTable(dp.schema);
-         }
-         BlueTable dels = new BlueTable(dp.schema),
-                     adds = new BlueTable(dp.schema);
-         foreach (IBlueRow r in dp.deletedRows) {
-            myBaseTable.deleteRow(r);
-            if (myStateTable.containsID((int)r["id"])) {
-               dels.addRow(r);
-               myStateTable.deleteRow(r);
+         try {
+            DataPacket dp = newValue as DataPacket;
+            if (myStateTable == null) {
+               bts.Log.assert(myBaseTable == null);
+               myStateTable = new BlueTable(dp.schema);
+               myBaseTable = new BlueTable(dp.schema);
             }
-         }
-         foreach (IBlueRow r in dp.addedRows) {
-            myBaseTable.addRow(r);
-            if (myFilter.filter(r)) {
-               adds.addRow(r);
-               myStateTable.addRow(r);
+            BlueTable dels = new BlueTable(dp.schema),
+                        adds = new BlueTable(dp.schema);
+            foreach (IBlueRow r in dp.deletedRows) {
+               myBaseTable.deleteRow(r);
+               if (myStateTable.containsID((int)r["id"])) {
+                  dels.addRow(r);
+                  myStateTable.deleteRow(r);
+               }
             }
+            foreach (IBlueRow r in dp.addedRows) {
+               myBaseTable.addRow(r);
+               if (filterRow(r)) {
+                  adds.addRow(r);
+                  myStateTable.addRow(r);
+               }
+            }
+            if ((dels.Count > 0 || adds.Count > 0) || (myLastValue == null))
+               myFireEvent(new DataPacket(myStateTable, adds, dels));
+         } catch (Exception ex) {
+            OnError(ex);
          }
-         if ((dels.Count > 0 || adds.Count > 0) || (myLastValue == null))
-            myFireEvent(new DataPacket(myStateTable, adds, dels));
       }
 
       public IStreamFilter filter {
@@ -69,9 +87,20 @@ namespace BCG.StreamWorld.Addition {
             if (myBaseTable != null) {
                IBlueRowCollection oldState = new BlueTable(myStateTable);
                myStateTable.clear();
-               foreach (IBlueRow r in myBaseTable)
-                  if (myFilter.filter(r))
-                     myStateTable.addRow(r);
+               // If the filter throws an exception, we can rethrow it as a StreamFilterException
+               // and include the 'currentRow' (for which the filter failed) to provide context.
+               IBlueRow? currentRow = null;
+               foreach (IBlueRow r in myBaseTable) {
+                  try {
+                     currentRow = r;
+                     if (filterRow(r)) {
+                        myStateTable.addRow(r);
+                     }
+                  } catch (Exception ex) {
+                     Log.Gen.error($"FilterStream delegate threw exception: {ex}");
+                     OnError(new StreamFilterException(StreamId, StreamName, null, ex, currentRow));
+                  }
+               }
                IBlueRowCollection dels, adds;
                BlueDataUtility.computeDifferences(myStateTable, oldState, out dels, out adds);
                myFireEvent(new DataPacket(myStateTable, adds, dels));
@@ -206,32 +235,36 @@ namespace BCG
          DataPacket retVal;
          DataPacket arrivedPacket = newValue as DataPacket;
          myLastReceivedRows = arrivedPacket.newRows;
-         if (myLastFiredRows != null)
-         {
-            retVal = new DataPacket(myLastFiredRows, new BlueTable(myLastFiredRows.schema),
-               new BlueTable(myLastFiredRows.schema));
-            foreach (IBlueRow r in arrivedPacket.deletedRows)
-               if (myLastFiredRows.containsID((int)r["id"]))
-               {
-                  myLastFiredRows.deleteRow(r);
-                  (retVal.deletedRows as BlueTable).addRow(r);
-               }
-            foreach (IBlueRow r in arrivedPacket.addedRows)
-               if (myFilter.filter(r))
-               {
-                  myLastFiredRows.addRow(r);
-                  (retVal.addedRows as BlueTable).addRow(r);
-               }
+         try {
+            if (myLastFiredRows != null)
+            {
+               retVal = new DataPacket(myLastFiredRows, new BlueTable(myLastFiredRows.schema),
+                  new BlueTable(myLastFiredRows.schema));
+               foreach (IBlueRow r in arrivedPacket.deletedRows)
+                  if (myLastFiredRows.containsID((int)r["id"]))
+                  {
+                     myLastFiredRows.deleteRow(r);
+                     (retVal.deletedRows as BlueTable).addRow(r);
+                  }
+               foreach (IBlueRow r in arrivedPacket.addedRows)
+                  if (filterRow(r))
+                  {
+                     myLastFiredRows.addRow(r);
+                     (retVal.addedRows as BlueTable).addRow(r);
+                  }
+            }
+            else
+            {
+               myLastFiredRows = new BlueTable(arrivedPacket.schema);
+               retVal = new DataPacket(myLastFiredRows, myLastFiredRows, new BlueTable(myLastFiredRows.schema));
+               foreach (IBlueRow r in arrivedPacket.newRows)
+                  if (filterRow(r))
+                     myLastFiredRows.addRow(r);
+            }
+            myFireEvent(retVal);
+         } catch (Exception ex) {
+            OnError(ex);
          }
-         else
-         {
-            myLastFiredRows = new BlueTable(arrivedPacket.schema);
-            retVal = new DataPacket(myLastFiredRows, myLastFiredRows, new BlueTable(myLastFiredRows.schema));
-            foreach (IBlueRow r in arrivedPacket.newRows)
-               if (myFilter.filter(r))
-                  myLastFiredRows.addRow(r);
-         }
-         myFireEvent(retVal);
       }
 
       public IStreamFilter filter
@@ -249,16 +282,20 @@ namespace BCG
                {
                    if (myLastReceivedRows != null)
                    {
+                     try {
                        BlueTable nRows = new BlueTable(myLastReceivedRows.schema);
                        foreach (IBlueRow r in myLastReceivedRows)
-                           if (myFilter.filter(r))
+                           if (filterRow(r))
                                nRows.addRow(r);
                        IBlueRowCollection dRows = new BlueTable(nRows.schema),
                           aRows = new BlueTable(nRows.schema);                      
                        BlueDataUtility.computeDifferences(nRows, myLastFiredRows, out dRows, out aRows);
                        myLastFiredRows = nRows;
                        myFireEvent(new DataPacket(nRows, aRows, dRows));
-                   }
+                     } catch (Exception ex) {
+                        OnError(ex);
+                     }
+                  }
                }
             }
          }
@@ -282,5 +319,15 @@ namespace BCG
       {
          filter = myFilter;
       }
+
+      bool filterRow(IBlueRow r) {
+         try {
+            return myFilter.filter(r);
+         } catch (Exception ex) {
+            Log.Gen.error($"FilterStream delegate threw exception: {ex}");
+            OnError(new StreamFilterException(StreamId, StreamName, null, ex, r));
+         }
+         return false;
+      }
    }
 }

src/DataStreamSuite/DataStream/Primitives/Filter/StreamFilterException.cs

@@ -0,0 +1,19 @@
+#nullable enable
+namespace Bts.DataStream.Primitives.Filter;
+
+using BCG;
+
+using System;
+
+public class StreamFilterException : Exception {
+   public StreamFilterException(Guid streamId, string streamName, string? message, Exception? innerException, IBlueRow? badRow = null)
+      : base(message, innerException) {
+      StreamId = streamId;
+      StreamName = streamName;
+      BadRow = badRow;
+   }
+
+   public Guid StreamId { get; }
+   public string StreamName { get; }
+   public IBlueRow? BadRow { get; }
+}

src/DataStreamSuite/DataStream/Primitives/Filter/TimeBasedFilter.cs

@@ -89,7 +89,7 @@ namespace BCG {
 
          if (now > myNextFilter) {
             // fire event
-            onFilterChange();
+            onFilterChange?.Invoke();
 
             myNextFilter = now.Add(_filter_period);
          }

src/DataStreamSuite/DataStream/Primitives/GroupBy/NonIncremental/NonIncGroupByStream.cs

@@ -96,6 +96,10 @@ namespace BCG {
          _timer.Restart();
          IBlueRowCollection total_state = _groupby_func.evaluate(grouped_rows);
          _timer.Stop();
+         if (total_state is null) {
+            Log.Gen.warn("NonIncGroupByStream _groupby_func.evaluate returned null (ignoring last update).");
+            return;
+         }
          var applyElapsedMs = _timer.ElapsedMilliseconds;
          _n_total_fires++;
          if (applyElapsedMs >= 50) {

src/DataStreamSuite/DataStream/Primitives/Join/ParallelJoinTest.cs

@@ -1,9 +1,7 @@
-using System;
+#nullable disable
+using System;
 using System.Collections;
 using System.Collections.Generic;
-using System.Text;
-using BCG.Utilities.General;
-using BCG.Utilities;
 
 namespace BCG
 {

src/DataStreamSuite/DataStream/Primitives/Literal/LiteralStream.cs

@@ -10,6 +10,8 @@ using BCG.Utilities;
 using bts;
 
 namespace BCG {
+   // Prefixing with "old" to avoid name conflict with the BlueOrb LiteralStream
+   [DataStreamUserRespresentation("oldLiteral")]
    public class LiteralStream : ViewStream {
 
       public static List<Tuple<ColumnInfo, object>> parse_columns(string key_value_pairs) {

src/DataStreamSuite/DataStream/Primitives/Map/MapStream.cs

@@ -1,6 +1,10 @@
+using bts;
+
 using Bts.DataStream;
+using Bts.DataStreamCore;
 
 using mbts.config;
+using mbts.utils;
 
 using System;
 
@@ -15,8 +19,8 @@ namespace BCG {
    /// </summary>
    [DataStreamUserRespresentation(false)]
    public class MapStream : MethodDrivenStream {
-      protected IStreamMapper myMapper;
-      protected IChangeableStreamMapper myChangeableMapper = null;
+      protected IStreamMapper? myMapper;
+      protected IChangeableStreamMapper? myChangeableMapper;
       private bool myUseNewMap = Configuration.get<DataStreamConfig>().UseNewMap;
 
       public class FuncWrapper : IStreamMapper {
@@ -71,7 +75,7 @@ namespace BCG {
 
 
       protected override void myHandleFirstCall() {
-
+         ArgumentNullException.ThrowIfNull(myMapper);
          IBlueTableSchema outputSchema = myMapper.getOutputSchema(myLastArrivedPacket.newRows.schema);
          myLastFiredPacket = new DataPacket();
          myLastFiredPacket.newRows = new BlueTable(outputSchema);
@@ -82,28 +86,34 @@ namespace BCG {
       }
 
       protected override void myHandleIncrementalAdd() {
-         //t.start ();
-         myLastFiredPacket.addedRows = new BlueTable(myLastFiredPacket.newRows.schema);
-         foreach (IBlueRow row in myLastArrivedPacket.addedRows) {
-            IBlueRow newRow = myMapper.map(row);
-            (myLastFiredPacket.newRows as BlueTable).addRow(newRow);
-            (myLastFiredPacket.addedRows as BlueTable).addRow(newRow);
+         if (myLastArrivedPacket.addedRows.Count == 0) {
+            Log.Gen.warn("MapStream.myHandleIncrementalAdd called with 0 addedRows");
+            return;
          }
-         //t.stop ();
-         //System.Diagnostics.Trace.WriteLine ("adding " + myLastFiredPacket.addedRows.Count.ToString () + " : " + t.lastIntervalTime);
-         //System.Diagnostics.Trace.WriteLine (" ");
+         ArgumentNullException.ThrowIfNull(myMapper);
+         var addedRows = new BlueTable(myLastFiredPacket.newRows.schema);
+         var newRows = ExceptionAssert.Cast(myLastFiredPacket.newRows).To<BlueTable>();
+         foreach (IBlueRow row in myLastArrivedPacket.addedRows) {
+            var newRow = myMapper.map(row);
+            newRows.addRow(newRow);
+            addedRows.addRow(newRow);
+         }
+         myLastFiredPacket.addedRows = addedRows;
       }
 
       protected override void myHandleIncrementalDelete() {
-         //t.start ();
-         myLastFiredPacket.deletedRows = new BlueTable(myLastFiredPacket.newRows.schema);
-         foreach (IBlueRow row in myLastArrivedPacket.deletedRows) {
-            IBlueRow delRow = (myLastFiredPacket.newRows as BlueTable)[(int)row["id"]];
-            (myLastFiredPacket.deletedRows as BlueTable).addRow(delRow);
-            (myLastFiredPacket.newRows as BlueTable).deleteRow(row);
+         if (myLastArrivedPacket.deletedRows.Count == 0) {
+            Log.Gen.warn("MapStream.myHandleIncrementalDelete called with 0 deletedRows");
+            return;
          }
-         //t.stop ();
-         //System.Diagnostics.Trace.WriteLine ("deleting " + myLastFiredPacket.deletedRows.Count.ToString () + " : " + t.lastIntervalTime);
+         var deletedRows = new BlueTable(myLastFiredPacket.newRows.schema);
+         var newRows = ExceptionAssert.Cast(myLastFiredPacket.newRows).To<BlueTable>();
+         foreach (IBlueRow row in myLastArrivedPacket.deletedRows) {
+            var delRow = newRows[row.Id()];
+            deletedRows.addRow(delRow);
+            newRows.deleteRow(row);
+         }
+         myLastFiredPacket.deletedRows = deletedRows;
       }
 
 

src/DataStreamSuite/DataStream/Primitives/Map/SuperMapStream.cs

@@ -1,11 +1,9 @@
+using Bts.DataStreamCore;
+
+using mbts.utils;
+
 using System;
 using System.Collections.Generic;
-using System.Text;
-
-using BCG.Utilities;
-using BCG.Utilities.General;
-
-using mbts;
 
 namespace BCG {
    /// <summary>
@@ -56,15 +54,15 @@ namespace BCG {
             return rows == null ? new IBlueRow[] { } : new List<IBlueRow>(rows).ToArray();
          }
 
-         public event MapperChangeDelegate onMapperChange;
-         private void _fake() { onMapperChange(); }
+         public event MapperChangeDelegate? onMapperChange;
       }
 
       //maps the id of the underlying stream to the
       //rows of the resulting stream
       private Dictionary<int, List<IBlueRow>> myMap = new Dictionary<int, List<IBlueRow>>();
       private int myID = 0;
-      private BlueTable myRows, myBaseRows;
+      private BlueTable? myRows;
+      private BlueTable? myBaseRows;
       private ViewStream myStream;
       private ISuperStreamMapper myMapper;
       public bool generate_ids = true;
@@ -92,49 +90,40 @@ namespace BCG {
          myRows = null;
          myBaseRows = null;
          myMap.Clear();
-         //myMapper.onMapperChange -= new MapperChangeDelegate(myHandleMapChange);
       }
 
       private void myHandleData(object newValue) {
-         DataPacket idp = newValue as DataPacket;
-         //HighPerformanceTimer timer = new HighPerformanceTimer();
-         //timer.start();
-         DataPacket dp = newValue as DataPacket;
-         myRows = myRows ?? new BlueTable(myMapper.getOutputSchema(dp.schema));
-         myBaseRows = myBaseRows ?? new BlueTable(dp.schema);
+         var dp = ExceptionAssert.Cast(newValue).To<DataPacket>();
+         myRows ??= new BlueTable(myMapper.getOutputSchema(dp.schema));
+         myBaseRows ??= new BlueTable(dp.schema);
          BlueDataUtility.absorbChanges(myBaseRows, dp.deletedRows, dp.addedRows);
-         BlueTable dels = new BlueTable(myRows.schema),
-                     adds = new BlueTable(myRows.schema);
+         var dels = new BlueTable(myRows.schema);
+         var adds = new BlueTable(myRows.schema);
 
          foreach (IBlueRow r in dp.deletedRows) {
-            List<IBlueRow> mappedRows = myMap[(int)r["id"]];
+            var mappedRows = myMap[r.Id()];
             foreach (IBlueRow mr in mappedRows) {
                dels.addRow(mr);
                myRows.deleteRow(mr);
             }
-            myMap.Remove((int)r["id"]);
+            myMap.Remove(r.Id());
          }
          foreach (IBlueRow r in dp.addedRows) {
             IBlueRow[] mappedRows = myMapper.map(r);
             foreach (IBlueRow mr in mappedRows) {
                IBlueRow idChangedMapRow = generate_ids ? BlueDataUtility.getNewReplacedRow(mr, myID++, "id") : mr;
-               if (!myMap.ContainsKey((int)r["id"])) {
-                  myMap.Add((int)r["id"], new List<IBlueRow>());
-               }
-               myMap[(int)r["id"]].Add(idChangedMapRow);
+               myMap.GetOrAdd(r.Id(), _ => new()).Add(idChangedMapRow);
                myRows.addRow(idChangedMapRow);
                adds.addRow(idChangedMapRow);
             }
          }
-         //timer.stop();
-         // if (timer.lastIntervalTime > 20)
-         //    {}//mbts.Logger.log(bts.log.Level.Info, bts.log.Subsystem.gen, String.Format("SuperMapper:myHandlerData for mapper {0} took {1} msecs", myMapper.GetType().ToString(), timer.lastIntervalTime));
          myFireEvent(new DataPacket(myRows, adds, dels));
       }
 
       private void myHandleMapChange() {
-         if (myBaseRows != null)
+         if (myBaseRows is not null) {
             myHandleData(new DataPacket(myBaseRows, new BlueTable(myBaseRows), new BlueTable(myBaseRows)));
+         }
       }
    }
 }

src/DataStreamSuite/DataStream/Primitives/Misc/PubSubTestStream.cs

@@ -70,7 +70,8 @@ namespace bts
         private readonly int _msg_size;
         private readonly TimeSpan _publish_period;
         private DateTime _last_publish_time = DateTime.MinValue;
-        private PublisherCore _pub_core = new PublisherCore();
+        private IPublisherCore _pub_core;
+        readonly ISubscriberCore subscriberCore;
         private BlueTable _table;
         private int _n_pubs = 0;
         private readonly int _max_pubs;
@@ -82,8 +83,10 @@ namespace bts
         private readonly int _payload_sz;
         private const byte _payload_byte = (byte)'A';
         private readonly string _tag_name;
-        public PubSubTestingStream(int publish_period_msecs, int max_pubs, int payload_size, string tag_name, ViewStreamDatabase vsd)
+        public PubSubTestingStream(int publish_period_msecs, int max_pubs, int payload_size, string tag_name, IPublisherCore publisherCore, ISubscriberCore subscriberCore, ViewStreamDatabase vsd)
         {
+            _pub_core = publisherCore;
+            this.subscriberCore = subscriberCore;
             _vsd = vsd;
             _payload_sz = payload_size;
             var schema = BlueDataUtility.form_schema("PubSubTest",
@@ -103,9 +106,6 @@ namespace bts
             _tag_name = tag_name;
         }
 
-        public PubSubTestingStream(int publish_period_seconds, int max_pubs, int msg_size, string tag_name)
-        : this(publish_period_seconds, max_pubs, msg_size, tag_name, ViewStreamDatabase.instance) { }
-
         protected override void myOnFirstSubscriber()
         {
             _vsd.addWorker(this);
@@ -132,33 +132,31 @@ namespace bts
 
         private void _subscribe_thread_func()
         {
+           var sub = subscriberCore.Updates().Subscribe(onUpdate);
             // subscription is done in another thread
-            SubscriberCore sub_core = new SubscriberCore();
-            sub_core.subscribe(_tag_name);
+            subscriberCore.subscribe(_tag_name);
             while (_continue_sub_thread)
             {
                 // TODO recv_id
-                var updates = sub_core.get_updates(TimeSpan.FromMilliseconds(250));
-                if (updates == null || updates.Count() == 0)
-                    continue;
-                _log_updates(updates);
-                _n_updates += updates.Count();
-                if (updates.Count() > 0)
-                {
-                    var server_msg = updates.Last();
-                    if (server_msg.data != null && server_msg.data.Length > 0)
-                    {
-                        _last_recvd_msg = PayloadMessage.deserialize(server_msg.data);
-                        _last_recvd_msg.travel_epoch_ms = epoch_now();
-                    }
-                    else
-                    {
-                        // why are we getting empty or null data ? (server is sending data regardless?)
-                    }
-                    _fire_dirty = true;
-                }
+                subscriberCore.get_updates(TimeSpan.FromMilliseconds(250));
             }
-        }
+            sub.Dispose();
+
+         void onUpdate(IEnumerable<PubSubServerMessage> updates) {
+            _log_updates(updates);
+            _n_updates += updates.Count();
+            if (updates.Count() > 0) {
+               var server_msg = updates.Last();
+               if (server_msg.data != null && server_msg.data.Length > 0) {
+                  _last_recvd_msg = PayloadMessage.deserialize(server_msg.data);
+                  _last_recvd_msg.travel_epoch_ms = epoch_now();
+               } else {
+                  // why are we getting empty or null data ? (server is sending data regardless?)
+               }
+               _fire_dirty = true;
+            }
+         }
+      }
 
         private void _publish_next_packet()
         {
@@ -224,4 +222,4 @@ namespace bts
         }
 
     }
-}
+}

src/DataStreamSuite/DataStream/Primitives/Misc/PubsubEchoTestStream.cs

@@ -8,10 +8,11 @@ using System.IO;
 using System.Linq;
 using System.Runtime.CompilerServices;
 using bts.api;
+using Bts.PubSub.Routing;
 
 namespace bts
 {
-   [DataStreamUserRespresentation("pss_echo")]
+   [DataStreamUserRespresentation(false)]
    public class PubsubEchoTestStream : ViewStream, IWorker
    {
       public class PayloadMessage
@@ -76,7 +77,7 @@ namespace bts
       private readonly int _msg_size;
       private readonly TimeSpan _publish_period;
       private DateTime _last_publish_time = DateTime.MinValue;
-      private PublisherCore _pub_core;
+      private IPublisherCore _pub_core;
       private ISubscriberCore _sub_core;
       private BlueTable _table;
       private int _n_pubs = 0;
@@ -95,10 +96,21 @@ namespace bts
       };
       IOState _state = IOState.SEND;
 
-      public PubsubEchoTestStream(string server_url, TimeSpan publish_period, int payload_size, string tag_name, ViewStreamDatabase vsd)
-      {
-         var route = new PubSubRoute(server_url, ".*");
-         _router = new PubSubRouter(route);
+      public PubsubEchoTestStream(
+         string server_url,
+         TimeSpan publish_period,
+         int payload_size,
+         string tag_name,
+         IPublisherCore publisherCore,
+         ISubscriberCore subscriberCore,
+         ViewStreamDatabase vsd
+      ) {
+         _router = new PubSubRouter(new PubSubRouterConfig() {
+            PublisherRoutes = new() { new(server_url, ".*") },
+            SubscriberRoutes = new() { new(server_url, ".*") },
+         });
+         _pub_core = publisherCore;
+         _sub_core = subscriberCore;
          _vsd = vsd;
          _payload_sz = payload_size;
          var schema = BlueDataUtility.form_schema("PubSubTest",
@@ -114,13 +126,8 @@ namespace bts
          _tag_name = tag_name;
       }
 
-      public PubsubEchoTestStream(string server_url, int publish_period_msecs, int payload_size, string tag_name)
-      : this(server_url, TimeSpan.FromMilliseconds(publish_period_msecs), payload_size, tag_name, ViewStreamDatabase.instance) { }
-
       protected override void myOnFirstSubscriber()
       {
-         _pub_core = new PublisherCore(_router);
-         _sub_core = new SubscriberCore(_router);
          _sub_core.subscribe(_tag_name);
          _vsd.addWorker(this);
 
@@ -220,4 +227,4 @@ namespace bts
       }
 
    }
-}
+}

src/DataStreamSuite/DataStream/Primitives/Misc/SnapshotReplayStream.cs

@@ -66,7 +66,7 @@ namespace bts {
          if (_in_rows.Count > 0) {
             // fire row
             IBlueRow curr_row = _in_rows.ElementAt(_in_row_ptr++);
-            _out_table.add_row(curr_row);
+            _out_table = _out_table.Add(curr_row);
             DataPacket dp = BlueDataUtility.build_dp_inc(_out_table,
                myLastValue == null ? null : myLastValue as DataPacket, false);
             myFireEvent(dp);
@@ -74,4 +74,4 @@ namespace bts {
       }
    }
 
-}
+}

src/DataStreamSuite/DataStream/Primitives/Misc/TestStream.cs

@@ -95,7 +95,7 @@ namespace BCG {
             sub_msg.serialize_into_str(str);
             str.Position = 0;
             bts.SubscribeMsg sub_msg_ = new bts.SubscribeMsg();
-            sub_msg_.consruct_deserialize(str);
+            sub_msg_.construct_deserialize(str);
             bool res = sub_msg.Equals(sub_msg_);
             return new TestResult {
                 test_name = "SubscribeMsg",
@@ -117,7 +117,7 @@ namespace BCG {
             pub_msg.serialize_into_str(str);
             str.Position = 0;
             var pub_msg_ = new bts.PublishMsg();
-            pub_msg_.consruct_deserialize(str);
+            pub_msg_.construct_deserialize(str);
             bool res = pub_msg.Equals(pub_msg_);
             return new TestResult {
                 test_name = "PublishMsg",
@@ -135,7 +135,7 @@ namespace BCG {
             msg.serialize_into_str(str);
             str.Position = 0;
             var msg_ = new bts.PubSubServerMessage();
-            msg_.consruct_deserialize(str);
+            msg_.construct_deserialize(str);
             bool res = msg.Equals(msg_);
             return new TestResult {
                 test_name = "PubSubServerMessage",
@@ -156,7 +156,7 @@ namespace BCG {
             msg.serialize_into_str(str);
             str.Position = 0;
             var msg_ = new PubSubClientMessage();
-            msg_.consruct_deserialize(str);
+            msg_.construct_deserialize(str);
             var res = msg.Equals(msg_);
             return new TestResult {
                 test_name = "PubSubClientMessage",
@@ -617,14 +617,15 @@ namespace BCG {
 
 
         private static TestResult _run_single_tag_data_test_clean() {
+            // var tagHandlerFactory
             // create message
-            var test_name = MethodBase.GetCurrentMethod().Name;
+            var test_name = MethodBase.GetCurrentMethod()?.Name;
             var schema = create_schema_with_all_types();
             var tag = "DS_" + schema.tableName;
             BlueTable table = new BlueTable(schema);
             IStreamIOMsgSerializer serializer = new StreamIOSerializationController();
             IStreamMsgDeserilizer deser = new StreamIODeserializationController();
-            bts.SingleTagData st_data = new bts.SingleTagData(tag, DSTagManager.DataStreamTagType);
+            bts.SingleTagData st_data = new bts.SingleTagData(tag, DSTagManager.DataStreamTagType, new PubsubServerStreamTagHandler());
             // create with 100 rows
             int n_rows = 100;
             bool is_succ = true;

src/DataStreamSuite/DataStream/Primitives/Misc/WebScrapingStream.cs

@@ -1,121 +1,121 @@
-using System;
-using System.Collections.Generic;
-using System.IO;
-using System.Security.Cryptography.X509Certificates;
+// using System;
+// using System.Collections.Generic;
+// using System.IO;
+// using System.Security.Cryptography.X509Certificates;
 
-using BCG;
-using BCG.Utilities.General;
+// using BCG;
+// using BCG.Utilities.General;
 
-using HtmlAgilityPack;
+// using HtmlAgilityPack;
 
-namespace BCG {
-    class Program {
-        static void Main() {
-            string url = "http://quotes.toscrape.com";
-            List<string> quotes = ScrapeQuotes(url);
+// namespace BCG {
+//     class Program {
+//       //   static void Main() {
+//       //       string url = "http://quotes.toscrape.com";
+//       //       List<string> quotes = ScrapeQuotes(url);
 
-            Console.WriteLine("Quotes from http://quotes.toscrape.com:");
-            foreach (var quote in quotes) {
-                Console.WriteLine(quote);
-            }
-        }
+//       //       Console.WriteLine("Quotes from http://quotes.toscrape.com:");
+//       //       foreach (var quote in quotes) {
+//       //           Console.WriteLine(quote);
+//       //       }
+//       //   }
 
-        static List<string> ScrapeQuotes(string url) {
-            List<string> quotes = new List<string>();
+//         static List<string> ScrapeQuotes(string url) {
+//             List<string> quotes = new List<string>();
 
-            // Create a new HtmlWeb instance
-            HtmlWeb web = new HtmlWeb();
+//             // Create a new HtmlWeb instance
+//             HtmlWeb web = new HtmlWeb();
 
-            try {
-                // Load the HTML document from the specified URL
-                HtmlDocument document = web.Load(url);
+//             try {
+//                 // Load the HTML document from the specified URL
+//                 HtmlDocument document = web.Load(url);
 
-                // Extract quotes from the HTML document
-                var quoteNodes = document.DocumentNode.SelectNodes("//span[@class='text']");
-                if (quoteNodes != null) {
-                    foreach (var quoteNode in quoteNodes) {
-                        string quote = quoteNode.InnerText.Trim();
-                        quotes.Add(quote);
-                    }
-                }
-            } catch (Exception ex) {
-                Console.WriteLine($"Error: {ex.Message}");
-            }
+//                 // Extract quotes from the HTML document
+//                 var quoteNodes = document.DocumentNode.SelectNodes("//span[@class='text']");
+//                 if (quoteNodes != null) {
+//                     foreach (var quoteNode in quoteNodes) {
+//                         string quote = quoteNode.InnerText.Trim();
+//                         quotes.Add(quote);
+//                     }
+//                 }
+//             } catch (Exception ex) {
+//                 Console.WriteLine($"Error: {ex.Message}");
+//             }
 
-            return quotes;
-        }
-    }
+//             return quotes;
+//         }
+//     }
 
-    public class WebScapperStream : ViewStream, IWorker {
-        private readonly string _url;
-        private readonly string _xpath;
-        private readonly ViewStreamDatabase _vsd;
-        private IDictionary<string, int> _line_id_map = new Dictionary<string, int>();
-        private readonly IBlueTableSchema _schema;
+//     public class WebScapperStream : ViewStream, IWorker {
+//         private readonly string _url;
+//         private readonly string _xpath;
+//         private readonly ViewStreamDatabase _vsd;
+//         private IDictionary<string, int> _line_id_map = new Dictionary<string, int>();
+//         private readonly IBlueTableSchema _schema;
 
-        public WebScapperStream()
-        :
-        this("http://quotes.toscrape.com", "//span[@class='text']", ViewStreamDatabase.instance) { }
-        public WebScapperStream(string url, string xpath)
-        : this(url, xpath, ViewStreamDatabase.instance) { }
-        public WebScapperStream(string url, string xpath, ViewStreamDatabase vsd) {
-            bts.Log.Gen.info($"WebScapperStream cons url: {url} xpath: {xpath}");
-            _schema = BlueDataUtility.form_schema($"{url}_{xpath}",
-                                                Tuple.Create("id", TypeCode.Int32),
-                                                Tuple.Create("url", TypeCode.UInt32),
-                                                Tuple.Create("line", TypeCode.String));
-            _url = url;
-            _xpath = xpath;
-            _vsd = vsd;
-        }
+//         public WebScapperStream()
+//         :
+//         this("http://quotes.toscrape.com", "//span[@class='text']", ViewStreamDatabase.instance) { }
+//         public WebScapperStream(string url, string xpath)
+//         : this(url, xpath, ViewStreamDatabase.instance) { }
+//         public WebScapperStream(string url, string xpath, ViewStreamDatabase vsd) {
+//             bts.Log.Gen.info($"WebScapperStream cons url: {url} xpath: {xpath}");
+//             _schema = BlueDataUtility.form_schema($"{url}_{xpath}",
+//                                                 Tuple.Create("id", TypeCode.Int32),
+//                                                 Tuple.Create("url", TypeCode.UInt32),
+//                                                 Tuple.Create("line", TypeCode.String));
+//             _url = url;
+//             _xpath = xpath;
+//             _vsd = vsd;
+//         }
 
-        protected override void myOnFirstSubscriber() {
-            MasterWorker.getInstance(false).addWorker(this, TimeSpan.FromSeconds(1));
-        }
+//         protected override void myOnFirstSubscriber() {
+//             MasterWorker.getInstance(false).addWorker(this, TimeSpan.FromSeconds(1));
+//         }
 
-        protected override void myOnLastSubscriber() {
-            MasterWorker.getInstance(false).removeWorker(this);
-        }
+//         protected override void myOnLastSubscriber() {
+//             MasterWorker.getInstance(false).removeWorker(this);
+//         }
 
-        public void doWork() {
-            var lines = scrape_lines(_url, _xpath);
-            Func<string, int> get_id = (line) => {
-                int rv = 0;
-                if (!_line_id_map.TryGetValue(line, out rv))
-                    _line_id_map.Add(line, _line_id_map.Count);
-                return rv;
-            };
-            BlueTable nrows = new BlueTable(_schema);
-            foreach (var l in lines) {
-                bts.Log.Gen.info($"line: {l}");
-                var id = get_id(l);
-                BlueRow nr = new BlueRow(_schema, get_id(l), _url, l);
-                nrows.addRow(nr);
-            }
-            var dp = BlueDataUtility.build_dp_inc(nrows, myLastValue as DataPacket, true);
-            myFireEvent(dp);
-        }
+//         public void doWork() {
+//             var lines = scrape_lines(_url, _xpath);
+//             Func<string, int> get_id = (line) => {
+//                 int rv = 0;
+//                 if (!_line_id_map.TryGetValue(line, out rv))
+//                     _line_id_map.Add(line, _line_id_map.Count);
+//                 return rv;
+//             };
+//             BlueTable nrows = new BlueTable(_schema);
+//             foreach (var l in lines) {
+//                 bts.Log.Gen.info($"line: {l}");
+//                 var id = get_id(l);
+//                 BlueRow nr = new BlueRow(_schema, get_id(l), _url, l);
+//                 nrows.addRow(nr);
+//             }
+//             var dp = BlueDataUtility.build_dp_inc(nrows, myLastValue as DataPacket, true);
+//             myFireEvent(dp);
+//         }
 
-        static List<string> scrape_lines(string url, string xpath) {
-            List<string> quotes = new List<string>();
-            // Create a new HtmlWeb instance
-            HtmlWeb web = new HtmlWeb();
-            try {
-                // Load the HTML document from the specified URL
-                HtmlDocument document = web.Load(url);
-                // Extract quotes from the HTML document
-                var quote_nodes = document.DocumentNode.SelectNodes(xpath);
-                if (quote_nodes != null) {
-                    foreach (var qn in quote_nodes) {
-                        string quote = qn.InnerText.Trim();
-                        quotes.Add(quote);
-                    }
-                }
-            } catch (Exception ex) {
-                Console.WriteLine($"Error: {ex.Message}");
-            }
+//         static List<string> scrape_lines(string url, string xpath) {
+//             List<string> quotes = new List<string>();
+//             // Create a new HtmlWeb instance
+//             HtmlWeb web = new HtmlWeb();
+//             try {
+//                 // Load the HTML document from the specified URL
+//                 HtmlDocument document = web.Load(url);
+//                 // Extract quotes from the HTML document
+//                 var quote_nodes = document.DocumentNode.SelectNodes(xpath);
+//                 if (quote_nodes != null) {
+//                     foreach (var qn in quote_nodes) {
+//                         string quote = qn.InnerText.Trim();
+//                         quotes.Add(quote);
+//                     }
+//                 }
+//             } catch (Exception ex) {
+//                 Console.WriteLine($"Error: {ex.Message}");
+//             }
 
-            return quotes;
-        }
-    }
-}
+//             return quotes;
+//         }
+//     }
+// }

src/DataStreamSuite/DataStream/Primitives/MongoDB/MongoSchemaAdaptor.cs

@@ -2,6 +2,8 @@ namespace Bts.DataStream;
 
 using bts;
 
+using Bts.DataStreamCore;
+
 using MongoDB.Bson;
 
 using System;

src/DataStreamSuite/DataStream/Primitives/SQL/SQLite/SQLiteEnumerator.cs

@@ -0,0 +1,54 @@
+namespace Bts.DataStream;
+
+using BCG;
+
+using System;
+using System.Collections.Generic;
+using System.Data;
+using System.Data.SQLite;
+using System.Linq;
+using System.Threading;
+
+public class SQLiteEnumerable : IAsyncEnumerable<IBlueRowCollection> {
+   readonly string location;
+   readonly string statement;
+   readonly int pageSize;
+
+   public SQLiteEnumerable(string location, string statement, int pageSize = 65536) {
+      this.location = location;
+      this.statement = statement;
+      this.pageSize = pageSize;
+   }
+
+   public async IAsyncEnumerator<IBlueRowCollection> GetAsyncEnumerator(CancellationToken cancellationToken = default) {
+      var dataTable = new DataTable();
+      await using var conn = new SQLiteConnection($"Data Source={location};Version=3");
+      await conn.OpenAsync(cancellationToken);
+      await using var cmd = new SQLiteCommand(statement, conn);
+      await using var reader = cmd.ExecuteReader();
+      var schema = await reader.GetColumnSchemaAsync(cancellationToken);
+      var columns = schema.OrderBy(c => c.ColumnOrdinal ?? int.MaxValue)
+         .Select((c, i) => new ColumnInfo(c.ColumnName, c.ColumnOrdinal ?? i, c.DataType))
+         .Append(new ColumnInfo("id", schema.Count, typeof(int)))
+         .ToArray();
+      var blueSchema = new BlueTableSchema("anon", columns);
+      BlueTable? table = null;
+      int rowId = 1;
+      while (await reader.ReadAsync(cancellationToken)) {
+         table ??= new BlueTable(blueSchema);
+         var rowVals = new object[blueSchema.columnInfo.Length];
+         if (reader.GetValues(rowVals) != rowVals.Length - 1) {
+            throw new InvalidOperationException("Sqlite reader value count for single row not equal to column count");
+         }
+         rowVals[^1] = rowId++;
+         table.addRow(new BlueRow(blueSchema, rowVals));
+         if (table.Count == pageSize) {
+            yield return table;
+            table = null;
+         }
+      }
+      if (table is not null) {
+         yield return table;
+      }
+   }
+}

src/DataStreamSuite/DataStream/Primitives/SideEffectStreams/CSVSaverStream.cs

@@ -5,8 +5,6 @@ using System.IO;
 using BCG.Utilities;
 using System.Diagnostics;
 using Microsoft.Data.Analysis;
-using Parquet;
-using Parquet.Data;
 
 
 namespace BCG {

src/DataStreamSuite/DataStream/Primitives/SideEffectStreams/SchemaGuesserStream.cs

@@ -1,6 +1,7 @@
 using BCG;
 
 namespace bts {
+    [DataStreamUserRespresentation("oldSchemaGuesser")]
     public class SchemaGuesserStream : ViewStream {
 
         private IBlueRowCollection _rows = null;
@@ -30,4 +31,4 @@ namespace bts {
 
 
     }
-}
+}

src/DataStreamSuite/DataStream/Primitives/Time/DateTimeExpansionStream.cs

@@ -4,9 +4,9 @@ namespace bts.SilverPine {
    using BCG;
 
    using System;
-   [DataStreamUserRespresentation("DateTimeExpanderStream")]
-   public class DateTimeExpanstionStream : ViewStream {
-      public DateTimeExpanstionStream(ViewStream parent_stream, string inp_col) {
+   [DataStreamUserRespresentation("DateTimeExpansionStream")]
+   public class DateTimeExpansionStream : ViewStream {
+      public DateTimeExpansionStream(ViewStream parent_stream, string inp_col) {
          Func<IBlueTableSchema, IBlueTableSchema> get_schema = (s) => {
             IBlueTableSchema rv = s.add_column(System.TypeCode.DateTime, $"{inp_col}_date");
             rv = rv.add_column(System.TypeCode.String, $"{inp_col}_day");
@@ -35,4 +35,4 @@ namespace bts.SilverPine {
          mySetRealSource(rs);
       }
    }
-}
+}

src/DataStreamSuite/DataStream/Primitives/Time/MillisecondStream.cs

@@ -44,7 +44,7 @@ namespace BCG {
             _last_fire_time = DateTime.Now;
             IBlueRow nr = new BlueRow(_schema, _id++, _last_fire_time,
             _last_fire_time.ToString("yyyy-MM-dd HH:mm:ss.fff"));
-            _table.add_row(nr);
+            _table = _table.Add(nr);
             var dp = BlueDataUtility.build_dp_inc(_table, myLastValue as DataPacket);
             myFireEvent(dp);
          }

src/DataStreamSuite/DataStream/Publish/CircularPublisher.cs

@@ -1,3 +1,5 @@
+using Bts.DataStreamCore;
+
 using System;
 
 namespace BCG {
@@ -7,14 +9,15 @@ namespace BCG {
    /// </summary>
 	public class CircularPublisher {
       private CircularRowCollection myBuffer;
-      private PubTableBroker myBroker;
+      private readonly IPubTableBroker myBroker;
       private int myBufferSize;
-      public CircularPublisher(int bufferSize) {
+      public CircularPublisher(int bufferSize, IPubTableBroker broker) {
          myBufferSize = bufferSize;
+         myBroker = broker;
       }
 
-      public CircularPublisher(int bufferSize, IBlueRowCollection rows)
-         : this(bufferSize) {
+      public CircularPublisher(int bufferSize, IPubTableBroker broker, IBlueRowCollection rows)
+         : this(bufferSize, broker) {
          publishRows(rows);
       }
 
@@ -23,7 +26,6 @@ namespace BCG {
             myBuffer = new CircularRowCollection(
                new BlueTableSchema(nr.schema.tableName, nr.schema.columnInfo),
                myBufferSize);
-            myBroker = new PubTableBroker();
             myBroker.create_table(myBuffer);
          }
          IBlueRow br = myBuffer.addRowWithReturn(nr);
@@ -38,7 +40,6 @@ namespace BCG {
                new BlueTableSchema(BlueDataUtility.extractRow(nrs, 0).schema.tableName,
                BlueDataUtility.extractRow(nrs, 0).schema.columnInfo),
                myBufferSize);
-            myBroker = new PubTableBroker();
             myBroker.create_table(myBuffer);
          }
          BlueTable deletes = new BlueTable(myBuffer.schema);

src/DataStreamSuite/DataStream/Publish/PublisherStream.cs

@@ -1,15 +1,19 @@
 namespace BCG {
+   using Bts.DataStreamCore;
+
    // this for testing purposes mainly, its wraps a ViewPublisher around a stream, 
    // and hence epublishes it out to the PS
+   [DataStreamUserRespresentation(false)]
    public class PublisherStream : ViewStream {
       private readonly ViewStream _main_stream;
       private readonly string _name;
       private ViewPublisher _vs_publisher;
-      private readonly PubTableBroker _broker = new PubTableBroker();
+      private readonly IPubTableBroker _broker;
 
-      public PublisherStream(ViewStream main_stream, string name) {
+      public PublisherStream(ViewStream main_stream, string name, IPubTableBroker broker) {
          _main_stream = main_stream;
          _name = name;
+         _broker = broker;
       }
 
       protected override void myOnFirstSubscriber() {
@@ -28,4 +32,4 @@ namespace BCG {
 
 
    }
-}
+}

src/DataStreamSuite/DataStream/Publish/ViewPublisher.cs

@@ -1,13 +1,13 @@
-using System;
-using System.Collections;
-using System.Data;
+using Bts.DataStreamCore;
 
-using BCG.Utilities;
+using mbts.utils;
+
+using System;
 
 namespace BCG {
    public class ViewPublisher : IDisposable {
       protected ViewPublisher() { }
-      private readonly PubTableBroker _broker;
+      private readonly IPubTableBroker _broker;
       protected readonly ViewStream _stream;
       private int _n_updates = 0;
 
@@ -15,7 +15,7 @@ namespace BCG {
       /// in the schema (be careful with that because this
       /// could lead to double publishes if the schema name
       /// is not uniquely chosen)
-      public ViewPublisher(string name, PubTableBroker broker, ViewStream vs) {
+      public ViewPublisher(string name, IPubTableBroker broker, ViewStream vs) {
          _stream = String.IsNullOrEmpty(name) ? vs : new ChangeTableNameStream(vs, name);
          _broker = broker;
          _stream.onChange += new DataStream.ChangeDelegate(_handle_update);
@@ -26,7 +26,7 @@ namespace BCG {
       }
 
       private void _handle_update(object o) {
-         DataPacket dp = o as DataPacket;
+         var dp = ExceptionAssert.of_type<IDataPacket>(o);
          if (_n_updates == 0) {
             _broker.create_table(dp.newRows);
             _n_updates++;

src/DataStreamSuite/DataStream/Services/DataStreamServices.cs

@@ -0,0 +1,11 @@
+namespace Bts.DataStream.Services;
+
+using BCG;
+
+using Microsoft.Extensions.DependencyInjection;
+
+public static class DataStreamServices {
+   public static IServiceCollection AddRemoteQueryManager(this IServiceCollection services) =>
+      services.AddSingleton<RemoteQueryManager>()
+         .AddHostedService(sp => sp.GetRequiredService<RemoteQueryManager>());
+}

src/DataStreamSuite/DataStream/StreamFactories/RemoteQueryStreamFactory.cs

@@ -0,0 +1,19 @@
+namespace Bts.DataStream.StreamFactories;
+
+using BCG;
+
+using Bts.DataStreamCore;
+
+using System.Collections.Generic;
+using System.Linq;
+
+public class RemoteQueryStreamFactory : DataStreamFactory {
+   public RemoteQueryStreamFactory(ITableStreamFactory tableStreamFactory, IPublisherCore publisherCore) {
+      this.Add("RemoteQueryStream", Create);
+
+      ViewStream? Create(IEnumerable<StreamArgument> args) {
+         var lines = args.Select(a => a.Value as string ?? a.Value?.ToString() ?? string.Empty).ToArray();
+         return new RemoteQueryStream(tableStreamFactory, publisherCore, lines);
+      }
+   }
+}

src/DataStreamSuite/DataStreamCore/BlueData/BlueColumnValueCollection.cs

@@ -0,0 +1,34 @@
+#nullable enable
+namespace Bts.DataStreamCore;
+
+using BCG;
+
+using Spark.Core.Collections;
+
+using System;
+using System.Collections.Generic;
+
+public static class BlueColumnValueCollection {
+   public static BlueColumnValueCollection<T> Create<T>(IBlueRowCollection rows, int columnIndex, Func<object?, T> convert) =>
+      new(rows, columnIndex, convert);
+}
+
+public class BlueColumnValueCollection<T> : IMutableCollectionFacade<T> {
+   readonly IBlueRowCollection rows;
+   readonly int columnIndex;
+   readonly Func<object?, T> convert;
+
+   public BlueColumnValueCollection(IBlueRowCollection rows, int columnIndex, Func<object?, T> convert) {
+      this.rows = rows;
+      this.columnIndex = columnIndex;
+      this.convert = convert;
+   }
+
+   public int Count => rows.Count;
+
+   public IEnumerator<T> GetEnumerator() {
+      foreach (IBlueRow r in rows) {
+         yield return convert(r[columnIndex]);
+      }
+   }
+}

src/DataStreamSuite/DataStreamCore/BlueData/BlueDataUtility.cs

@@ -19,6 +19,8 @@ using Newtonsoft.Json;
 using BCG;
 using System.Runtime.CompilerServices;
 using bts;
+using Spark.Core;
+using System.Numerics;
 
 namespace bts {
 
@@ -91,13 +93,13 @@ namespace BCG {
          return BlueDataUtility.get_row_datetime_value(r, col_no, err_value);
       }
 
-      public static double? GetDouble(this IBlueRow r, int colNo) =>
-         GetDouble(r[colNo]);
+      public static double? GetFloat64(this IBlueRow r, int colNo) =>
+         GetFloat64(r[colNo]);
 
-      public static double? GetDouble(this IBlueRow r, string colName) =>
-         GetDouble(r[colName]);
+      public static double? GetFloat64(this IBlueRow r, string colName) =>
+         GetFloat64(r[colName]);
 
-      public static double? GetDouble(object obj) {
+      public static double? GetFloat64(object obj) {
          if (obj is double d) {
             return d;
          }
@@ -125,6 +127,145 @@ namespace BCG {
          return null;
       }
 
+      public static int? GetInt32(this IBlueRow r, int colNo) =>
+         GetInt32(r[colNo]);
+
+      public static int? GetInt32(this IBlueRow r, string colName) =>
+         GetInt32(r[colName]);
+
+      public static int? GetInt32(object obj) {
+         checked {
+            if (obj is double d) {
+               return (int)d;
+            }
+            if (obj is int i32) {
+               return i32;
+            }
+            if (obj is long i64) {
+               return (int)i64;
+            }
+            if (obj is string s) {
+               return double.TryParse(s, out d) ? (int)d : null;
+            }
+            if (obj is uint u32) {
+               return (int)u32;
+            }
+            if (obj is ulong u64) {
+               return (int)u64;
+            }
+            if (obj is short i16) {
+               return i16;
+            }
+            if (obj is ushort u16) {
+               return u16;
+            }
+         }
+         return null;
+      }
+
+      public static long? GetInt64(this IBlueRow r, int colNo) =>
+         GetInt64(r[colNo]);
+
+      public static long? GetInt64(this IBlueRow r, string colName) =>
+         GetInt64(r[colName]);
+
+      public static long? GetInt64(object obj) {
+         checked {
+            if (obj is double d) {
+               return (long)d;
+            }
+            if (obj is int i32) {
+               return i32;
+            }
+            if (obj is long i64) {
+               return i64;
+            }
+            if (obj is string s) {
+               return double.TryParse(s, out d) ? (long)d : null;
+            }
+            if (obj is uint u32) {
+               return u32;
+            }
+            if (obj is ulong u64) {
+               return (long)u64;
+            }
+            if (obj is short i16) {
+               return i16;
+            }
+            if (obj is ushort u16) {
+               return u16;
+            }
+         }
+         return null;
+      }
+
+      public static DateTime? GetUnixEpochDateTime(this IBlueRow row, string colName) =>
+         GetUnixEpochDateTime(row[colName]);
+
+      public static DateTime? GetUnixEpochDateTime(this IBlueRow row, int colNo) =>
+         GetUnixEpochDateTime(row[colNo]);
+
+      public static DateTime? GetUnixEpochDateTime(object obj) {
+         var tsNanos = GetInt64(obj);
+         return tsNanos.HasValue ? DateTime.UnixEpoch.Add(new TimeSpan(tsNanos.Value / 100)) : null;
+      }
+
+      public static DateTime? GetDateTime(this IBlueRow row, int colIndex) =>
+         GetDateTime(row[colIndex]);
+
+      public static DateTime? GetDateTime(this IBlueRow row, string colName) =>
+         GetDateTime(row[colName]);
+
+      public static DateTime? GetDateTime(object obj) {
+         var res = _get_datetime_value(obj, DateTime.MinValue);
+         if (res == DateTime.MinValue) {
+            return null;
+         }
+         return res;
+      }
+
+      public static object GetEnum(Type type, object o) {
+         try {
+            if (o is null || o == DBNull.Value) {
+               return null;
+            } else if (o is string str) {
+               return Enum.TryParse(type, str, true, out var res) ? res : null;
+            } else if (o is int || o is uint || o is long || o is ulong || o is short || o is ushort) {
+               return Convert.ChangeType(o, type);
+            }
+         } catch {
+         }
+         return null;
+      }
+
+      public static T? GetEnum<T>(this IBlueRow row, string colName) where T : struct, Enum =>
+         GetEnum<T>(row[colName]);
+
+      public static T? GetEnum<T>(object o) where T : struct, Enum {
+         try {
+            if (o is null || o == DBNull.Value) {
+               return null;
+            } else if (o is string str) {
+               return Enum.TryParse(str, true, out T res) ? res : null;
+            } else if (o is int || o is short || o is ushort || o is uint || o is long || o is ulong)
+               return (T)o;
+            else if (o is T t) {
+               return t;
+            }
+         } catch {
+         }
+         return null;
+      }
+
+      public static string GetString(this IBlueRow row, string colName) =>
+         GetString(row[colName]);
+
+      public static string GetString(this IBlueRow row, int colNo) =>
+         GetString(row[colNo]);
+
+      public static string GetString(object obj) =>
+         obj as string ?? obj?.ToString();
+
       public static T get_enum<T>(this IBlueRow r, string col_name, T err_value = default(T)) where T : struct {
          return BlueDataUtility.get_row_enum_value<T>(r, col_name, err_value);
       }

src/DataStreamSuite/DataStreamCore/BlueData/BlueRowCollectionExtensions.cs

@@ -0,0 +1,17 @@
+#nullable enable
+namespace Bts.DataStreamCore;
+
+using BCG;
+
+using Spark.Core.Collections;
+
+
+using System.Collections.Generic;
+
+public static class BlueRowCollectionExtensions {
+   public static IReadOnlyCollection<object?> GetColumn(this IBlueRowCollection rows, string columnName) =>
+      rows.GetColumn(rows.schema.columnInfo.ResolveIndices(columnName));
+
+   public static IReadOnlyCollection<object?> GetColumn(this IBlueRowCollection rows, int columnIndex) =>
+      SelectorCollection.Create(rows, r => ((IBlueRow)r!)[columnIndex]);
+}

src/DataStreamSuite/DataStreamCore/BlueData/BlueRowEqualityComparer.cs

@@ -8,12 +8,15 @@ using System.Diagnostics.CodeAnalysis;
 using System.Linq;
 
 public class BlueRowEqualityComparer : IEqualityComparer<IBlueRow> {
-   public static BlueRowEqualityComparer Instance { get; } = new();
+   public static IEqualityComparer<IBlueRow> Instance { get; } = new BlueRowEqualityComparer();
 
-   public bool Equals(IBlueRow x, IBlueRow y) =>
+   public static bool Equals(IBlueRow x, IBlueRow y) =>
       x.schema.columnInfo.SequenceEqual(y.schema.columnInfo) &&
       x.Values.Cast<object>().SequenceEqual(y.Values.Cast<object>());
 
+   bool IEqualityComparer<IBlueRow>.Equals(IBlueRow x, IBlueRow y) =>
+      Equals(x, y);
+
    public int GetHashCode([DisallowNull] IBlueRow obj) {
       var hash = new HashCode();
       foreach (var v in obj.Values) {

src/DataStreamSuite/DataStreamCore/BlueData/BlueRowStore.cs

@@ -0,0 +1,311 @@
+#nullable enable
+namespace Bts.DataStreamCore;
+
+using BCG;
+
+using mbts.utils;
+
+using System;
+using System.Collections;
+using System.Collections.Generic;
+using System.Diagnostics.CodeAnalysis;
+using System.Linq;
+
+public class BlueRowStore<T1, T2, T3, T4, T5, T6> : IBlueRow, IReadOnlyDictionary<string, object?> {
+   readonly string? name;
+   IBlueTableSchema? _schema;
+
+   public BlueRowStore() { }
+
+   public BlueRowStore(
+      int id,
+      (string Name, T1 Value) item1,
+      (string Name, T2 Value) item2,
+      (string Name, T3 Value) item3,
+      (string Name, T4 Value) item4,
+      (string Name, T5 Value) item5,
+      (string Name, T6 Value) item6
+   ) {
+      Id = id;
+      Item1 = item1;
+      Item2 = item2;
+      Item3 = item3;
+      Item4 = item4;
+      Item5 = item5;
+      Item6 = item6;
+   }
+
+   public int Id { get; set; }
+   public (string Name, T1 Value) Item1 { get; set; }
+   public (string Name, T2 Value) Item2 { get; set; }
+   public (string Name, T3 Value) Item3 { get; set; }
+   public (string Name, T4 Value) Item4 { get; set; }
+   public (string Name, T5 Value) Item5 { get; set; }
+   public (string Name, T6 Value) Item6 { get; set; }
+
+   public int Count => 5;
+   public IReadOnlyList<string> Keys => new KeyList(this);
+   public IReadOnlyList<object?> Values => new ValueList(this);
+
+   public IBlueTableSchema schema =>
+      _schema ??= new BlueTableSchema(name, new ColumnInfo[]{
+         new("id", 0, typeof(int)),
+         new(Item1.Name, 1, typeof(T1)),
+         new(Item2.Name, 2, typeof(T2)),
+         new(Item3.Name, 3, typeof(T3)),
+         new(Item4.Name, 4, typeof(T4)),
+         new(Item5.Name, 5, typeof(T5)),
+         new(Item6.Name, 6, typeof(T6)),
+      });
+
+   ICollection IDictionary.Keys => new KeyList(this);
+   ICollection IDictionary.Values => new ValueList(this);
+
+   IEnumerable<string> IReadOnlyDictionary<string, object?>.Keys => Keys;
+   IEnumerable<object?> IReadOnlyDictionary<string, object?>.Values => Values;
+
+   bool ICollection.IsSynchronized => false;
+   object ICollection.SyncRoot => this;
+
+   public object? this[string colName] {
+      get {
+         if (colName == "id") {
+            return Id;
+         }
+         if (colName == Item1.Name) {
+            return Item1.Value;
+         }
+         if (colName == Item2.Name) {
+            return Item2.Value;
+         }
+         if (colName == Item3.Name) {
+            return Item3.Value;
+         }
+         if (colName == Item4.Name) {
+            return Item4.Value;
+         }
+         if (colName == Item5.Name) {
+            return Item5.Value;
+         }
+         if (colName == Item6.Name) {
+            return Item6.Value;
+         }
+         throw new KeyNotFoundException();
+      }
+      set {
+         if (colName == "id") {
+            Id = ExceptionAssert.Cast(value).To<int>();
+         }
+         if (colName == Item1.Name) {
+            Item1 = (Item1.Name, ExceptionAssert.Cast(value).To<T1>());
+         }
+         if (colName == Item2.Name) {
+            Item2 = (Item2.Name, ExceptionAssert.Cast(value).To<T2>());
+         }
+         if (colName == Item3.Name) {
+            Item3 = (Item3.Name, ExceptionAssert.Cast(value).To<T3>());
+         }
+         if (colName == Item4.Name) {
+            Item4 = (Item4.Name, ExceptionAssert.Cast(value).To<T4>());
+         }
+         if (colName == Item5.Name) {
+            Item5 = (Item5.Name, ExceptionAssert.Cast(value).To<T5>());
+         }
+         if (colName == Item6.Name) {
+            Item6 = (Item6.Name, ExceptionAssert.Cast(value).To<T6>());
+         }
+         throw new KeyNotFoundException();
+      }
+   }
+
+   public object? this[int index] => index switch {
+      0 => Id,
+      1 => Item1.Value,
+      2 => Item2.Value,
+      3 => Item3.Value,
+      4 => Item4.Value,
+      5 => Item5.Value,
+      6 => Item6.Value,
+      _ => throw new IndexOutOfRangeException()
+   };
+
+   object? IDictionary.this[object key] {
+      get => this[ExceptionAssert.Cast(key).To<string>()];
+      set => this[ExceptionAssert.Cast(key).To<string>()] = value;
+   }
+
+   public void CopyTo(Array array, int index) {
+      array.SetValue(Id, index++);
+      array.SetValue(Item1.Value, index++);
+      array.SetValue(Item2.Value, index++);
+      array.SetValue(Item3.Value, index++);
+      array.SetValue(Item4.Value, index++);
+      array.SetValue(Item5.Value, index++);
+      array.SetValue(Item6.Value, index++);
+   }
+
+   public bool Contains(object key) =>
+      key is string keystr && ContainsKey(keystr);
+
+   public bool ContainsKey(string key) =>
+      Keys.Contains(key);
+
+   public void SetId(int id) =>
+      Id = id;
+
+   public void SetValue1(T1 value) =>
+      Item1 = (Item1.Name, value);
+
+   public void SetValue1(string colName, T1 value) =>
+      Item1 = (colName, value);
+
+   public void SetValue2(T2 value) =>
+      Item2 = (Item2.Name, value);
+
+   public void SetValue2(string colName, T2 value) =>
+      Item2 = (colName, value);
+
+   public void SetValue3(T3 value) =>
+      Item3 = (Item3.Name, value);
+
+   public void SetValue3(string colName, T3 value) =>
+      Item3 = (colName, value);
+
+   public void SetValue4(T4 value) =>
+      Item4 = (Item4.Name, value);
+
+   public void SetValue4(string colName, T4 value) =>
+      Item4 = (colName, value);
+
+   public void SetValue5(T5 value) =>
+      Item5 = (Item5.Name, value);
+
+   public void SetValue5(string colName, T5 value) =>
+      Item5 = (colName, value);
+
+   public void SetValue6(T6 value) =>
+      Item6 = (Item6.Name, value);
+
+   public void SetValue6(string colName, T6 value) =>
+      Item6 = (colName, value);
+
+   public bool TryGetValue(string key, [MaybeNullWhen(false)] out object? value) {
+      if (!Keys.TryFindIndex(key, out int index)) {
+         value = null;
+         return false;
+      }
+      value = this[index];
+      return true;
+   }
+
+   void IDictionary.Add(object key, object? value) =>
+      throw new NotSupportedException();
+
+   void IDictionary.Clear() =>
+      throw new NotSupportedException();
+
+   void IDictionary.Remove(object key) =>
+      throw new NotSupportedException();
+
+   IDictionaryEnumerator IDictionary.GetEnumerator() =>
+      throw new NotImplementedException();
+
+   IEnumerator IEnumerable.GetEnumerator() =>
+      throw new NotImplementedException();
+
+   public IEnumerator<KeyValuePair<string, object?>> GetEnumerator() {
+      throw new NotImplementedException();
+   }
+
+   class KeyList : ICollection, IReadOnlyList<string> {
+      readonly BlueRowStore<T1, T2, T3, T4, T5, T6> store;
+
+      public KeyList(BlueRowStore<T1, T2, T3, T4, T5, T6> store) =>
+         this.store = store;
+
+      public int Count => 5;
+      bool ICollection.IsSynchronized => false;
+      object ICollection.SyncRoot => this;
+
+      public string this[int index] =>
+         index switch {
+            0 => "id",
+            1 => store.Item1.Name,
+            2 => store.Item2.Name,
+            3 => store.Item3.Name,
+            4 => store.Item4.Name,
+            5 => store.Item5.Name,
+            6 => store.Item6.Name,
+            _ => throw new IndexOutOfRangeException()
+         };
+
+      public void CopyTo(Array array, int index) {
+         array.SetValue(store.Id, index++);
+         array.SetValue(store.Item1.Value, index++);
+         array.SetValue(store.Item2.Value, index++);
+         array.SetValue(store.Item3.Value, index++);
+         array.SetValue(store.Item4.Value, index++);
+         array.SetValue(store.Item5.Value, index++);
+         array.SetValue(store.Item6.Value, index++);
+      }
+
+      public IEnumerator<string> GetEnumerator() {
+         yield return "id";
+         yield return store.Item1.Name;
+         yield return store.Item2.Name;
+         yield return store.Item3.Name;
+         yield return store.Item4.Name;
+         yield return store.Item5.Name;
+         yield return store.Item6.Name;
+      }
+
+      IEnumerator IEnumerable.GetEnumerator() =>
+         GetEnumerator();
+   }
+
+   class ValueList : ICollection, IReadOnlyList<object?> {
+      readonly BlueRowStore<T1, T2, T3, T4, T5, T6> store;
+
+      public ValueList(BlueRowStore<T1, T2, T3, T4, T5, T6> store) =>
+         this.store = store;
+
+      public object? this[int index] =>
+         index switch {
+            0 => store.Id,
+            1 => store.Item1.Value,
+            2 => store.Item2.Value,
+            3 => store.Item3.Value,
+            4 => store.Item4.Value,
+            5 => store.Item5.Value,
+            6 => store.Item6.Value,
+            _ => throw new IndexOutOfRangeException()
+         };
+
+      public int Count => 5;
+      public bool IsSynchronized => false;
+      public object SyncRoot => this;
+
+      public void CopyTo(Array array, int index) {
+         array.SetValue(store.Id, index++);
+         array.SetValue(store.Item1.Value, index++);
+         array.SetValue(store.Item2.Value, index++);
+         array.SetValue(store.Item3.Value, index++);
+         array.SetValue(store.Item4.Value, index++);
+         array.SetValue(store.Item5.Value, index++);
+         array.SetValue(store.Item6.Value, index++);
+      }
+
+      public IEnumerator<object?> GetEnumerator() {
+         yield return store.Id;
+         yield return store.Item1.Value;
+         yield return store.Item2.Value;
+         yield return store.Item3.Value;
+         yield return store.Item4.Value;
+         yield return store.Item5.Value;
+         yield return store.Item6.Value;
+      }
+
+      IEnumerator IEnumerable.GetEnumerator() =>
+         GetEnumerator();
+   }
+}

src/DataStreamSuite/DataStreamCore/BlueData/BlueTableListWrapper.cs

@@ -0,0 +1,49 @@
+#nullable enable
+namespace Bts.DataStreamCore;
+
+using BCG;
+
+using System;
+using System.Collections;
+using System.Collections.Generic;
+using System.Linq;
+
+public class BlueTableListWrapper : IBlueRowCollection {
+   readonly IBlueTableSchema schema;
+   readonly IList<IBlueRow> rows;
+   readonly Lazy<Dictionary<int, (int Index, IBlueRow Row)>> rowIndex;
+
+   public BlueTableListWrapper(IBlueTableSchema schema, IList<IBlueRow> rows) {
+      this.schema = schema;
+      this.rows = rows;
+      rowIndex = new(() => rows.Select((r, i) => (i, r)).ToDictionary(r => r.r.Id()));
+   }
+
+   public IBlueRow this[int id] =>
+      rowIndex.Value.TryGetValue(id, out var r)
+         ? r.Row
+         : throw new KeyNotFoundException();
+
+   public IBlueTableSchema Schema => schema;
+
+   IBlueTableSchema IBlueRowCollection.schema => schema;
+
+   public int Count => rows.Count;
+
+   public bool containsID(int id) =>
+      rowIndex.Value.ContainsKey(id);
+
+   public IEnumerator<IBlueRow> GetEnumerator() =>
+      rows.GetEnumerator();
+
+   public IBlueRow GetRow(int index) =>
+      rows[index];
+
+   public int IndexOf(IBlueRow item) =>
+      rowIndex.Value.TryGetValue(item.Id(), out var r) && BlueRowEqualityComparer.Equals(r.Row, item)
+         ? r.Index
+         : -1;
+
+   IEnumerator IEnumerable.GetEnumerator() =>
+      GetEnumerator();
+}

src/DataStreamSuite/DataStreamCore/BlueData/BlueTableSchemaAdapter.cs

@@ -1,5 +1,5 @@
 #nullable enable
-namespace Bts.DataStream;
+namespace Bts.DataStreamCore;
 
 using BCG;
 
@@ -73,6 +73,21 @@ public class BlueTableSchemaAdapter<T> : IMutableBlueRowFactory<T>, IBlueTableSc
       columnNames = null;
    }
 
+   public void AddColumn<TResult>(string name, Func<T, TResult?> getValue) {
+      var dataType = typeof(TResult);
+      if (!ColumnInfo.isAllowedType(dataType)) {
+         throw new ArgumentException(
+            $"'{dataType}' is not a supported column type", nameof(dataType));
+      }
+      // Invalidate cached schema (set it to null)
+      schema = null;
+      var adapter = new ColumnAdapter(new(name, columns.Count, dataType), r => getValue(r));
+      columns.Add(adapter);
+      columnNameIndex[name] = adapter;
+      columnInfos = null;
+      columnNames = null;
+   }
+
    public ColumnAdapter this[int columnIndex] =>
       columns[columnIndex];
 

src/DataStreamSuite/DataStreamCore/BlueData/BlueTableSchemaExtensions.cs

@@ -0,0 +1,119 @@
+namespace Bts.DataStreamCore;
+
+using BCG;
+
+using bts.BlueOrb;
+using bts.collections;
+
+using System;
+using System.Diagnostics.CodeAnalysis;
+using System.Linq;
+
+public static class BlueTableSchemaExtensions {
+   public static int ResolveIndices(this IBlueTableSchema schema, string name) {
+      var indices = ResolveIndices(schema, new[] { name });
+      return indices[0];
+   }
+
+   public static (int, int) ResolveIndices(this IBlueTableSchema schema, string name1, string name2) {
+      var indices = ResolveIndices(schema, new[] { name1, name2 });
+      return (indices[0], indices[1]);
+   }
+
+   public static (int, int, int) ResolveIndices(this IBlueTableSchema schema, string name1, string name2, string name3) {
+      var indices = ResolveIndices(schema, new[] { name1, name2, name3 });
+      return (indices[0], indices[1], indices[2]);
+   }
+
+   public static (int, int, int, int) ResolveIndices(this IBlueTableSchema schema, string name1, string name2, string name3, string name4) {
+      var indices = ResolveIndices(schema, new[] { name1, name2, name3, name4 });
+      return (indices[0], indices[1], indices[2], indices[3]);
+   }
+
+   public static int[] ResolveIndices(this IBlueTableSchema schema, string[] columnNames) =>
+      ResolveIndices(schema.columnInfo, columnNames);
+
+   public static int ResolveIndices(this ColumnInfo[] columns, string name) {
+      var indices = ResolveIndices(columns, new[] { name });
+      return indices[0];
+   }
+
+   public static int[] ResolveIndices(this ColumnInfo[] columns, string[] columnNames) {
+      var indices = new int[columnNames.Length];
+      int nResolved = 0;
+      for (int i = 0; i < columnNames.Length; i++) {
+         int origNResolved = nResolved;
+         for (int j = 0; j < columns.Length; j++) {
+            if (columns[j].colName == columnNames[i]) {
+               indices[i] = j;
+               nResolved++;
+               break;
+            }
+         }
+         if (nResolved == origNResolved) {
+            throw new ArgumentException($"No column named '{columnNames[i]}'", nameof(columns));
+         }
+      }
+      return indices;
+   }
+
+   public static bool TryResolveIndices(this IBlueTableSchema schema, string name, out int result) {
+      var res = TryResolveIndices(schema, new[] { name })[0];
+      result = res ?? -1;
+      return res.HasValue;
+   }
+
+   public static int? TryResolveIndices(this IBlueTableSchema schema, string name) =>
+      TryResolveIndices(schema, new[] { name })[0];
+
+   public static int? TryResolveIndices(this ColumnInfo[] columns, string name) =>
+      TryResolveIndices(columns, new[] { name })[0];
+
+   public static int?[] TryResolveIndices(this IBlueTableSchema schema, string[] columnNames) =>
+      TryResolveIndices(schema.columnInfo, columnNames);
+
+   public static int?[] TryResolveIndices(this ColumnInfo[] columns, string[] columnNames) {
+      var indices = new int?[columnNames.Length];
+      indices.Fill(null);
+      int nResolved = 0;
+      for (int i = 0; i < columnNames.Length; i++) {
+         for (int j = 0; j < columns.Length; j++) {
+            if (columns[j].colName == columnNames[i]) {
+               indices[i] = j;
+               nResolved++;
+               break;
+            }
+         }
+      }
+      return indices;
+   }
+
+   public static IBlueTableSchema Validate([NotNullWhen(true)]this IBlueTableSchema? schema) {
+      if (schema is null) {
+         throw SchemaValidationExceptionFactory.Null();
+      }
+      var cols = schema.columnInfo ?? throw SchemaValidationExceptionFactory.NullOrEmptyColumns();
+      if (cols.Length == 0) {
+         throw SchemaValidationExceptionFactory.NoColumns();
+      }
+      var colNames = schema.columnNames ?? throw SchemaValidationExceptionFactory.NullOrEmptyColumnNames();
+      if (colNames.Any(string.IsNullOrWhiteSpace)) {
+         throw SchemaValidationExceptionFactory.NullEmptyWhitespaceColumnName();
+      }
+      if (!schema.columnNames.SequenceEqual(schema.columnInfo.Select(c => c.colName))) {
+         throw SchemaValidationExceptionFactory.MismatchedColumnNames();
+      }
+      if (!cols.Any(c => c.colName == "id" && c.colType == typeof(int))) {
+         throw SchemaValidationExceptionFactory.NoIdColumn();
+      }
+      if (cols.Contains(null)) {
+         throw SchemaValidationExceptionFactory.NullColumn();
+      }
+      for (int i = 0; i < cols.Length; i++) {
+         if (cols[i].colNo != i) {
+            throw SchemaValidationExceptionFactory.MismatchedColumnIndices();
+         }
+      }
+      return schema;
+   }
+}

src/DataStreamSuite/DataStreamCore/BlueData/CircularBufferTable.cs

@@ -1,53 +1,93 @@
 using System;
 using System.Collections;
 using System.Collections.Generic;
-using System.Data;
-using System.Linq;
+using System.Collections.Immutable;
 
 using BCG;
 
+using bts.collections;
+
+using Bts.DataStreamCore;
+
 namespace bts {
    public class CircularBufferTable : IBlueRowCollection {
-      private BlueTable _table;
-      private List<IBlueRow> _ordered_list = new List<IBlueRow>();
+      readonly ImmutableList<IBlueRow> orderedRows;
+      readonly ImmutableDictionary<int, IBlueRow> rowIndex;
       private int _buffer_sz;
-      public CircularBufferTable(IBlueTableSchema schema, int buffer_sz) {
-         _table = new BlueTable(schema);
-         _buffer_sz = buffer_sz;
+      public CircularBufferTable(IBlueTableSchema schema, int buffer_sz)
+         : this(schema, buffer_sz, ImmutableList<IBlueRow>.Empty, ImmutableDictionary<int, IBlueRow>.Empty) { }
+
+      CircularBufferTable(IBlueTableSchema schema, int bufferSize, ImmutableList<IBlueRow> orderedRows, ImmutableDictionary<int, IBlueRow> rowIndex) {
+         this.schema = schema;
+         _buffer_sz = bufferSize;
+         this.orderedRows = orderedRows;
+         this.rowIndex = rowIndex;
       }
 
+      public CircularBufferTable Add(IBlueRow row) =>
+         Add(row, out _);
+
       // returns the row that might need to be deleted to keep
       // the circular buffer intact
-      public IBlueRow add_row(IBlueRow row) {
-         _ordered_list.Add(row);
-         _table.addRow(row);
-         if (_ordered_list.Count > _buffer_sz) {
-            IBlueRow rv = _ordered_list.ElementAt(0);
-            _ordered_list.RemoveAt(0);
-            _table.deleteRow(rv);
-            return rv;
+      public CircularBufferTable Add(IBlueRow row, out IBlueRow oldRow) {
+         int id = row.Id();
+         var rowIndexBd = rowIndex.ToBuilder();
+         var orderedRowsBd = orderedRows.ToBuilder();
+         if (rowIndexBd.TryGetValue(id, out oldRow)) {
+            Log.assert(orderedRowsBd.RemoveFirst(r => r.Id() == id));
+         } else if (rowIndexBd.Count > _buffer_sz) {
+            oldRow = orderedRowsBd[0];
+            rowIndexBd.Remove(oldRow.Id());
+            orderedRowsBd.RemoveAt(0);
          }
-         return null;
+         rowIndexBd[id] = row;
+         orderedRowsBd.Add(row);
+         return new(schema, _buffer_sz, orderedRowsBd.ToImmutable(), rowIndexBd.ToImmutable());
       }
 
       public IBlueRow this[int id] {
          get {
-            return _table[id];
-         }
-      }
-      public IBlueTableSchema schema {
-         get {
-            return _table.schema;
+            return orderedRows[id];
          }
       }
+      public IBlueTableSchema schema { get; }
       public bool containsID(int id) {
-         return _table.containsID(id);
+         return rowIndex.ContainsKey(id);
       }
 
-      public void CopyTo(Array array, int index) { _table.CopyTo((IBlueRow[])array, index); }
-      public int Count { get { return _table.Count; } }
+      public void CopyTo(Array array, int index) {
+         foreach (var row in orderedRows) {
+            array.SetValue(row, index++);
+         }
+      }
+      public int Count => rowIndex.Count;
       public bool IsSynchronized { get { return false; } }
       public object SyncRoot { get { return this; } }
-      public System.Collections.IEnumerator GetEnumerator() { return _table.GetEnumerator(); }
+      public IEnumerator<IBlueRow> GetEnumerator() =>
+         orderedRows.GetEnumerator();
+
+      public IList<object> GetColumn(int index) =>
+         orderedRows.WrapSelector(r => r[index], null);
+
+      public IBlueRow GetRow(int index) =>
+         orderedRows[index];
+
+      public int IndexOf(IBlueRow item) {
+         int id = item.Id();
+         if (rowIndex.TryGetValue(id, out var row) && BlueRowEqualityComparer.Equals(item, row)) {
+            int i = 0;
+            foreach (var r in orderedRows) {
+               if (r.Id() == id) {
+                  return i;
+               }
+               i++;
+            }
+         }
+         return -1;
+      }
+
+      IEnumerator IEnumerable.GetEnumerator() =>
+         GetEnumerator();
    }
 }
+

src/DataStreamSuite/DataStreamCore/BlueData/ColumnInfo.cs

@@ -21,11 +21,17 @@ namespace BCG {
       }
 
       public static bool isAllowedType(Type type) =>
+         type == typeof(short) ||
          type == typeof(int) ||
          type == typeof(long) ||
+         type == typeof(ushort) ||
+         type == typeof(uint) ||
+         type == typeof(ulong) ||
+         type == typeof(float) ||
          type == typeof(double) ||
          type == typeof(string) ||
-         type == typeof(DateTime);
+         type == typeof(DateTime) ||
+         type == typeof(TimeSpan);
 
       public int CompareTo(object obj) {
          int otherColNo = ((ColumnInfo)(obj)).colNo;

src/DataStreamSuite/DataStreamCore/BlueData/IBlueRowFactory.cs

@@ -1,5 +1,5 @@
 #nullable enable
-namespace Bts.DataStream;
+namespace Bts.DataStreamCore;
 
 using BCG;
 

src/DataStreamSuite/DataStreamCore/BlueData/IMutableBlueRowFactory.cs

@@ -1,5 +1,5 @@
 #nullable enable
-namespace Bts.DataStream;
+namespace Bts.DataStreamCore;
 
 using BCG;
 

src/DataStreamSuite/DataStreamCore/BlueData/ITimeSeriesRow.cs

@@ -0,0 +1,12 @@
+namespace Bts.DataStreamCore;
+
+using BCG;
+
+using mbts.utils;
+
+using System;
+
+public interface ITimeSeriesRow : IBlueRow {
+   int Id => ExceptionAssert.of_type<int>(this["id"]);
+   DateTime Timestamp { get; }
+}

src/DataStreamSuite/DataStreamCore/BlueData/SchemaValidationException.cs

@@ -0,0 +1,42 @@
+namespace bts.BlueOrb;
+
+using System;
+
+public class SchemaValidationException : Exception {
+   public SchemaValidationException(string message) : base(message) { }
+}
+
+static class SchemaValidationExceptionFactory {
+   public static SchemaValidationException NoColumns() =>
+      new("IBlueTableSchema has no columns");
+
+   public static SchemaValidationException NoIdColumn() =>
+      new("IBlueTableSchema has no id column");
+
+   public static SchemaValidationException IdColumnInvalidType(Type type) =>
+      new($"IBlueTableSchema id column has invalid type {type}");
+
+   public static SchemaValidationException Null() =>
+      new("IBlueTableSchema is null");
+
+   public static SchemaValidationException NullColumn() =>
+      new("IBlueTableSchema has null column");
+
+   public static SchemaValidationException NullOrEmptyColumns() =>
+      new("IBlueTableSchema columnInfo array is null/empty");
+
+   public static SchemaValidationException NullOrEmptyColumnNames() =>
+      new("IBlueTableSchema columnNames array is null/empty");
+
+   public static SchemaValidationException NullEmptyWhitespaceColumnName() =>
+      new("IBlueTableSchema columnName is null/empty/whitespace");
+
+   public static SchemaValidationException DuplicateColumnName() =>
+      new("IBlueTableSchema has duplicate column names");
+
+   public static SchemaValidationException MismatchedColumnNames() =>
+      new("IBlueTableSchema columnNames array does not match names in columnInfo array");
+
+   public static SchemaValidationException MismatchedColumnIndices() =>
+      new("IBlueTableSchema column numbers do not match corresponding indices");
+}

src/DataStreamSuite/DataStreamCore/DataPacketJsonConverter.cs

@@ -128,6 +128,12 @@ public class DataPacketJsonConverter : JsonConverter<DataPacket> {
          if (t == typeof(bool)) {
             return NullCheckWrapper(v => writer.WriteBooleanValue((bool)v));
          }
+         if (t == typeof(DateTime)) {
+            return NullCheckWrapper(v => writer.WriteStringValue(((DateTime)v).ToUniversalTime().ToString("yyyy-MM-ddTHH:mm:ss.ffffff")));
+         }
+         if (t == typeof(TimeSpan)) {
+            return NullCheckWrapper(v => writer.WriteStringValue(((TimeSpan)v).ToString(@"hh\:mm\:ss\.ffffff")));
+         }
          return NullCheckWrapper(v => writer.WriteStringValue(v?.ToString()));
 
          Action<object?> NullCheckWrapper(Action<object> writeNonNull) =>

src/DataStreamSuite/DataStreamCore/DataStream.cs

@@ -32,6 +32,9 @@ namespace BCG
       public DataStream (DataStream realSource)
       {
          myRealSource = realSource;
+         if (myRealSource is not null) {
+            myRealSource.ErrorChanged += OnError;
+         }
          myOnChangeDelegate = null;
          myPendingAdds = new Stack();
       }
@@ -48,6 +51,8 @@ namespace BCG
          }
       }
 
+      public event Action<Exception> ErrorChanged;
+
       public delegate void ChangeDelegate(object newValue);
       public event ChangeDelegate onChange
       {
@@ -120,8 +125,15 @@ namespace BCG
       protected virtual object myGetValueForInitialSubscription()
       {return myLastValue;}
 
-      protected void mySetRealSource (DataStream src)
-      {myRealSource = src;}
+      protected void mySetRealSource (DataStream src) {
+         if (myRealSource is not null) {
+            myRealSource.ErrorChanged -= OnError;
+         }
+         myRealSource = src;
+         if (myRealSource is not null) {
+            myRealSource.ErrorChanged += OnError;
+         }
+      }
 
       protected virtual void myOnFirstSubscriber() {myFirstSubImplemented = false;}
       protected virtual void myOnLastSubscriber () {myLastSubImplemented = false;}
@@ -143,6 +155,11 @@ namespace BCG
          }
       }
 
+      protected void OnError(Exception error) {
+         Log.Gen.error($"{GetType()} OnError: {error}");
+         ErrorChanged?.Invoke(error);
+      }
+
       public Guid ID
       {
          get

src/DataStreamSuite/DataStreamCore/DataStreamCore.csproj

@@ -11,6 +11,7 @@
     <PackageReference Include="Newtonsoft.Json" Version="13.0.3" />
     <PackageReference Include="OpenAI" Version="2.0.0" />
     <PackageReference Include="SimMetrics.Net" Version="1.0.5" />
+    <PackageReference Include="System.Linq.AsyncEnumerable" Version="10.0.0" />
   </ItemGroup>
 
   <ItemGroup>

src/DataStreamSuite/DataStreamCore/DynamicLoading/CompositeDataStreamFactory.cs

@@ -0,0 +1,27 @@
+#nullable enable
+namespace Bts.DataStreamCore;
+
+using BCG;
+
+using System.Collections.Generic;
+using System.Linq;
+
+public class CompositeDataStreamFactory : IDataStreamFactory {
+   readonly List<IDataStreamFactory> streamFactories;
+
+   public CompositeDataStreamFactory(IEnumerable<IDataStreamFactory> streamFactories) {
+      this.streamFactories = streamFactories.ToList();
+      foreach (var rsf in this.streamFactories.OfType<IRecursiveDataStreamFactory>()) {
+         rsf.OnCompositeFactory(this);
+      }
+   }
+
+   public IEnumerable<IDataStreamMetadata> GetSupportedStreams() =>
+      streamFactories.SelectMany(f => f.GetSupportedStreams());
+
+   public ViewStream? Create(string streamId, IEnumerable<StreamArgument> args) =>
+      streamFactories.Where(f => f.CanCreate(streamId))
+         .Select(f => f.Create(streamId, args))
+         .ExceptNull()
+         .FirstOrDefault();
+}

src/DataStreamSuite/DataStreamCore/DynamicLoading/DataStreamFactory.cs

@@ -0,0 +1,228 @@
+#nullable enable
+namespace Bts.DataStreamCore;
+
+using BCG;
+
+using Microsoft.Extensions.Logging;
+
+using System;
+using System.Collections.Generic;
+using System.Linq;
+
+public class DataStreamFactory : IDataStreamFactory {
+   readonly List<IDataStreamFactory> factories = new();
+
+   public static IDataStreamFactory Create(IEnumerable<IDataStreamFactory> streamFactories) {
+      var dsf = new DataStreamFactory();
+      dsf.factories.AddRange(streamFactories);
+      return dsf.factories.Count == 1 ? dsf.factories[0] : dsf;
+   }
+
+   public ViewStream? Create(string streamId, IEnumerable<StreamArgument> args) =>
+      factories.Where(sf => sf.GetSupportedStreams().Any(m => m.IsMatch(streamId)))
+         .Select(sf => sf.Create(streamId, args))
+         .ExceptNull()
+         .FirstOrDefault();
+
+   public IEnumerable<IDataStreamMetadata> GetSupportedStreams() =>
+      factories.SelectMany(sf => sf.GetSupportedStreams());
+
+   public void Add(IDataStreamFactory streamFactory) =>
+      factories.Add(streamFactory);
+}
+
+public static class DataStreamFactoryExtensions {
+   public static void Add(this DataStreamFactory streamFactory, IDataStreamMetadata metadata, Func<ViewStream?> singleStreamfactory, ILogger? logger = null) {
+      var sf = SingleDataStreamFactory.FromDelegate(metadata, singleStreamfactory);
+      streamFactory.Add(sf);
+   }
+
+   public static void Add(this DataStreamFactory streamFactory, IDataStreamMetadata metadata, Func<IEnumerable<StreamArgument>, ViewStream?> singleStreamfactory, ILogger? logger = null) {
+      var sf = new SingleDataStreamFactory(metadata, singleStreamfactory);
+      streamFactory.Add(sf);
+   }
+
+   public static void Add(this DataStreamFactory streamFactory, string name, Func<ViewStream?> singleStreamfactory) {
+      var sf = new SingleDataStreamFactory(new DataStreamMetadata(name), FactoryWrapper);
+      streamFactory.Add(sf);
+
+      ViewStream? FactoryWrapper(IEnumerable<StreamArgument> _) =>
+         singleStreamfactory();
+   }
+
+   public static void Add(this DataStreamFactory streamFactory, string name, Func<IEnumerable<StreamArgument>, ViewStream?> singleStreamfactory, ILogger? logger = null) {
+      var sf = new SingleDataStreamFactory(new DataStreamMetadata() { Name = name }, singleStreamfactory);
+      streamFactory.Add(sf);
+   }
+
+   public static void Add<TParams>(this DataStreamFactory streamFactory, IDataStreamMetadata metadata, Func<TParams, ViewStream?> singleStreamfactory, ILogger? logger = null) {
+      var sf = new SingleDataStreamFactory<TParams>(metadata, singleStreamfactory, logger);
+      streamFactory.Add(sf);
+   }
+
+   public static void Add<T>(this DataStreamFactory streamFactory, string name, Func<T, ViewStream?> singleStreamfactory) {
+      var sf = new SingleDataStreamFactory<T>(new DataStreamMetadata(name), singleStreamfactory);
+      streamFactory.Add(sf);
+   }
+
+   public static void Add<T>(this DataStreamFactory streamFactory, string name, Func<T, ViewStream?> singleStreamfactory, ILogger? logger) {
+      var sf = new SingleDataStreamFactory<T>(new DataStreamMetadata(name), singleStreamfactory, logger);
+      streamFactory.Add(sf);
+   }
+
+   public static void Add<T>(this DataStreamFactory streamFactory, string name, Func<T, ViewStream?> factory, IStreamArgumentParser<T>? parser = null) {
+      var meta = new DataStreamMetadata() { Name = name };
+      parser ??= StreamArgumentParser.Default<T>();
+      var sf = new SingleDataStreamFactory(meta, FactoryWrapper);
+      streamFactory.Add(sf);
+
+      ViewStream? FactoryWrapper(IEnumerable<StreamArgument> args) {
+         if (!args.TryGetSingle(out var a)) {
+            return null;
+         }
+         if (!parser.CanParse(a, typeof(T))) {
+            return null;
+         }
+         if (!parser.TryParse(a, out var res, out var err)) {
+            return null;
+         }
+         return factory(res);
+      }
+   }
+
+   public static void Add<T1, T2>(
+      this DataStreamFactory streamFactory,
+      string name,
+      Func<T1, T2, ViewStream?> factory,
+      IStreamArgumentParser<T1>? parser1 = null,
+      IStreamArgumentParser<T2>? parser2 = null
+   ) {
+      var meta = new DataStreamMetadata() { Name = name };
+      parser1 ??= StreamArgumentParser.Default<T1>();
+      parser2 ??= StreamArgumentParser.Default<T2>();
+      var sf = new SingleDataStreamFactory(meta, FactoryWrapper);
+      streamFactory.Add(sf);
+
+      ViewStream? FactoryWrapper(IEnumerable<StreamArgument> args) {
+         if (!args.TryGetExactCount(out var a1, out var a2)) {
+            return null;
+         }
+         if (!parser1.CanAndTryParse(a2, out var v1, out var err1)) {
+            return null;
+         }
+         if (!parser2.CanAndTryParse(a2, out var v2, out var err2)) {
+            return null;
+         }
+         return factory(v1, v2);
+      }
+   }
+
+   public static void Add<T1, T2, T3>(
+      this DataStreamFactory streamFactory,
+      string name,
+      Func<T1, T2, T3, ViewStream?> factory,
+      IStreamArgumentParser<T1>? parser1 = null,
+      IStreamArgumentParser<T2>? parser2 = null,
+      IStreamArgumentParser<T3>? parser3 = null
+   ) {
+      var meta = new DataStreamMetadata() { Name = name };
+      parser1 ??= StreamArgumentParser.Default<T1>();
+      parser2 ??= StreamArgumentParser.Default<T2>();
+      parser3 ??= StreamArgumentParser.Default<T3>();
+      var sf = new SingleDataStreamFactory(meta, FactoryWrapper);
+      streamFactory.Add(sf);
+
+      ViewStream? FactoryWrapper(IEnumerable<StreamArgument> args) {
+         if (!args.TryGetExactCount(out var a1, out var a2, out var a3)) {
+            return null;
+         }
+         if (!parser1.CanAndTryParse(a2, out var v1, out var err1)) {
+            return null;
+         }
+         if (!parser2.CanAndTryParse(a2, out var v2, out var err2)) {
+            return null;
+         }
+         if (!parser3.CanAndTryParse(a3, out var v3, out var err3)) {
+            return null;
+         }
+         return factory(v1, v2, v3);
+      }
+   }
+
+   public static void Add<T1, T2, T3, T4>(
+      this DataStreamFactory streamFactory,
+      string name,
+      Func<T1, T2, T3, T4, ViewStream?> factory,
+      IStreamArgumentParser<T1>? parser1 = null,
+      IStreamArgumentParser<T2>? parser2 = null,
+      IStreamArgumentParser<T3>? parser3 = null,
+      IStreamArgumentParser<T4>? parser4 = null
+   ) {
+      var meta = new DataStreamMetadata() { Name = name };
+      parser1 ??= StreamArgumentParser.Default<T1>();
+      parser2 ??= StreamArgumentParser.Default<T2>();
+      parser3 ??= StreamArgumentParser.Default<T3>();
+      parser4 ??= StreamArgumentParser.Default<T4>();
+      var sf = new SingleDataStreamFactory(meta, FactoryWrapper);
+      streamFactory.Add(sf);
+
+      ViewStream? FactoryWrapper(IEnumerable<StreamArgument> args) {
+         if (!args.TryGetExactCount(out var a1, out var a2, out var a3, out var a4)) {
+            return null;
+         }
+         if (!parser1.CanAndTryParse(a2, out var v1, out var err1)) {
+            return null;
+         }
+         if (!parser2.CanAndTryParse(a2, out var v2, out var err2)) {
+            return null;
+         }
+         if (!parser3.CanAndTryParse(a3, out var v3, out var err3)) {
+            return null;
+         }
+         if (!parser4.CanAndTryParse(a4, out var v4, out var err4)) {
+            return null;
+         }
+         return factory(v1, v2, v3, v4);
+      }
+   }
+
+   public static void Add<T1, T2, T3, T4, T5>(this DataStreamFactory streamFactory, string name, Func<T1, T2, T3, T4, T5, ViewStream?> factory, IStreamArgumentParser<T1> parser1, IStreamArgumentParser<T2> parser2, IStreamArgumentParser<T3> parser3, IStreamArgumentParser<T4> parser4, IStreamArgumentParser<T5> parser5) {
+      var meta = new DataStreamMetadata() { Name = name };
+      var sf = new SingleDataStreamFactory(meta, FactoryWrapper);
+      streamFactory.Add(sf);
+
+      ViewStream? FactoryWrapper(IEnumerable<StreamArgument> args) {
+         if (!args.TryGetExactCount(out var a1, out var a2, out var a3, out var a4, out var a5)) {
+            return null;
+         }
+         if (!parser1.CanAndTryParse(a2, out var v1, out var err1)) {
+            return null;
+         }
+         if (!parser2.CanAndTryParse(a2, out var v2, out var err2)) {
+            return null;
+         }
+         if (!parser3.CanAndTryParse(a3, out var v3, out var err3)) {
+            return null;
+         }
+         if (!parser4.CanAndTryParse(a4, out var v4, out var err4)) {
+            return null;
+         }
+         if (!parser5.CanAndTryParse(a5, out var v5, out var err5)) {
+            return null;
+         }
+         return factory(v1, v2, v3, v4, v5);
+      }
+   }
+
+   public static void Add<TParams>(this DataStreamFactory streamFactory, string name, Func<TParams, ViewStream?> singleStreamfactory, ILogger? logger = null, StreamArgumentsParser.TryParse<TParams>? tryParse = null) {
+      var meta = new DataStreamMetadata(name);
+      var parser = tryParse is null
+         ? new StreamArgumentsParser<TParams>()
+         : StreamArgumentsParser.FromDelegate(tryParse);
+      var sf = new SingleDataStreamFactory<TParams>(meta, singleStreamfactory, parser, logger);
+      streamFactory.Add(sf);
+   }
+
+   public static ViewStream? Create(this IDataStreamFactory streamFactory, string streamId, params object?[] args) =>
+      streamFactory.Create(streamId, args.Select((a, i) => new StreamArgument(null, i, a)));
+}

src/DataStreamSuite/DataStreamCore/DynamicLoading/DataStreamMetadata.cs

@@ -0,0 +1,45 @@
+#nullable enable
+namespace Bts.DataStreamCore;
+
+using BCG;
+
+using System;
+using System.Reflection;
+
+public interface IDataStreamMetadata {
+   public string Name { get; }
+}
+
+public class DataStreamMetadata : IDataStreamMetadata {
+   public DataStreamMetadata() { }
+
+   public DataStreamMetadata(string name) =>
+      Name = name;
+
+   public string? Name { get; set; }
+
+   string IDataStreamMetadata.Name => Name ?? string.Empty;
+
+   public static DataStreamMetadata FromType(Type type) {
+      var friendlyName = type.GetCustomAttribute<DataStreamUserRespresentation>()?.friendlyName;
+      var rv = new DataStreamMetadata() {
+         Name = friendlyName ?? type.Name,
+      };
+      return rv;
+   }
+}
+
+public static class DataStreamMetadataExtensions {
+   public static bool IsMatch(this IDataStreamMetadata dsm, string name, bool ignoreCase = true, bool ignoreUnderscores = true) {
+      var strComp = ignoreCase ? StringComparison.CurrentCultureIgnoreCase : StringComparison.CurrentCulture;
+      var nameWithoutUnderscores = new Lazy<string>(() => RemoveUnderscores(name));
+      return IsNameMatch(dsm.Name);
+
+      static string RemoveUnderscores(string s) =>
+         s.Replace("_", "");
+
+      bool IsNameMatch(string otherName) =>
+         string.Equals(name, otherName, strComp)
+            || (ignoreUnderscores && string.Equals(RemoveUnderscores(name), RemoveUnderscores(otherName), strComp));
+   }
+}

src/DataStreamSuite/DataStreamCore/DynamicLoading/IDataStreamFactory.cs

@@ -0,0 +1,25 @@
+#nullable enable
+namespace Bts.DataStreamCore;
+
+using BCG;
+
+using System.Collections.Generic;
+using System.Linq;
+
+public interface IDataStreamFactory {
+   IEnumerable<IDataStreamMetadata> GetSupportedStreams();
+
+   ViewStream? Create(string streamId, IEnumerable<StreamArgument> args);
+
+   bool CanCreate(string streamId) =>
+      GetSupportedStreams().Any(s => s.IsMatch(streamId));
+}
+
+/// <summary>
+/// This interface provides a way for IDataStreamFactory's that create ViewStreams
+/// that depend specifically on the composite IDataStreamFactory. This was designed
+/// specifically for the NLStreamDataFactory.
+/// </summary>
+public interface IRecursiveDataStreamFactory : IDataStreamFactory {
+   void OnCompositeFactory(IDataStreamFactory compositeFactory);
+}

src/DataStreamSuite/DataStreamCore/DynamicLoading/ISingleDataStreamFactory.cs

@@ -0,0 +1,18 @@
+#nullable enable
+namespace Bts.DataStreamCore;
+
+using BCG;
+
+using System.Collections.Generic;
+
+public interface ISingleStreamFactory : IDataStreamFactory {
+   IDataStreamMetadata Metadata { get; }
+
+   ViewStream? Create(IEnumerable<StreamArgument> args);
+
+   ViewStream? IDataStreamFactory.Create(string streamId, IEnumerable<StreamArgument> args) =>
+      Metadata.IsMatch(streamId) ? Create(args) : null;
+
+   IEnumerable<IDataStreamMetadata> IDataStreamFactory.GetSupportedStreams() =>
+      new[] { Metadata };
+}