Reversehedging fixed

Format: ran clang format on file
Add: config changes to make server controller work and set up hk zen server
2025-04-08 22:13:33 +00:00 · 2025-04-07 15:39:57 +00:00 · 2025-04-07 15:33:26 +00:00 · 2025-04-05 23:38:05 +00:00 · 2025-04-05 15:38:30 +00:00
3046 changed files with 24544 additions and 74911 deletions
@@ -25,12 +25,6 @@
      "commands": [
        "fake"
      ]
-    },
-    "dotnet-t4": {
-      "version": "3.0.0",
-      "commands": [
-        "t4"
-      ]
    }
  }
 }
@@ -65,7 +65,7 @@ RUN \
   && apt-get install -y cmake

 # Dotnet
-RUN apt-get update && apt-get install -y dotnet8
+RUN apt-get update && apt-get install -y dotnet7

 # Node
 ARG node_major=18
@@ -22,7 +22,6 @@
      "extensions": [
        "eamodio.gitlens",
        "llvm-vs-code-extensions.vscode-clangd",
-        "ms-python.python",
        "ms-vscode.cmake-tools",
        "ms-vscode.cpptools",
        "xaver.clang-format"
@@ -207,23 +207,17 @@ dotnet_naming_symbols.parameters.applicable_accessibilities = public,internal,pr
 dotnet_naming_symbols.private_symbols.applicable_kinds = property,method,field,event,delegate
 dotnet_naming_symbols.private_symbols.applicable_accessibilities = protected,private

-# local symbols
-dotnet_naming_symbols.local_symbols.applicable_kinds = local
-
-# parameter symbols
-dotnet_naming_symbols.parameter_symbols.applicable_kinds = parameter
-
 # Naming styles

-dotnet_naming_style.pascal_case.required_prefix =
-dotnet_naming_style.pascal_case.required_suffix =
-dotnet_naming_style.pascal_case.word_separator =
-dotnet_naming_style.pascal_case.capitalization = pascal_case
+dotnet_naming_style.snake_case.required_prefix =
+dotnet_naming_style.snake_case.required_suffix =
+dotnet_naming_style.snake_case.word_separator = _
+dotnet_naming_style.snake_case.capitalization = all_lower

-dotnet_naming_style.camel_case.required_prefix =
-dotnet_naming_style.camel_case.required_suffix =
-dotnet_naming_style.camel_case.word_separator =
-dotnet_naming_style.camel_case.capitalization = camel_case
+dotnet_naming_style.private_snake_case.required_prefix = _
+dotnet_naming_style.private_snake_case.required_suffix =
+dotnet_naming_style.private_snake_case.word_separator = _
+dotnet_naming_style.private_snake_case.capitalization = all_lower

 dotnet_naming_style.user_types.required_prefix =
 dotnet_naming_style.user_types.required_suffix =
@@ -235,10 +229,6 @@ dotnet_naming_style.interface.required_suffix =
 dotnet_naming_style.interface.word_separator =
 dotnet_naming_style.interface.capitalization = pascal_case

-dotnet_naming_style.local.capitalization = camel_case
-
-dotnet_naming_style.parameter.capitalization = camel_case
-
 # rules
 dotnet_naming_rule.user_types.symbols = user_types
 dotnet_naming_rule.user_types.style = user_types
@@ -248,18 +238,14 @@ dotnet_naming_rule.interfaces.symbols = interface
 dotnet_naming_rule.interfaces.style = interface_style
 dotnet_naming_rule.interfaces.severity = suggestion

-dotnet_naming_rule.public_members.symbols = public_symbols
-dotnet_naming_rule.public_members.style = pascal_case
-dotnet_naming_rule.public_members.severity = suggestion
+dotnet_naming_rule.public_members_snake_case.symbols = public_symbols
+dotnet_naming_rule.public_members_snake_case.style = snake_case
+dotnet_naming_rule.public_members_snake_case.severity = suggestion

-dotnet_naming_rule.private_members.symbols = private_symbols
-dotnet_naming_rule.private_members.style = camel_case
-dotnet_naming_rule.private_members.severity = suggestion
+dotnet_naming_rule.private_members_snake_case.symbols = private_symbols
+dotnet_naming_rule.private_members_snake_case.style = private_snake_case
+dotnet_naming_rule.private_members_snake_case.severity = suggestion

-dotnet_naming_rule.variables_are_camel_case.severity = suggestion
-dotnet_naming_rule.variables_are_camel_case.symbols = local_symbols
-dotnet_naming_rule.variables_are_camel_case.style = local
-
-dotnet_naming_rule.parameters.symbols = parameter_symbols
-dotnet_naming_rule.parameters.style = parameter
+dotnet_naming_rule.parameters.symbols = parameters
+dotnet_naming_rule.parameters.style = snake_case
 dotnet_naming_rule.parameters.severity = suggestion
@@ -34,7 +34,7 @@
 #   src/server/sparkd/Dockerfile
 SPARKD__IMAGE=sparkd
 # The config dir for this service to use
-SPARKD__CONFIG_DIR=src/app_sparkd/configurations/dev
+SPARKD__CONFIG_DIR=src/server/sparkd/configurations/dev
 # Mount a specific volume or dir to "~/sparkdata/data" inside the docker.
 # Defaults to a volume named "sparkd_<config_port>" (e.g., sparkd_8000).
 # SPARKD__DATA_DIR=/home/bts/sparkdata/data
@@ -44,13 +44,5 @@ SPARKD__CONFIG_PORT=8000
 SPARKD__QUICKBUS_PORT=8001
 SPARKD__SIM_FIX_PORT=8002
 SPARKD__SHM_SIZE=4gb
-SPARKD__PYTHON_REQUIREMENTS=requirements.txt

-### PubSubServer ###
-PUBSUB__CONFIG_DIR=src/DataStreamSuite/PubSubServer/Configurations/Development
-PUBSUB__SOURCE_DIR=build/Release/bin/net8.0/linux-x64/publish
-
-### MdPublisher ###
-MD_PUBLISHER__CONFIG_DIR=src/DataStreamSuite/DataStreamPublisher/Configurations/ReplayMarketData
-MD_PUBLISHER__SOURCE_DIR=build/Release/bin/net8.0/linux-x64/publish
 # TODO: add config for other services (pubsub, md-publisher, webapp, api-server, etc.)
@@ -11,7 +11,6 @@ cmake-build-debug/
 **.vscode/*
 !.vscode/launch.json
 !.vscode/tasks.json
-!.vscode/*.env
 .vs
 obj/
 *.csproj.user
@@ -55,12 +54,9 @@ trace_data/
 .cache/
 /src/Spark.Admin.WebApp/tsconfig.*.g.json
 /src/Spark.Admin.WebApp/src/bin/
-/src/**/*appsettings.Local.json
-/src/**/*/Configurations/Local/
 **/CMakeDeps.cmake
 **/CMakeDeps_win.cmake
 .devcontainer/
 *.Generated.cs
 *.lst
 .env.local
-.secrets
@@ -1,62 +1,11 @@
 {
  "version": "0.2.0",
  "configurations": [
-    {
-      "name": "Python Debugger: Current File with Arguments",
-      "type": "debugpy",
-      "request": "launch",
-      "program": "${file}",
-      "console": "integratedTerminal",
-      "args": "${command:pickArgs}"
-    },
    {
      "name": ".NET Core Attach",
      "type": "coreclr",
      "request": "attach"
    },
-    {
-      "name": "Api Server",
-      "type": "coreclr",
-      "request": "launch",
-      "program": "",
-      "preLaunchTask": "Build Api Server (Debug)",
-      "windows": {
-        "program": "spark_api_server.exe",
-      },
-      "linux": {
-        "program": "spark_api_server",
-      },
-      "args": [
-        "run",
-      ],
-      "cwd": "build/Debug/bin/net8.0",
-      "env": {
-        "DOTNET_ENVIRONMENT": "${input:apiServerEnvironment}",
-      },
-      "stopAtEntry": false,
-      "console": "integratedTerminal"
-    },
-    {
-      "name": "Api Server (Skip Build)",
-      "type": "coreclr",
-      "request": "launch",
-      "program": "",
-      "windows": {
-        "program": "spark_api_server.exe",
-      },
-      "linux": {
-        "program": "spark_api_server",
-      },
-      "args": [
-        "run",
-      ],
-      "cwd": "build/Debug/bin/net8.0",
-      "env": {
-        "DOTNET_ENVIRONMENT": "${input:apiServerEnvironment}",
-      },
-      "stopAtEntry": false,
-      "console": "integratedTerminal"
-    },
    {
      "name": "DataStreamPublisher",
      "type": "coreclr",
@@ -69,28 +18,11 @@
      "windows": {
        "program": "DataStreamPublisher.exe",
      },
-      "cwd": "build/Debug/bin/net8.0",
-      "env": {
-        "DOTNET_ENVIRONMENT": "${input:dataStreamPublisherEnvironment}",
-      },
-      "stopAtEntry": false,
-      "console": "integratedTerminal"
-    },
-    {
-      "name": "DataStreamPublisher (Skip Build)",
-      "type": "coreclr",
-      "request": "launch",
-      "program": "",
-      "linux": {
-        "program": "DataStreamPublisher",
-      },
-      "windows": {
-        "program": "DataStreamPublisher.exe",
-      },
-      "cwd": "${workspaceFolder}/build/Debug/bin/net8.0",
-      "env": {
-        "DOTNET_ENVIRONMENT": "${input:dataStreamPublisherEnvironment}",
-      },
+      "args": [
+        "--config",
+        "${workspaceFolder}/src/DataStreamSuite/DataStreamPublisher/Configurations/${input:dataStreamPublisherConfig}.json"
+      ],
+      "cwd": "build/Debug/bin/net7.0",
      "stopAtEntry": false,
      "console": "integratedTerminal"
    },
@@ -142,29 +74,16 @@
      "name": "DataStreamQueryAnalyzer",
      "type": "coreclr",
      "request": "launch",
-      "preLaunchTask": "Build DataStreamQueryAnalyzer (Debug)",
+      //"preLaunchTask": "Build DataStreamQueryAnalyzer (Debug)",
      "program": "",
      "windows": {
        "program": "DataStreamQueryAnalyzer.exe",
-        "cwd": "build/Debug/bin/net8.0-windows",
+        "args": [
+          "--config",
+          "${workspaceFolder}/src/DataStreamSuite/DataStreamQueryAnalyzer/Configurations/DataStreamQueryAnalyzer.json"
+        ],
+        "cwd": "build/Debug/bin/net7.0-windows",
        "env": {
-          "DOTNET_ENVIRONMENT": "${input:dataStreamQueryAnalyzerEnvironment}",
-          "PYTHON_EXE": "${workspaceFolder}/.venv/Scripts/python.exe"
-        },
-      },
-      "stopAtEntry": false,
-      "console": "integratedTerminal",
-    },
-    {
-      "name": "DataStreamQueryAnalyzer (Skip Build)",
-      "type": "coreclr",
-      "request": "launch",
-      "program": "",
-      "windows": {
-        "program": "DataStreamQueryAnalyzer.exe",
-        "cwd": "build/Debug/bin/net8.0-windows",
-        "env": {
-          "DOTNET_ENVIRONMENT": "${input:dataStreamQueryAnalyzerEnvironment}",
          "PYTHON_EXE": "${workspaceFolder}/.venv/Scripts/python.exe"
        },
      },
@@ -187,28 +106,7 @@
        "--config",
        "${workspaceFolder}/src/DataStreamSuite/PubSubServer/Configurations/${input:pubSubServerConfig}.json"
      ],
-      "cwd": "build/Debug/bin/net8.0",
-      "env": {
-        "DOTNET_ENVIRONMENT": "${input:pubSubServerEnvironment}",
-      },
-      "stopAtEntry": false,
-      "console": "integratedTerminal"
-    },
-    {
-      "name": "PubSubServer (Skip Build)",
-      "type": "coreclr",
-      "request": "launch",
-      "program": "",
-      "linux": {
-        "program": "PubSubServer",
-      },
-      "windows": {
-        "program": "PubSubServer.exe",
-      },
-      "cwd": "build/Debug/bin/net8.0",
-      "env": {
-        "DOTNET_ENVIRONMENT": "${input:pubSubServerEnvironment}",
-      },
+      "cwd": "build/Debug/bin/net7.0",
      "stopAtEntry": false,
      "console": "integratedTerminal"
    },
@@ -220,10 +118,7 @@
      "windows": {
        "preLaunchTask": "Build Spark (Debug)",
        "program": "Spark.exe",
-        "cwd": "build/Debug/bin/net8.0-windows",
-        "env": {
-          "DOTNET_ENVIRONMENT": "${input:sparkEnvironment}",
-        }
+        "cwd": "build/Debug/bin/net7.0-windows",
      },
      "stopAtEntry": false,
      "console": "integratedTerminal"
@@ -235,10 +130,7 @@
      "program": "",
      "windows": {
        "program": "Spark.exe",
-        "cwd": "build/Debug/bin/net8.0-windows",
-        "env": {
-          "DOTNET_ENVIRONMENT": "${input:sparkEnvironment}",
-        }
+        "cwd": "build/Debug/bin/net7.0-windows",
      },
      "stopAtEntry": false,
      "console": "integratedTerminal"
@@ -255,7 +147,7 @@
          "--sparkd",
          "${input:sparkConfigServer}"
        ],
-        "cwd": "build/Debug/bin/net8.0-windows",
+        "cwd": "build/Debug/bin/net7.0-windows",
      },
      "stopAtEntry": false,
      "console": "integratedTerminal"
@@ -268,7 +160,7 @@
      "windows": {
        "preLaunchTask": "Build Spark (Debug)",
        "program": "Spark.Console.exe",
-        "cwd": "build/Debug/bin/net8.0",
+        "cwd": "build/Debug/bin/net7.0",
        "args": [
          "test",
          "${input:sparkTest}"
@@ -284,7 +176,7 @@
      "program": "",
      "windows": {
        "program": "Spark.Console.exe",
-        "cwd": "build/Debug/bin/net8.0",
+        "cwd": "build/Debug/bin/net7.0",
        "args": [
          "test",
          "${input:sparkTest}"
@@ -299,27 +191,24 @@
      "request": "launch",
      "program": "",
      "linux": {
-        "program": "${workspaceFolder}/build/sparkd-debug-clang/src/app_sparkd/sparkd",
-      },
-      "args": [
-        "run"
-      ],
-      "cwd": "${workspaceFolder}/build/sparkd-debug-clang/src/app_sparkd",
-      "stopAtEntry": false,
-      "externalConsole": false,
-      "MIMode": "gdb",
-      "setupCommands": [
-        {
-          "description": "Enable pretty-printing for gdb",
-          "text": "-enable-pretty-printing",
-          "ignoreFailures": true
-        },
-        {
-          "description": "Set Disassembly Flavor to Intel",
-          "text": "-gdb-set disassembly-flavor intel",
-          "ignoreFailures": true
-        }
-      ]
+        "preLaunchTask": "Build Sparkd (Debug)",
+        "program": "${workspaceFolder}/build/sparkd-debug-clang/server/sparkd",
+        "args": [
+          "run"
+        ],
+        "stopAtEntry": false,
+        "cwd": "${workspaceFolder}/build/sparkd-debug-clang/server",
+        "environment": [],
+        "externalConsole": false,
+        "MIMode": "gdb",
+        "setupCommands": [
+          {
+            "description": "Enable pretty-printing for gdb",
+            "text": "-enable-pretty-printing",
+            "ignoreFailures": true
+          }
+        ]
+      }
    },
    {
      "name": "Sparkd (Skip Build)",
@@ -327,41 +216,77 @@
      "request": "launch",
      "program": "",
      "linux": {
-        "program": "${workspaceFolder}/build/sparkd-debug-clang/src/app_sparkd/sparkd",
-      },
-      "args": [
-        "run"
-      ],
-      "cwd": "${workspaceFolder}/build/sparkd-debug-clang/src/app_sparkd",
-      "stopAtEntry": false,
-      "externalConsole": false,
-      "MIMode": "gdb",
-      "setupCommands": [
-        {
-          "description": "Enable pretty-printing for gdb",
-          "text": "-enable-pretty-printing",
-          "ignoreFailures": true
-        },
-        {
-          "description": "Set Disassembly Flavor to Intel",
-          "text": "-gdb-set disassembly-flavor intel",
-          "ignoreFailures": true
-        }
-      ]
+        "program": "${workspaceFolder}/build/sparkd-debug-clang/server/sparkd",
+        "args": [
+          "run"
+        ],
+        "stopAtEntry": false,
+        "cwd": "${workspaceFolder}/build/sparkd-debug-clang/server",
+        "environment": [],
+        "externalConsole": false,
+        "MIMode": "gdb",
+        "setupCommands": [
+          {
+            "description": "Enable pretty-printing for gdb",
+            "text": "-enable-pretty-printing",
+            "ignoreFailures": true
+          }
+        ]
+      }
+    },
+    {
+      "name": "Sparkd Test",
+      "type": "cppdbg",
+      "request": "launch",
+      "program": "",
+      "linux": {
+        "preLaunchTask": "Build Sparkd (Debug)",
+        "program": "${workspaceFolder}/build/sparkd-debug-clang/server/sparkd",
+        "args": [
+          "test",
+          "${input:sparkdTest}",
+        ],
+        "stopAtEntry": false,
+        "cwd": "${workspaceFolder}/build/sparkd-debug-clang/server",
+        "environment": [],
+        "externalConsole": false,
+        "MIMode": "gdb",
+        "setupCommands": [
+          {
+            "description": "Enable pretty-printing for gdb",
+            "text": "-enable-pretty-printing",
+            "ignoreFailures": true
+          }
+        ]
+      }
+    },
+    {
+      "name": "Sparkd Test (Skip Build)",
+      "type": "cppdbg",
+      "request": "launch",
+      "program": "",
+      "linux": {
+        "program": "${workspaceFolder}/build/sparkd-debug-clang/server/sparkd",
+        "args": [
+          "test",
+          "${input:sparkdTest}",
+        ],
+        "stopAtEntry": false,
+        "cwd": "${workspaceFolder}/build/sparkd-debug-clang/server",
+        "environment": [],
+        "externalConsole": false,
+        "MIMode": "gdb",
+        "setupCommands": [
+          {
+            "description": "Enable pretty-printing for gdb",
+            "text": "-enable-pretty-printing",
+            "ignoreFailures": true
+          }
+        ]
+      }
    },
  ],
  "inputs": [
-    {
-      "id": "apiServerEnvironment",
-      "type": "pickString",
-      "description": "Api Server Configuration",
-      "options": [
-        "Development",
-        "Local",
-        "Staging",
-      ],
-      "default": "Development",
-    },
    {
      "id": "dataStreamPublisherConfig",
      "type": "pickString",
@@ -374,16 +299,6 @@
      ],
      "default": "MarketDataPublisher",
    },
-    {
-      "id": "dataStreamPublisherEnvironment",
-      "type": "pickString",
-      "description": "DataStreamPublisher Configuration",
-      "options": [
-        "Local",
-        "ReplayMarketData",
-      ],
-      "default": "Local",
-    },
    {
      "id": "pubSubServerConfig",
      "type": "pickString",
@@ -394,37 +309,6 @@
      ],
      "default": "PubSubServer",
    },
-    {
-      "id": "pubSubServerEnvironment",
-      "type": "pickString",
-      "description": "Api Server Configuration",
-      "options": [
-        "Development",
-        "Local",
-        "Staging",
-      ],
-      "default": "Development",
-    },
-    {
-      "id": "dataStreamQueryAnalyzerEnvironment",
-      "type": "pickString",
-      "description": "DataStreamQueryAnalyzer Configuration",
-      "options": [
-        "Development",
-        "Local",
-      ],
-      "default": "Development",
-    },
-    {
-      "id": "sparkEnvironment",
-      "type": "pickString",
-      "description": "Spark Configuration",
-      "options": [
-        "Development",
-        "Local",
-      ],
-      "default": "Development",
-    },
    {
      "id": "sparkConfigServer",
      "type": "promptString",
@@ -442,4 +326,4 @@
      "description": "Test to run",
    },
  ]
-}
+}
@@ -1 +0,0 @@
-PYTHONPATH=/workspaces/spark/build/sparkd-debug-clang/src/lib_python_interop;/workspaces/spark/src/lib_python_interop/python_interop_plugins;/workspaces/spark/tools/py
@@ -121,57 +121,6 @@
        "color": "terminal.ansiCyan"
      },
    },
-    {
-      "label": "Build DataStreamQueryAnalyzer",
-      "group": "build",
-      "type": "process",
-      "command": "",
-      "windows": {
-        "command": "tools/build_data_stream_query_analyzer.cmd",
-      },
-      "args": [
-        "--configuration",
-        "${input:buildConfiguration}",
-      ],
-      "icon": {
-        "id": "terminal",
-        "color": "terminal.ansiCyan"
-      },
-    },
-    {
-      "label": "Build DataStreamQueryAnalyzer (Debug)",
-      "group": "build",
-      "type": "process",
-      "command": "",
-      "windows": {
-        "command": "tools/build_data_stream_query_analyzer.cmd",
-      },
-      "args": [
-        "--configuration",
-        "Debug",
-      ],
-      "icon": {
-        "id": "terminal",
-        "color": "terminal.ansiCyan"
-      },
-    },
-    {
-      "label": "Build DataStreamQueryAnalyzer (Release)",
-      "group": "build",
-      "type": "process",
-      "command": "",
-      "windows": {
-        "command": "tools/build_data_stream_query_analyzer.cmd",
-      },
-      "args": [
-        "--configuration",
-        "Release",
-      ],
-      "icon": {
-        "id": "terminal",
-        "color": "terminal.ansiCyan"
-      },
-    },
    {
      "label": "Build Spark",
      "group": "build",
@@ -402,132 +351,9 @@
        "id": "wrench"
      },
    },
-    {
-      "label": "Run Api Server",
-      "dependsOn": "Build Api Server",
-      "type": "process",
-      "command": "",
-      "windows": {
-        "command": "spark_api_server.exe",
-      },
-      "linux": {
-        "command": "spark_api_server",
-      },
-      "args": [
-        "run",
-      ],
-      "options": {
-        "cwd": "${workspaceFolder}/build/${input:buildConfiguration}/bin/net8.0",
-        "env": {
-          "DOTNET_ENVIRONMENT": "${input:apiServerEnvironment}",
-        }
-      },
-      "icon": {
-        "id": "run",
-        "color": "terminal.ansiGreen",
-      },
-    },
-    {
-      "label": "Run Api Server (Debug)",
-      "dependsOn": "Build Api Server (Debug)",
-      "type": "process",
-      "command": "",
-      "windows": {
-        "command": "spark_api_server.exe",
-      },
-      "linux": {
-        "command": "spark_api_server",
-      },
-      "args": [
-        "run",
-      ],
-      "options": {
-        "cwd": "${workspaceFolder}/build/Debug/bin/net8.0",
-        "env": {
-          "DOTNET_ENVIRONMENT": "${input:apiServerEnvironment}",
-        }
-      },
-      "icon": {
-        "id": "run",
-        "color": "terminal.ansiGreen",
-      },
-    },
-    {
-      "label": "Run Api Server (Release)",
-      "dependsOn": "Build Api Server (Release)",
-      "type": "process",
-      "command": "",
-      "windows": {
-        "command": "spark_api_server.exe",
-      },
-      "linux": {
-        "command": "spark_api_server",
-      },
-      "args": [
-        "run",
-      ],
-      "options": {
-        "cwd": "${workspaceFolder}/build/Release/bin/net8.0",
-        "env": {
-          "DOTNET_ENVIRONMENT": "${input:apiServerEnvironment}",
-        }
-      },
-      "icon": {
-        "id": "run",
-        "color": "terminal.ansiGreen",
-      },
-    },
-    {
-      "label": "Run Api Server (Debug | Skip Build)",
-      "type": "process",
-      "command": "",
-      "windows": {
-        "command": "spark_api_server.exe",
-      },
-      "linux": {
-        "command": "spark_api_server",
-      },
-      "args": [
-        "run",
-      ],
-      "options": {
-        "cwd": "${workspaceFolder}/build/Debug/bin/net8.0",
-        "env": {
-          "DOTNET_ENVIRONMENT": "${input:apiServerEnvironment}",
-        }
-      },
-      "icon": {
-        "id": "run",
-        "color": "terminal.ansiGreen",
-      },
-    },
-    {
-      "label": "Run Api Server (Release | Skip Build)",
-      "type": "process",
-      "command": "",
-      "windows": {
-        "command": "spark_api_server.exe",
-      },
-      "linux": {
-        "command": "spark_api_server",
-      },
-      "args": [
-        "run",
-      ],
-      "options": {
-        "cwd": "${workspaceFolder}/build/Release/bin/net8.0",
-        "env": {
-          "DOTNET_ENVIRONMENT": "${input:apiServerEnvironment}",
-        }
-      },
-      "icon": {
-        "id": "run",
-        "color": "terminal.ansiGreen",
-      },
-    },
    {
      "label": "Run DataStreamQueryAnalyzer",
-      "dependsOn": "Build DataStreamQueryAnalyzer",
+      "dependsOn": "Build DataStream",
      "type": "process",
      "command": "",
      "windows": {
@@ -537,7 +363,7 @@
          "${workspaceFolder}/src/DataStreamSuite/DataStreamQueryAnalyzer/Configurations/DataStreamQueryAnalyzer.json",
        ],
        "options": {
-          "cwd": "${workspaceFolder}/build/${input:buildConfiguration}/bin/net8.0-windows"
+          "cwd": "${workspaceFolder}/build/${input:buildConfiguration}/bin/net7.0-windows"
        },
      },
      "icon": {
@@ -547,7 +373,7 @@
    },
    {
      "label": "Run DataStreamQueryAnalyzer (Debug)",
-      "dependsOn": "Build DataStreamQueryAnalyzer (Debug)",
+      "dependsOn": "Build DataStream (Debug)",
      "type": "process",
      "command": "",
      "windows": {
@@ -557,7 +383,7 @@
          "${workspaceFolder}/src/DataStreamSuite/DataStreamQueryAnalyzer/Configurations/DataStreamQueryAnalyzer.json",
        ],
        "options": {
-          "cwd": "${workspaceFolder}/build/Debug/bin/net8.0-windows"
+          "cwd": "${workspaceFolder}/build/Debug/bin/net7.0-windows"
        },
      },
      "icon": {
@@ -567,7 +393,7 @@
    },
    {
      "label": "Run DataStreamQueryAnalyzer (Release)",
-      "dependsOn": "Build DataStreamQueryAnalyzer (Release)",
+      "dependsOn": "Build DataStream (Release)",
      "type": "process",
      "command": "",
      "windows": {
@@ -577,7 +403,7 @@
          "${workspaceFolder}/src/DataStreamSuite/DataStreamQueryAnalyzer/Configurations/DataStreamQueryAnalyzer.json",
        ],
        "options": {
-          "cwd": "${workspaceFolder}/build/Release/bin/net8.0-windows"
+          "cwd": "${workspaceFolder}/build/Release/bin/net7.0-windows"
        },
      },
      "icon": {
@@ -596,11 +422,12 @@
      "windows": {
        "command": "DataStreamPublisher.exe",
      },
+      "args": [
+        "--config",
+        "${workspaceFolder}/src/DataStreamSuite/DataStreamPublisher/Configurations/${input:dataStreamPublisherConfig}.json",
+      ],
      "options": {
-        "cwd": "${workspaceFolder}/build/${input:buildConfiguration}/bin/net8.0",
-        "env": {
-          "DOTNET_ENVIRONMENT": "${input:dataStreamPublisherEnvironment}",
-        },
+        "cwd": "${workspaceFolder}/build/${input:buildConfiguration}/bin/net7.0"
      },
      "icon": {
        "id": "run",
@@ -618,11 +445,12 @@
      "windows": {
        "command": "DataStreamPublisher.exe",
      },
+      "args": [
+        "--config",
+        "${workspaceFolder}/src/DataStreamSuite/DataStreamPublisher/Configurations/${input:dataStreamPublisherConfig}.json",
+      ],
      "options": {
-        "cwd": "${workspaceFolder}/build/Debug/bin/net8.0",
-        "env": {
-          "DOTNET_ENVIRONMENT": "${input:dataStreamPublisherEnvironment}",
-        },
+        "cwd": "${workspaceFolder}/build/Debug/bin/net7.0"
      },
      "icon": {
        "id": "run",
@@ -640,11 +468,12 @@
      "windows": {
        "command": "DataStreamPublisher.exe",
      },
+      "args": [
+        "--config",
+        "${workspaceFolder}/src/DataStreamSuite/DataStreamPublisher/Configurations/${input:dataStreamPublisherConfig}.json",
+      ],
      "options": {
-        "cwd": "${workspaceFolder}/build/Release/bin/net8.0",
-        "env": {
-          "DOTNET_ENVIRONMENT": "${input:dataStreamPublisherEnvironment}",
-        },
+        "cwd": "${workspaceFolder}/build/Release/bin/net7.0"
      },
      "icon": {
        "id": "run",
@@ -662,11 +491,12 @@
      "windows": {
        "command": "PubSubServer.exe",
      },
+      "args": [
+        "--config",
+        "${workspaceFolder}/src/DataStreamSuite/PubSubServer/Configurations/${input:pubSubServerConfig}.json",
+      ],
      "options": {
-        "cwd": "${workspaceFolder}/build/${input:buildConfiguration}/bin/net8.0",
-        "env": {
-          "DOTNET_ENVIRONMENT": "${input:pubSubServerEnvironment}",
-        },
+        "cwd": "${workspaceFolder}/build/${input:buildConfiguration}/bin/net7.0"
      },
      "icon": {
        "id": "run",
@@ -684,11 +514,12 @@
      "windows": {
        "command": "PubSubServer.exe",
      },
+      "args": [
+        "--config",
+        "${workspaceFolder}/src/DataStreamSuite/PubSubServer/Configurations/${input:pubSubServerConfig}.json",
+      ],
      "options": {
-        "cwd": "${workspaceFolder}/build/Debug/bin/net8.0",
-        "env": {
-          "DOTNET_ENVIRONMENT": "${input:pubSubServerEnvironment}",
-        },
+        "cwd": "${workspaceFolder}/build/Debug/bin/net7.0"
      },
      "icon": {
        "id": "run",
@@ -705,11 +536,12 @@
      "windows": {
        "command": "PubSubServer.exe",
      },
+      "args": [
+        "--config",
+        "${workspaceFolder}/src/DataStreamSuite/PubSubServer/Configurations/${input:pubSubServerConfig}.json",
+      ],
      "options": {
-        "cwd": "${workspaceFolder}/build/Debug/bin/net8.0",
-        "env": {
-          "DOTNET_ENVIRONMENT": "${input:pubSubServerEnvironment}",
-        },
+        "cwd": "${workspaceFolder}/build/Debug/bin/net7.0"
      },
      "icon": {
        "id": "run",
@@ -727,11 +559,12 @@
      "windows": {
        "command": "PubSubServer.exe",
      },
+      "args": [
+        "--config",
+        "${workspaceFolder}/src/DataStreamSuite/PubSubServer/Configurations/${input:pubSubServerConfig}.json",
+      ],
      "options": {
-        "cwd": "${workspaceFolder}/build/Release/bin/net8.0",
-        "env": {
-          "DOTNET_ENVIRONMENT": "${input:pubSubServerEnvironment}",
-        },
+        "cwd": "${workspaceFolder}/build/Release/bin/net7.0"
      },
      "icon": {
        "id": "run",
@@ -748,11 +581,12 @@
      "windows": {
        "command": "PubSubServer.exe",
      },
+      "args": [
+        "--config",
+        "${workspaceFolder}/src/DataStreamSuite/PubSubServer/Configurations/${input:pubSubServerConfig}.json",
+      ],
      "options": {
-        "cwd": "${workspaceFolder}/build/Release/bin/net8.0",
-        "env": {
-          "DOTNET_ENVIRONMENT": "${input:pubSubServerEnvironment}",
-        },
+        "cwd": "${workspaceFolder}/build/Release/bin/net7.0"
      },
      "icon": {
        "id": "run",
@@ -767,7 +601,7 @@
      "windows": {
        "command": "Spark.exe",
        "options": {
-          "cwd": "${workspaceFolder}/build/${input:buildConfiguration}/bin/net8.0-windows"
+          "cwd": "${workspaceFolder}/build/${input:buildConfiguration}/bin/net7.0-windows"
        },
      },
      "icon": {
@@ -783,7 +617,7 @@
      "windows": {
        "command": "Spark.exe",
        "options": {
-          "cwd": "${workspaceFolder}/build/Debug/bin/net8.0-windows"
+          "cwd": "${workspaceFolder}/build/Debug/bin/net7.0-windows"
        },
      },
      "icon": {
@@ -799,7 +633,7 @@
      "windows": {
        "command": "Spark.exe",
        "options": {
-          "cwd": "${workspaceFolder}/build/Release/bin/net8.0-windows"
+          "cwd": "${workspaceFolder}/build/Release/bin/net7.0-windows"
        },
      },
      "icon": {
@@ -814,7 +648,7 @@
      "windows": {
        "command": "Spark.exe",
        "options": {
-          "cwd": "${workspaceFolder}/build/${input:buildConfiguration}/bin/net8.0-windows"
+          "cwd": "${workspaceFolder}/build/${input:buildConfiguration}/bin/net7.0-windows"
        },
      },
      "icon": {
@@ -839,7 +673,7 @@
        "${input:sparkConfigQuery}",
      ],
      "options": {
-        "cwd": "${workspaceFolder}/build/${input:buildConfiguration}/bin/net8.0"
+        "cwd": "${workspaceFolder}/build/${input:buildConfiguration}/bin/net7.0"
      },
      "icon": {
        "id": "run",
@@ -920,7 +754,7 @@
          "${input:sparkTest}"
        ],
        "options": {
-          "cwd": "${workspaceFolder}/build/Debug/bin/net8.0",
+          "cwd": "${workspaceFolder}/build/Debug/bin/net7.0",
        }
      },
    },
@@ -1018,17 +852,6 @@
    },
  ],
  "inputs": [
-    {
-      "id": "apiServerEnvironment",
-      "type": "pickString",
-      "description": "Api Server Configuration",
-      "options": [
-        "Development",
-        "Local",
-        "Staging",
-      ],
-      "default": "Development",
-    },
    {
      "id": "buildConfiguration",
      "type": "pickString",
@@ -1062,31 +885,26 @@
      "default": "sparkd-debug-clang",
    },
    {
-      "id": "dataStreamPublisherEnvironment",
+      "id": "dataStreamPublisherConfig",
      "type": "pickString",
      "description": "Publisher Configuration",
      "options": [
-        "Development",
-        "MarketData",
-        "Production",
-        "ProductionAws",
-        "Staging",
+        "MarketDataPublisher",
+        "SimpleEchoClient",
+        "SimpleTestDataPublisher",
+        "TestDataPublisher",
      ],
      "default": "MarketDataPublisher",
    },
    {
-      "id": "pubSubServerEnvironment",
+      "id": "pubSubServerConfig",
      "type": "pickString",
      "description": "PubSub Configuration",
      "options": [
-        "Development",
-        "EchoServer",
-        "Local",
-        "Production",
-        "ProductionAws",
-        "Staging",
+        "PubSubServer",
+        "SimpleEchoClient",
      ],
-      "default": "Development",
+      "default": "PubSubServer",
    },
    {
      "id": "sparkConfigQuery",
@@ -21,10 +21,10 @@ set(
 # SHARED_MEMORY_PREFIX serves as a kind of namespace so multiple applications
 # can run simultaneously without fighting over the same shm files (e.g., sparkd,
 # api server, etc.).
-set(SHARED_MEMORY_PREFIX "\${BTS_SHARED_MEMORY_PREFIX}" CACHE STRING "Prefix to use when creating shared memory files")
-set(EXTERNALS_BIN "${EXTERNALS_ROOT}/bin")
-set(EXTERNALS_INCLUDE "${EXTERNALS_ROOT}/include")
-set(EXTERNALS_LIB "${EXTERNALS_ROOT}/lib/$<CONFIG>")
+set(SHARED_MEMORY_PREFIX "bts" CACHE STRING "Prefix to use when creating shared memory files")
+set(SPARK_BIN "${EXTERNALS_ROOT}/bin")
+set(SPARK_INCLUDE "${EXTERNALS_ROOT}/include")
+set(SPARK_LIB "${EXTERNALS_ROOT}/lib/$<CONFIG>")
 set(BUILD_DOCS OFF CACHE BOOL "Build documentation")
 set(BUILD_SDK OFF CACHE BOOL "Build the sdk")
 set(EXPORT_MD_CONSUMER OFF CACHE BOOL "Export (extern) C functions for MD consumer interop")
@@ -36,18 +36,23 @@ else()
  set(BUILD_ADMIN_WEB_APP ON CACHE BOOL "Build the Admin WebApp")
 endif()

-include_directories(BEFORE SYSTEM ${EXTERNALS_INCLUDE} ${EXTERNALS_ROOT}/include/zlib)
-link_directories(${EXTERNALS_LIB})
+include_directories(BEFORE SYSTEM ${SPARK_INCLUDE} ${EXTERNALS_ROOT}/include/zlib)
+link_directories(${SPARK_LIB})

 add_compile_definitions(SHARED_MEMORY_PREFIX="${SHARED_MEMORY_PREFIX}")
-if(DEFINED APP_DATA_DIR)
-  add_compile_definitions(APP_DATA_DIR="${APP_DATA_DIR}")
+add_compile_definitions(APP_DATA_DIR="${APP_DATA_DIR}")
+if(DEFINED APP_DATA_DIR_OVERRIDE)
+  add_compile_definitions(APP_DATA_DIR_OVERRIDE="${APP_DATA_DIR_OVERRIDE}")
 endif()
 add_compile_definitions(EXPORT_MD_CONSUMER="${EXPORT_MD_CONSUMER}")
 if(WIN32)
  # subdirs
-  add_subdirectory(src/lib_spark)
+  add_subdirectory(src/lib)
 else()
+  # make sparkd look for libspark.so in its directory
+  set(CMAKE_BUILD_WITH_INSTALL_RPATH TRUE)
+  set(CMAKE_INSTALL_RPATH ".")
+
  if(BUILD_SPARKD)
    # Note: All macros that begin w/ 'CCAPI' are ccapi's macros (defined and/or
    # read by their code). We override some below.
@@ -112,15 +117,12 @@ else()
  endif()

  # subdirs
-  add_subdirectory(src/lib_spark)
  if (BUILD_SPARKD)
-    add_subdirectory(src/lib_python_interop)
-    add_subdirectory(src/lib_onixs_fix)
-    add_subdirectory(src/lib_binance)
-    add_subdirectory(src/lib_okx_execution)
-    add_subdirectory(src/app_sparkd)
+    add_subdirectory(src/lib_binance_execution)
+    add_subdirectory(src/server/sparkd)
    add_subdirectory(src/data_files)
  endif()
+  add_subdirectory(src/lib)
  if (BUILD_ADMIN_WEB_APP)
    add_subdirectory(src/Spark.Admin.WebApp)
  endif()
@@ -130,4 +132,37 @@ else()
  if(BUILD_SDK)
    add_subdirectory(src/sdk)
  endif()
+
+  if(BUILD_SPARKD)
+    add_custom_command(
+      OUTPUT ${CMAKE_BINARY_DIR}/server
+      COMMAND mkdir -p ${CMAKE_BINARY_DIR}/server
+      COMMAND ln ARGS -sf ${CMAKE_BINARY_DIR}/src/server/sparkd/sparkd ${CMAKE_BINARY_DIR}/server
+      COMMAND ln ARGS -sf ${CMAKE_BINARY_DIR}/data_files ${CMAKE_BINARY_DIR}/server
+      COMMAND ln ARGS -sf ${CMAKE_BINARY_DIR}/src/lib/libspark.so ${CMAKE_BINARY_DIR}/server
+      COMMAND ln ARGS -sf ${EXTERNALS_ROOT}/lib/${CMAKE_BUILD_TYPE}/libonixs-fix-engine.so.4 ${CMAKE_BINARY_DIR}/server
+      
+      COMMAND
+        ln ARGS -sf
+        ${EXTERNALS_ROOT}/lib/Release/libOnixS.IceImpactMulticastPriceFeedHandlerCpp.so.3.16.0.0
+        ${CMAKE_BINARY_DIR}/server
+      COMMAND ln ARGS -sf ${EXTERNALS_ROOT}/lib/Release/libACE-5.7.5.so ${CMAKE_BINARY_DIR}/server
+      COMMAND mkdir ARGS -p ${CMAKE_BINARY_DIR}/server/plugins
+      COMMENT "Creating server runtime layout"
+      DEPENDS sparkd data_files_dir)
+
+    add_custom_target(server_layout ALL DEPENDS ${CMAKE_BINARY_DIR}/server)
+
+    install(
+      DIRECTORY "${CMAKE_BINARY_DIR}/server/plugins"
+      DESTINATION "${CMAKE_INSTALL_BINDIR}")
+    install(
+      FILES
+      "${SPARK_LIB}/libACE-5.7.5.so"
+      "${SPARK_LIB}/libOnixS.IceImpactMulticastPriceFeedHandlerCpp.so.3.16.0.0"
+      "${SPARK_LIB}/libonixs-fix-engine.so.4.10.1"
+      "${SPARK_LIB}/libonixs-fix-engine.so.4"
+      "${SPARK_LIB}/libonixs-fix-engine.so"
+      DESTINATION "${CMAKE_INSTALL_BINDIR}")
+  endif()
 endif()
@@ -20,6 +20,8 @@
        "SPARK_CMAKE_CXX_FLAGS": "/MP /WX /wd4250 /wd4251 /wd4244 /wd4267 /wd4355 /wd4275 /wd4345 /bigobj /EHsc /DCURL_STATICLIB /D_WINSOCK_DEPRECATED_NO_WARNINGS /D__TBB_NO_IMPLICIT_LINKAGE /DBOOST_ALL_NO_LIB /std:c++20 /D_SCL_SECURE_NO_WARNINGS /D_CRT_SECURE_NO_WARNINGS /D_WIN32_WINNT=0x0600"
      },
      "cacheVariables": {
+        "APP_DATA_DIR": "${dollar}{APPDATA}/bts",
+        "APP_DATA_DIR_OVERRIDE": "${dollar}{SPARK_DATA_DIR}",
        "CMAKE_CXX_FLAGS": "$env{SPARK_CMAKE_CXX_FLAGS}",
        "CMAKE_CXX_FLAGS_DEBUG": "$env{SPARK_CMAKE_CXX_FLAGS} /MTd",
        "CMAKE_CXX_FLAGS_RELEASE": "$env{SPARK_CMAKE_CXX_FLAGS} /MT",
@@ -29,12 +31,14 @@
        "CMAKE_SHARED_LINKER_FLAGS_RELEASE": "/OPT:REF /OPT:ICF /LTCG",
        "CMAKE_TOOLCHAIN_FILE": "${sourceDir}/vcpkg/scripts/buildsystems/vcpkg.cmake",
        "EXTERNALS_ROOT": "${sourceDir}/externals/windows",
-        "CONFIG_DEFAULTS_PATH": "spark/config/profiles/default/*.conf",
-        "DOTNET_BINARY_DIR": "${sourceDir}/build/$<CONFIG>/bin/net8.0;${sourceDir}/build/$<CONFIG>/bin/net8.0-windows",
+        "CONFIG_DEFAULTS_PATH": "config/profiles/default/*.conf",
+        "DOTNET_BINARY_DIR": "${sourceDir}/build/$<CONFIG>/bin/net7.0",
+        "DOTNET_WINDOWS_BINARY_DIR": "${sourceDir}/build/$<CONFIG>/bin/net7.0-windows",
        "EXPORT_MD_CONSUMER": {
          "type": "BOOL",
          "value": "ON"
-        }
+        },
+        "SHARED_MEMORY_PREFIX": "spark"
      }
    },

@@ -50,20 +54,16 @@
      "binaryDir": "${sourceDir}/build/${presetName}",
      "cacheVariables": {
        "platform": "",
+        "APP_DATA_DIR": "${dollar}{HOME}/sparkdata",
+        "APP_DATA_DIR_OVERRIDE": "${dollar}{SPARKD_DATA_DIR}",
        "CMAKE_BUILD_WITH_INSTALL_RPATH": {
          "type": "BOOL",
-          "value": "OFF"
+          "value": "ON"
        },
        "CMAKE_CXX_STANDARD": "20",
        "CMAKE_INSTALL_RPATH": ".",
        "CMAKE_INSTALL_PREFIX": "${sourceDir}/install/${presetName}",
-        "CMAKE_PREFIX_PATH": "${sourceDir}/cmake/;${sourceDir}/.venv/lib/python3.10/site-packages/pybind11/share/cmake/pybind11",
        "CMAKE_EXE_LINKER_FLAGS": "-Wno-undef",
-        "CMAKE_TOOLCHAIN_FILE": "${sourceDir}/vcpkg/scripts/buildsystems/vcpkg.cmake",
-        "DATABENTO_USE_EXTERNAL_JSON": {
-          "type": "BOOL",
-          "value": "ON"
-        },
        "EXTERNALS_ROOT": "${sourceDir}/externals/linux",
        "BUILD_SDK": {
          "type": "BOOL",
@@ -72,7 +72,8 @@
        "BUILD_ADMIN_WEB_APP": {
          "type": "BOOL",
          "value": "ON"
-        }
+        },
+        "SHARED_MEMORY_PREFIX": "sparkd"
      }
    },

@@ -87,7 +88,7 @@
          "value": "ON"
        },
        "CCAPI_LOG_LEVEL": "Info",
-        "CONFIG_DEFAULTS_PATH": "spark/config/profiles/md_dev/*.conf;spark/config/profiles/default/*.conf"
+        "CONFIG_DEFAULTS_PATH": "config/profiles/md_dev/*.conf;config/profiles/default/*.conf"
      }
    },

@@ -98,7 +99,7 @@
      "cacheVariables": {
        "CMAKE_BUILD_TYPE": "Release",
        "CCAPI_LOG_LEVEL": "Warn",
-        "CONFIG_DEFAULTS_PATH": "spark/config/profiles/md_prod/*.conf;spark/config/profiles/default/*.conf"
+        "CONFIG_DEFAULTS_PATH": "config/profiles/md_prod/*.conf;config/profiles/default/*.conf"
      }
    },

@@ -153,7 +154,11 @@
    {
      "name": "spark-api-server-windows",
      "inherits": "spark",
-      "displayName": "Spark Api Server (Windows)"
+      "displayName": "Spark Api Server (Windows)",
+      "cacheVariables": {
+        "APP_DATA_DIR_OVERRIDE": "${dollar}{SPARK_API_SERVER_DATA_DIR}",
+        "SHARED_MEMORY_PREFIX": "spark_api_server"
+      }
    },

    {
@@ -162,6 +167,7 @@
      "displayName": "Spark Api Server (Linux)",
      "generator": "Ninja Multi-Config",
      "cacheVariables": {
+        "APP_DATA_DIR_OVERRIDE": "${dollar}{SPARK_API_SERVER_DATA_DIR}",
        "BUILD_ADMIN_WEB_APP": {
          "type": "BOOL",
          "value": "OFF"
@@ -175,12 +181,8 @@
        "CMAKE_CXX_FLAGS_DEBUG": "-g -fPIC -Wall -pedantic -Wextra -Werror -Wno-missing-braces -Wno-ignored-qualifiers --std=c++20 -fdiagnostics-color -DDEBUG",
        "CMAKE_CXX_FLAGS_RELEASE": "-fPIC -Wall -pedantic -Wextra -Werror -Wno-missing-braces -Wno-ignored-qualifiers --std=c++20 -fdiagnostics-color -fno-omit-frame-pointer -DNDEBUG -O3",
        "CMAKE_INSTALL_PREFIX": "${sourceDir}/install/${presetName}",
-        "CMAKE_EXPORT_COMPILE_COMMANDS": {
-          "type": "BOOL",
-          "value": "ON"
-        },
-        "CONFIG_DEFAULTS_PATH": "spark/config/profiles/default/*.conf",
-        "DOTNET_BINARY_DIR": "${sourceDir}/build/$<CONFIG>/bin/net8.0",
+        "CONFIG_DEFAULTS_PATH": "config/profiles/default/*.conf",
+        "DOTNET_BINARY_DIR": "${sourceDir}/build/$<CONFIG>/bin/net7.0",
        "EXPORT_MD_CONSUMER": {
          "type": "BOOL",
          "value": "ON"
@@ -218,8 +220,10 @@
      "description": "Data Stream utils ported over from Edge",
      "binaryDir": "${sourceDir}/build/${presetName}",
      "cacheVariables": {
-        "CONFIG_DEFAULTS_PATH": "spark/config/profiles/default/*.conf;spark/config/profiles/data_stream/*.conf",
-        "DOTNET_BINARY_DIR": "${sourceDir}/build/$<CONFIG>/bin/net8.0;${sourceDir}/build/$<CONFIG>/bin/net8.0-windows"
+        "APP_DATA_DIR": "${dollar}{APPDATA}/bts/DataStream",
+        "APP_DATA_DIR_OVERRIDE": "${dollar}{DATA_STREAM_DATA_DIR}",
+        "CONFIG_DEFAULTS_PATH": "config/profiles/default/*.conf;config/profiles/data_stream/*.conf",
+        "SHARED_MEMORY_PREFIX": "data_stream"
      }
    },

@@ -235,6 +239,8 @@
      "inherits": "sparkd",
      "generator": "Ninja Multi-Config",
      "cacheVariables": {
+        "APP_DATA_DIR": "${dollar}{HOME}/.local/share/DataStream",
+        "APP_DATA_DIR_OVERRIDE": "${dollar}{DATA_STREAM_DATA_DIR}",
        "BUILD_ADMIN_WEB_APP": {
          "type": "BOOL",
          "value": "OFF"
@@ -251,12 +257,14 @@
        "CMAKE_CXX_COMPILER": "clang++",
        "CMAKE_CXX_FLAGS_DEBUG": "-g -fPIC -Wall -pedantic -Wextra -Werror -Wno-missing-braces -Wno-ignored-qualifiers --std=c++20 -fdiagnostics-color -DDEBUG",
        "CMAKE_CXX_FLAGS_RELEASE": "-fPIC -Wall -pedantic -Wextra -Werror -Wno-missing-braces -Wno-ignored-qualifiers --std=c++20 -fdiagnostics-color -fno-omit-frame-pointer -DNDEBUG -O3",
-        "CONFIG_DEFAULTS_PATH": "spark/config/profiles/default/*.conf;spark/config/profiles/data_stream/*.conf",
-        "DOTNET_BINARY_DIR": "${sourceDir}/build/$<CONFIG>/bin/net8.0",
+        "CMAKE_INSTALL_PREFIX": "${sourceDir}/install/${presetName}",
+        "CONFIG_DEFAULTS_PATH": "config/profiles/default/*.conf;config/profiles/data_stream/*.conf",
+        "DOTNET_BINARY_DIR": "${sourceDir}/build/$<CONFIG>/bin/net7.0",
        "EXPORT_MD_CONSUMER": {
          "type": "BOOL",
          "value": "ON"
-        }
+        },
+        "SHARED_MEMORY_PREFIX": "data_stream"
      }
    }
  ],
@@ -42,10 +42,6 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Bts.ColumnExpressions", "sr
 EndProject
 Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "DataStream", "src\DataStreamSuite\DataStream\DataStream.csproj", "{39AEA075-6457-46C0-8821-1158BE6EFA57}"
 EndProject
-Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "sdk", "sdk", "{56ACFCD1-60C7-4E2C-B90D-B1F95ECE33A1}"
-EndProject
-Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Bts.Spark.Sdk", "src\sdk\Bts.Spark.Sdk\Bts.Spark.Sdk.csproj", "{ADA9E802-8EFA-4E72-8E9A-6AA1E4181184}"
-EndProject
 Global
 	GlobalSection(SolutionConfigurationPlatforms) = preSolution
 		Debug|x64 = Debug|x64
@@ -198,14 +194,6 @@ Global
 		{6BD3F6D8-B6FC-44E8-B8FE-06352F638B9B}.Release|x64.Build.0 = Release|Any CPU
 		{6BD3F6D8-B6FC-44E8-B8FE-06352F638B9B}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
 		{6BD3F6D8-B6FC-44E8-B8FE-06352F638B9B}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
-		{ADA9E802-8EFA-4E72-8E9A-6AA1E4181184}.Debug|x64.ActiveCfg = Debug|Any CPU
-		{ADA9E802-8EFA-4E72-8E9A-6AA1E4181184}.Debug|x64.Build.0 = Debug|Any CPU
-		{ADA9E802-8EFA-4E72-8E9A-6AA1E4181184}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
-		{ADA9E802-8EFA-4E72-8E9A-6AA1E4181184}.MinSizeRel|x64.Build.0 = Debug|Any CPU
-		{ADA9E802-8EFA-4E72-8E9A-6AA1E4181184}.Release|x64.ActiveCfg = Release|Any CPU
-		{ADA9E802-8EFA-4E72-8E9A-6AA1E4181184}.Release|x64.Build.0 = Release|Any CPU
-		{ADA9E802-8EFA-4E72-8E9A-6AA1E4181184}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
-		{ADA9E802-8EFA-4E72-8E9A-6AA1E4181184}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
 	EndGlobalSection
 	GlobalSection(SolutionProperties) = preSolution
 		HideSolutionNode = FALSE
@@ -230,7 +218,5 @@ Global
 		{39AEA075-6457-46C0-8821-1158BE6EFA57} = {C09ED521-D2DF-40BC-A76F-EB3CB870B2F6}
 		{9F6D7DE8-185B-4C1D-B809-3314D092DF28} = {C09ED521-D2DF-40BC-A76F-EB3CB870B2F6}
 		{6BD3F6D8-B6FC-44E8-B8FE-06352F638B9B} = {C09ED521-D2DF-40BC-A76F-EB3CB870B2F6}
-		{56ACFCD1-60C7-4E2C-B90D-B1F95ECE33A1} = {267F81EF-A8AB-49FD-9F56-D7EC3C1F7C7C}
-		{ADA9E802-8EFA-4E72-8E9A-6AA1E4181184} = {56ACFCD1-60C7-4E2C-B90D-B1F95ECE33A1}
 	EndGlobalSection
 EndGlobal
@@ -1,7 +0,0 @@
-include_guard(GLOBAL)
-
-macro(export_to_parent)
-  foreach(var IN ITEMS ${ARGN})
-    set(${var} ${${var}} PARENT_SCOPE)
-  endforeach()
-endmacro()
@@ -1,142 +0,0 @@
-include_guard(DIRECTORY)
-include(${CMAKE_SOURCE_DIR}/cmake/ExportToParent.cmake)
-include(${CMAKE_SOURCE_DIR}/cmake/InstallBasic.cmake)
-include(${CMAKE_SOURCE_DIR}/cmake/ProtobufGenerateCpp.cmake)
-include(${CMAKE_SOURCE_DIR}/cmake/SetFirstDefined.cmake)
-include(${CMAKE_SOURCE_DIR}/cmake/safe_dep_glob.cmake)
-
-function(init_library)
-  set(_options SHARED STATIC)
-  set(_one_value_args
-    NAME
-    SOURCE_DIR
-    INCLUDE_DIR
-  )
-  set(_multi_value_args)
-  cmake_parse_arguments(_ARGS "${_options}" "${_one_value_args}" "${_multi_value_args}" ${ARGN})
-  if (_ARGS_SHARED AND _ARGS_STATIC)
-    message(FATAL_ERROR "Invalid args passed to init_library: cannot specify STATIC and SHARED")
-  endif()
-  set_first_defined(_name _ARGS_NAME PROJECT_NAME)
-  set_first_defined(_source_dir _ARGS_SOURCE_DIR _name)
-  set_first_defined(_include_dir _ARGS_INCLUDE_DIR _source_dir)
-
-  # Cpp sources
-  file(
-    GLOB_RECURSE ${_name}_SOURCES
-    RELATIVE ${CMAKE_CURRENT_SOURCE_DIR}
-    ${_source_dir}/*.cpp
-  )
-  # Note: cpp files inside 'tests' dir(s) are considered tests.
-  # Remove tests from main sources, and add them to test sources
-  foreach(item ${${_name}_SOURCES})
-    if(${item} MATCHES "tests/.*\.cpp$")
-      list(APPEND ${_name}_TEST_SOURCES ${item})
-    endif()
-  endforeach()
-  foreach(item ${${_name}_TEST_SOURCES})
-    list(REMOVE_ITEM ${_name}_SOURCES ${item})
-  endforeach()
-
-  # Proto sources 
-  file(
-    GLOB_RECURSE ${_name}_PROTO_SOURCES
-    RELATIVE ${CMAKE_CURRENT_SOURCE_DIR}
-    ${_source_dir}/protobufs/*.proto
-  )
-  update_deps_file("${${_name}_PROTO_SOURCES};")
-  protobuf_generate_cpp(
-    CXX_SOURCES ${_name}_CXX_PROTO_SOURCES
-    CXX_HEADERS ${_name}_CXX_PROTO_HEADERS
-    CXX_OUT_DIR ${CMAKE_CURRENT_BINARY_DIR}
-    PROTO_INCLUDE_DIRS
-      ${EXTERNALS_INCLUDE}
-      ${CMAKE_CURRENT_SOURCE_DIR}
-    PROTO_SOURCES
-      ${${_name}_PROTO_SOURCES}
-  )
-
-  export_to_parent(
-    ${_name}_SOURCES
-    ${_name}_TEST_SOURCES
-    ${_name}_PROTO_SOURCES
-    ${_name}_CXX_PROTO_SOURCES
-    ${_name}_CXX_PROTO_HEADERS
-  )
-
-  # Add main lib
-  if (_ARGS_STATIC)
-    add_library(${_name} STATIC ${${_name}_SOURCES} ${${_name}_CXX_PROTO_SOURCES})
-  elseif(_ARGS_SHARED)
-    add_library(${_name} SHARED ${${_name}_SOURCES} ${${_name}_CXX_PROTO_SOURCES})
-  else()
-    add_library(${_name} ${${_name}_SOURCES} ${${_name}_CXX_PROTO_SOURCES})
-  endif()
-  target_include_directories(
-    ${_name}
-    SYSTEM
-    BEFORE
-    PUBLIC
-      $<BUILD_INTERFACE:${EXTERNALS_ROOT}/include>
-      $<BUILD_INTERFACE:${EXTERNALS_ROOT}/include/zlib>
-  )
-  target_include_directories(
-    ${_name}
-    PUBLIC
-      $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}>
-      $<INSTALL_INTERFACE:${CMAKE_INSTALL_INCLUDEDIR}>
-  )
-  # Include protos (if any)
-  if (${_name}_PROTO_SOURCES)
-    target_include_directories(
-      ${_name}
-      PUBLIC
-        $<BUILD_INTERFACE:${CMAKE_CURRENT_BINARY_DIR}>
-    )
-  endif()
-  target_link_directories(
-    ${_name}
-    PUBLIC
-      $<BUILD_INTERFACE:${CMAKE_CURRENT_BINARY_DIR}>
-      $<BUILD_INTERFACE:${EXTERNALS_ROOT}/lib/$<CONFIG>>
-      $<INSTALL_INTERFACE:${CMAKE_INSTALL_LIBDIR}>
-  )
-  # Install main lib
-  foreach(_hdr ${${_name}_CXX_PROTO_HEADERS})
-    file(RELATIVE_PATH _relative_path ${CMAKE_CURRENT_BINARY_DIR} ${_hdr})
-    install(FILES ${_hdr} DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}/${_relative_path})
-  endforeach()
-  install_basic(
-    NAME ${_name}
-    INCLUDE_DIR ${_include_dir}
-  )
-
-  # Add tests executable (if there are any)
-  if (${_name}_TEST_SOURCES AND NOT WIN32)
-    add_executable(${_name}-tests ${${_name}_TEST_SOURCES})
-
-    get_target_property(_lib_type ${_name} TYPE)
-    if (_lib_type STREQUAL "SHARED_LIBRARY")
-      # Set RPATH so the tests exe knows where to find the shared lib.
-      set_target_properties(
-        ${_name}-tests
-        PROPERTIES
-          BUILD_RPATH "$ORIGIN"
-          INSTALL_RPATH "$ORIGIN/../${CMAKE_INSTALL_LIBDIR}"
-      )
-    endif()
-
-    # Link main lib
-    target_link_directories(
-      ${_name}-tests
-      PRIVATE
-        $<BUILD_INTERFACE:${CMAKE_CURRENT_BINARY_DIR}>
-        $<INSTALL_INTERFACE:${CMAKE_INSTALL_LIBDIR}>
-    )
-    target_link_libraries(${_name}-tests PRIVATE ${_name})
-
-    # Catch2
-    find_package(Catch2 CONFIG REQUIRED COMPONENTS Catch2WithMain)
-    target_link_libraries(${_name}-tests PRIVATE Catch2::Catch2WithMain)
-  endif()
-endfunction()
@@ -1,71 +0,0 @@
-include_guard(DIRECTORY)
-include(GNUInstallDirs)
-include(${CMAKE_SOURCE_DIR}/cmake/ExportToParent.cmake)
-include(${CMAKE_SOURCE_DIR}/cmake/SetFirstDefined.cmake)
-
-function(init_python_module)
-  set(_options)
-  set(_one_value_args
-    NAME
-  )
-  set(_multi_value_args
-    SOURCES
-  )
-  cmake_parse_arguments(_ARGS "${_options}" "${_one_value_args}" "${_multi_value_args}" ${ARGN})
-  set_first_defined(_name _ARGS_NAME PROJECT_NAME)
-  set_first_defined(_sources _ARGS_SOURCES ${_name}_SOURCES)
-
-  set(module_proj ${_name}_module)
-  set(${module_proj}_SOURCES ${_sources})
-  pybind11_add_module(${module_proj} SHARED ${_sources})
-  target_link_libraries(
-    ${module_proj}
-    PUBLIC
-      pybind11::embed
-      spark
-      ${_name}
-  )
-  target_include_directories(
-    ${module_proj}
-    PRIVATE
-      $<BUILD_INTERFACE:${pybind11_INCLUDE_DIRS}>
-  )
-  target_include_directories(
-    ${module_proj}
-    PUBLIC
-      ${Python3_INCLUDE_DIRS}
-      $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}>
-      $<INSTALL_INTERFACE:${CMAKE_INSTALL_INCLUDEDIR}>
-  )
-  add_custom_command(
-    OUTPUT ${CMAKE_CURRENT_BINARY_DIR}/stubs/${_name}.pyi
-    DEPENDS ${_name} ${module_proj}
-    COMMAND PYTHONPATH=${CMAKE_CURRENT_BINARY_DIR} pybind11-stubgen --numpy-array-use-type-var --ignore-invalid-expressions '.*' ${_name}_module
-    COMMENT "Generating py stubs"
-  )
-  add_custom_target(
-    "${_name}_py_stubs"
-    ALL
-    DEPENDS
-      ${CMAKE_CURRENT_BINARY_DIR}/stubs/${_name}.pyi
-  )
-  install(
-    TARGETS ${module_proj}
-    EXPORT ${module_proj}Targets
-    RUNTIME
-      DESTINATION ${CMAKE_INSTALL_BINDIR}
-    LIBRARY
-      DESTINATION ${CMAKE_INSTALL_LIBDIR}
-    ARCHIVE
-      DESTINATION ${CMAKE_INSTALL_LIBDIR}
-  )
-  foreach(dir ${pybind11_INCLUDE_DIRS})
-    install(DIRECTORY ${dir} DESTINATION ${CMAKE_INSTALL_INCLUDEDIR})
-  endforeach()
-  install(
-    EXPORT ${module_proj}Targets
-    FILE ${module_proj}Targets.cmake
-    NAMESPACE ${_name}::
-    DESTINATION ${CMAKE_INSTALL_LIBDIR}/cmake/${_name}
-  )
-endfunction()
@@ -1,65 +0,0 @@
-include_guard(DIRECTORY)
-include(GNUInstallDirs)
-
-function(init_python_plugin)
-  set(_options)
-  set(_one_value_args NAME PLUGIN_NAME)
-  set(_multi_value_args SOURCES)
-  cmake_parse_arguments(_ARGS "${_options}" "${_one_value_args}" "${_multi_value_args}" ${ARGN})
-  set_first_defined(_name _ARGS_NAME PROJECT_NAME)
-  set(_plugin_name ${_ARGS_PLUGIN_NAME})
-  if (NOT DEFINED _ARGS_PLUGIN_NAME)
-    set(_plugin_name ${_name}_plugins)
-  endif()
-  set(${_plugin_name}_SOURCES ${_ARGS_SOURCES})
-  if (NOT DEFINED _ARGS_SOURCES)
-    file(
-      GLOB_RECURSE ${_plugin_name}_SOURCES
-      RELATIVE ${CMAKE_CURRENT_SOURCE_DIR}
-      ${_plugin_name}/*.py)
-  endif()
-  set(src_outs)
-  foreach(_src ${${_plugin_name}_SOURCES})
-    set(_src_in ${CMAKE_CURRENT_SOURCE_DIR}/${_src})
-    set(_src_out ${CMAKE_CURRENT_BINARY_DIR}/${_src})
-    add_custom_command(
-      OUTPUT ${_src_out}
-      COMMAND ${CMAKE_COMMAND} -E copy ${_src_in} ${_src_out}
-      DEPENDS ${_src_in}
-      COMMENT "Copying ${_plugin_name} plugin ${_src} to ${_src_out}"
-    )
-    list(APPEND src_outs ${_src_out})
-  endforeach()
-  
-  add_custom_target(copy_${_plugin_name} ALL DEPENDS ${src_outs})
-  
-  add_library(${_plugin_name} INTERFACE)
-  add_library(${_name}::${_plugin_name} ALIAS ${_plugin_name})
-  
-  set_target_properties(
-    ${_plugin_name}
-    PROPERTIES
-      INTERFACE_PYTHON_SCRIPTS_DIR
-        $<BUILD_INTERFACE:${CMAKE_CURRENT_BINARY_DIR}/${_plugin_name}>$<INSTALL_INTERFACE:${CMAKE_INSTALL_BINDIR}/${_plugin_name}>
-  )
-  
-  set_target_properties(
-    ${_plugin_name}
-    PROPERTIES
-    INTERFACE_PYTHON_SCRIPTS $<BUILD_INTERFACE:${CMAKE_CURRENT_BINARY_DIR}/${${_plugin_name}_SOURCES}>$<INSTALL_INTERFACE:${CMAKE_INSTALL_BINDIR}/${${_plugin_name}_SOURCES}>
-  )
-  
-  install(
-    TARGETS ${_plugin_name}
-    EXPORT ${_plugin_name}Targets
-    DESTINATION ${CMAKE_INSTALL_BINDIR}
-  )
-  
-  install(
-    EXPORT ${_plugin_name}Targets
-    FILE ${_plugin_name}Targets.cmake
-    NAMESPACE ${_name}::
-    DESTINATION ${CMAKE_INSTALL_LIBDIR}/cmake/${_name}
-  )
-  install(DIRECTORY ${_plugin_name} DESTINATION ${CMAKE_INSTALL_BINDIR})
-endfunction()
@@ -1,39 +0,0 @@
-include_guard(DIRECTORY)
-include(GNUInstallDirs)
-include(${CMAKE_SOURCE_DIR}/cmake/PackBasic.cmake)
-include(${CMAKE_SOURCE_DIR}/cmake/SetFirstDefined.cmake)
-
-function(install_basic)
-  set(_options)
-  set(_one_value_args NAME INCLUDE_DIR)
-  set(_multi_value_args)
-  cmake_parse_arguments(_ARGS "${_options}" "${_one_value_args}" "${_multi_value_args}" ${ARGN})
-  set_first_defined(_name _ARGS_NAME PROJECT_NAME)
-  set_first_defined(_include_dir _ARGS_INCLUDE_DIR _name)
-
-  file(
-    GLOB_RECURSE ${_name}_HEADERS
-    RELATIVE ${CMAKE_CURRENT_SOURCE_DIR}
-    ${_include_dir}/*.h)
-
-  target_sources(${_name}
-    PUBLIC
-      FILE_SET HEADERS FILES ${${_name}_HEADERS})
-
-  install(
-    TARGETS ${_name}
-    EXPORT ${_name}Targets
-    RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR}
-    LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR}
-    ARCHIVE DESTINATION ${CMAKE_INSTALL_LIBDIR}
-    FILE_SET HEADERS DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}
-  )
-
-  install(
-    EXPORT ${_name}Targets
-    FILE ${_name}Targets.cmake
-    NAMESPACE ${_name}::
-    DESTINATION ${CMAKE_INSTALL_LIBDIR}/cmake/${_name})
-
-  pack_basic()
-endfunction()
@@ -1,44 +0,0 @@
-include_guard(DIRECTORY)
-include(CMakePackageConfigHelpers)
-include(${CMAKE_SOURCE_DIR}/cmake/SetFirstDefined.cmake)
-
-function(pack_basic)
-  set(_options)
-  set(_one_value_args NAME)
-  set(_multi_value_args)
-  cmake_parse_arguments(_ARGS "${_options}" "${_one_value_args}" "${_multi_value_args}" ${ARGN})
-  set_first_defined(_name _ARGS_NAME PROJECT_NAME)
-
-  find_package(Git QUIET)
-
-  if(GIT_FOUND)
-    execute_process(
-      COMMAND ${GIT_EXECUTABLE} describe --tags --always --dirty
-      WORKING_DIRECTORY ${CMAKE_SOURCE_DIR}
-      OUTPUT_VARIABLE PROJECT_VERSION
-      OUTPUT_STRIP_TRAILING_WHITESPACE
-    )
-  endif()
-
-  if(NOT GIT_FOUND OR PROJECT_VERSION STREQUAL "")
-    set(PROJECT_VERSION "0.0.0")
-  endif()
-
-  write_basic_package_version_file(
-    "${CMAKE_CURRENT_BINARY_DIR}/${_name}ConfigVersion.cmake"
-    VERSION ${PROJECT_VERSION}
-    COMPATIBILITY AnyNewerVersion
-  )
-
-  configure_package_config_file(
-    "${CMAKE_CURRENT_SOURCE_DIR}/Config.cmake.in"
-    "${CMAKE_CURRENT_BINARY_DIR}/${_name}Config.cmake"
-    INSTALL_DESTINATION ${CMAKE_INSTALL_LIBDIR}/cmake/${_name}
-  )
-
-  install(
-    FILES
-      "${CMAKE_CURRENT_BINARY_DIR}/${_name}Config.cmake"
-      "${CMAKE_CURRENT_BINARY_DIR}/${_name}ConfigVersion.cmake"
-    DESTINATION ${CMAKE_INSTALL_LIBDIR}/cmake/${_name})
-endfunction()
@@ -1,23 +1,12 @@
-set(Protobuf_PROTOC_EXECUTABLE "${EXTERNALS_BIN}/protoc")
-set(PROTOBUF_INCLUDE_DIRS "${EXTERNALS_INCLUDE}")
+set(Protobuf_PROTOC_EXECUTABLE "${SPARK_BIN}/protoc")
+set(PROTOBUF_INCLUDE_DIRS "${SPARK_INCLUDE}")
 set(PROTOBUF_STATIC_LIB "libprotobuf")
-set(_protobuf_src_dir ${CMAKE_SOURCE_DIR}/src/lib_spark)
-set(_protobuf_bin_dir ${CMAKE_BINARY_DIR}/src/lib_spark)

-function(Protobuf_Generate_Cpp)
-  set(_options)
-  set(_one_value_args CXX_SOURCES CXX_HEADERS CXX_OUT_DIR)
-  set(_multi_value_args PROTO_INCLUDE_DIRS PROTO_SOURCES)
-  cmake_parse_arguments(_ARGS "${_options}" "${_one_value_args}" "${_multi_value_args}" ${ARGN})
-  set(${_ARGS_CXX_SOURCES})
-  set(${_ARGS_CXX_HEADERS})
+function(Protobuf_Generate_Cpp SRCS HDRS)
+  set(${SRCS})
+  set(${HDRS})

-  set(_proto_include_cli_args)
-  foreach(_dir ${_ARGS_PROTO_INCLUDE_DIRS})
-    list(APPEND _proto_include_cli_args -I ${_dir})
-  endforeach()
-
-  foreach(FIL ${_ARGS_PROTO_SOURCES})
+  foreach(FIL ${ARGN})
    get_filename_component(ABS_FIL ${FIL} ABSOLUTE)
    get_filename_component(FIL_WE ${FIL} NAME_WE)
    get_filename_component(FIL_DIR ${FIL} DIRECTORY)
@@ -25,63 +14,48 @@ function(Protobuf_Generate_Cpp)
      set(FIL_WE "${FIL_DIR}/${FIL_WE}")
    endif()

-    list(APPEND ${_ARGS_CXX_SOURCES} "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.pb.cc")
-    list(APPEND ${_ARGS_CXX_HEADERS} "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.pb.h")
+    list(APPEND ${SRCS} "${CMAKE_CURRENT_BINARY_DIR}/${FIL_WE}.pb.cc")
+    list(APPEND ${HDRS} "${CMAKE_CURRENT_BINARY_DIR}/${FIL_WE}.pb.h")

    add_custom_command(
-      OUTPUT
-        "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.pb.cc"
-        "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.pb.h"
+      OUTPUT "${CMAKE_CURRENT_BINARY_DIR}/${FIL_WE}.pb.cc"
+             "${CMAKE_CURRENT_BINARY_DIR}/${FIL_WE}.pb.h"
      COMMAND
-        ${Protobuf_PROTOC_EXECUTABLE}
-      ARGS
-        --cpp_out=${_ARGS_CXX_OUT_DIR} ${_proto_include_cli_args} ${ABS_FIL}
+        ${Protobuf_PROTOC_EXECUTABLE} ARGS --cpp_out=${CMAKE_BINARY_DIR}/src -I
+        ${PROTOBUF_INCLUDE_DIRS} -I ${CMAKE_SOURCE_DIR}/src ${ABS_FIL}
      DEPENDS ${ABS_FIL}
      COMMENT "Running C++ protocol buffer compiler on ${FIL}"
-      VERBATIM
-    )
+      VERBATIM)
  endforeach()

-  set_source_files_properties(
-    ${${_ARGS_CXX_SOURCES}}
-    ${${_ARGS_CXX_HEADERS}}
-    PROPERTIES
-      GENERATED TRUE
-  )
+  set_source_files_properties(${${SRCS}} ${${HDRS}} PROPERTIES GENERATED TRUE)

  # remove some warnings from protobuf generated files
  if(WIN32)
-    set_source_files_properties(
-      ${${_ARGS_CXX_SOURCES}}
-      PROPERTIES
-        COMPILE_FLAGS "/wd4309 /wd4838"
-    )
+    set_source_files_properties(${${SRCS}} PROPERTIES COMPILE_FLAGS
+                                                      "/wd4309 /wd4838")
  else()
    set_source_files_properties(
-      ${${_ARGS_CXX_SOURCES}}
+      ${${SRCS}}
      PROPERTIES
-        COMPILE_FLAGS "-Wno-unused-parameter -Wno-effc++ -Wno-pedantic -Wno-sign-compare -Wno-uninitialized"
+        COMPILE_FLAGS
+        "-Wno-unused-parameter -Wno-effc++ -Wno-pedantic -Wno-sign-compare -Wno-uninitialized"
    )
  endif()

-  set(${_ARGS_CXX_SOURCES} ${${_ARGS_CXX_SOURCES}} PARENT_SCOPE)
-  set(${_ARGS_CXX_HEADERS} ${${_ARGS_CXX_HEADERS}} PARENT_SCOPE)
+  set(${SRCS}
+      ${${SRCS}}
+      PARENT_SCOPE)
+  set(${HDRS}
+      ${${HDRS}}
+      PARENT_SCOPE)
 endfunction()

-function(Service_Compiler_Generate_Cpp)
-  set(_options)
-  set(_one_value_args CXX_SOURCES CXX_HEADERS CXX_OUT_DIR)
-  set(_multi_value_args PROTO_INCLUDE_DIRS PROTO_SOURCES)
-  cmake_parse_arguments(_ARGS "${_options}" "${_one_value_args}" "${_multi_value_args}" ${ARGN})
-  set(${_ARGS_CXX_SOURCES})
-  set(${_ARGS_CXX_HEADERS})
+function(Service_Compiler_Generate_Cpp SRCS HDRS)
+  set(${SRCS})
+  set(${HDRS})

-  set(_proto_include_cli_args)
-  foreach(_dir ${_ARGS_PROTO_INCLUDE_DIRS})
-    list(APPEND _proto_include_cli_args -I ${_dir})
-  endforeach()
-
-  foreach(FIL ${_ARGS_PROTO_SOURCES})
+  foreach(FIL ${ARGN})
    get_filename_component(ABS_FIL ${FIL} ABSOLUTE)
    get_filename_component(FIL_WE ${FIL} NAME_WE)
    get_filename_component(FIL_DIR ${FIL} DIRECTORY)
@@ -89,94 +63,83 @@ function(Service_Compiler_Generate_Cpp)
      set(FIL_WE "${FIL_DIR}/${FIL_WE}")
    endif()

-    list(APPEND ${_ARGS_CXX_SOURCES} "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.pb.cc")
-    list(APPEND ${_ARGS_CXX_HEADERS} "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.pb.h")
+    list(APPEND ${SRCS} "${CMAKE_CURRENT_BINARY_DIR}/${FIL_WE}.pb.cc")
+    list(APPEND ${HDRS} "${CMAKE_CURRENT_BINARY_DIR}/${FIL_WE}.pb.h")

-    list(APPEND ${_ARGS_CXX_SOURCES} "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.client.pb.cc")
-    list(APPEND ${_ARGS_CXX_HEADERS} "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.client.pb.h")
+    list(APPEND ${SRCS} "${CMAKE_CURRENT_BINARY_DIR}/${FIL_WE}.client.pb.cc")
+    list(APPEND ${HDRS} "${CMAKE_CURRENT_BINARY_DIR}/${FIL_WE}.client.pb.h")

-    list(APPEND ${_ARGS_CXX_SOURCES} "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.server.pb.cc")
-    list(APPEND ${_ARGS_CXX_HEADERS} "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.server.pb.h")
+    list(APPEND ${SRCS} "${CMAKE_CURRENT_BINARY_DIR}/${FIL_WE}.server.pb.cc")
+    list(APPEND ${HDRS} "${CMAKE_CURRENT_BINARY_DIR}/${FIL_WE}.server.pb.h")

    set(TEMPLATES
-      ${CMAKE_SOURCE_DIR}/src/lib_spark/spark/api/service_compiler/templates/cpp/client.pb.cc
-      ${CMAKE_SOURCE_DIR}/src/lib_spark/spark/api/service_compiler/templates/cpp/client.pb.h
-      ${CMAKE_SOURCE_DIR}/src/lib_spark/spark/api/service_compiler/templates/cpp/server.pb.cc
-      ${CMAKE_SOURCE_DIR}/src/lib_spark/spark/api/service_compiler/templates/cpp/server.pb.h
+        ${CMAKE_CURRENT_SOURCE_DIR}/api/service_compiler/templates/cpp/client.pb.cc
+        ${CMAKE_CURRENT_SOURCE_DIR}/api/service_compiler/templates/cpp/client.pb.h
+        ${CMAKE_CURRENT_SOURCE_DIR}/api/service_compiler/templates/cpp/server.pb.cc
+        ${CMAKE_CURRENT_SOURCE_DIR}/api/service_compiler/templates/cpp/server.pb.h
    )

-    set(_plugin_py ${CMAKE_SOURCE_DIR}/src/lib_spark/spark/api/service_compiler/plugin.py)
    if(WIN32)
-      set(_plugin_cmd ${CMAKE_SOURCE_DIR}/src/lib_spark/spark/api/service_compiler/plugin.cmd)
      add_custom_command(
-        OUTPUT
-          "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.pb.cc"
-          "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.pb.h"
-          "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.client.pb.cc"
-          "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.client.pb.h"
-          "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.server.pb.cc"
-          "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.server.pb.h"
+        OUTPUT "${CMAKE_CURRENT_BINARY_DIR}/${FIL_WE}.pb.cc"
+               "${CMAKE_CURRENT_BINARY_DIR}/${FIL_WE}.pb.h"
+               "${CMAKE_CURRENT_BINARY_DIR}/${FIL_WE}.client.pb.cc"
+               "${CMAKE_CURRENT_BINARY_DIR}/${FIL_WE}.client.pb.h"
+               "${CMAKE_CURRENT_BINARY_DIR}/${FIL_WE}.server.pb.cc"
+               "${CMAKE_CURRENT_BINARY_DIR}/${FIL_WE}.server.pb.h"
        COMMAND SET PYTHONPATH=${SPARK_PYTHON}
        COMMAND ${CMAKE_SOURCE_DIR}/.venv/Scripts/activate.bat
        COMMAND
-          ${Protobuf_PROTOC_EXECUTABLE}
-          ARGS
-            --cpp_out=${_ARGS_CXX_OUT_DIR}
-            --plugin=protoc-gen-service=${_plugin_cmd}
-            --service_out=lang=cpp:${_ARGS_CXX_OUT_DIR}
-            -I ${PROTOBUF_INCLUDE_DIRS}
-            -I ${_protobuf_src_dir}
-            ${ABS_FIL}
-        DEPENDS
-          ${ABS_FIL}
-          ${TEMPLATES}
-          ${_plugin_cmd}
-          ${_plugin_py}
+          ${Protobuf_PROTOC_EXECUTABLE} ARGS --cpp_out=${CMAKE_BINARY_DIR}/src
+          --plugin=protoc-gen-service=${CMAKE_SOURCE_DIR}/src/lib/api/service_compiler/plugin.cmd
+          --service_out=lang=cpp:${CMAKE_BINARY_DIR}/src -I
+          ${PROTOBUF_INCLUDE_DIRS} -I ${CMAKE_SOURCE_DIR}/src ${ABS_FIL}
+        DEPENDS ${ABS_FIL} ${TEMPLATES}
+                ${CMAKE_CURRENT_SOURCE_DIR}/api/service_compiler/plugin.py
        COMMENT "Running C++ service compiler on ${FIL}"
-        VERBATIM
-      )
+        VERBATIM)
    else()
      add_custom_command(
-        OUTPUT
-          "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.pb.cc"
-          "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.pb.h"
-          "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.client.pb.cc"
-          "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.client.pb.h"
-          "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.server.pb.cc"
-          "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.server.pb.h"
+        OUTPUT "${CMAKE_CURRENT_BINARY_DIR}/${FIL_WE}.pb.cc"
+               "${CMAKE_CURRENT_BINARY_DIR}/${FIL_WE}.pb.h"
+               "${CMAKE_CURRENT_BINARY_DIR}/${FIL_WE}.client.pb.cc"
+               "${CMAKE_CURRENT_BINARY_DIR}/${FIL_WE}.client.pb.h"
+               "${CMAKE_CURRENT_BINARY_DIR}/${FIL_WE}.server.pb.cc"
+               "${CMAKE_CURRENT_BINARY_DIR}/${FIL_WE}.server.pb.h"
        COMMAND ${CMAKE_SOURCE_DIR}/tools/venv_exec.sh
        ARGS
          ${Protobuf_PROTOC_EXECUTABLE}
-          --cpp_out=${_ARGS_CXX_OUT_DIR}
-          --plugin=protoc-gen-service=${_plugin_py}
-          --service_out=lang=cpp,make_file_name=${ABS_FIL}:${_ARGS_CXX_OUT_DIR}
-          ${_proto_include_cli_args} ${ABS_FIL}
-        DEPENDS
-          ${ABS_FIL}
-          ${TEMPLATES}
-          ${_plugin_py}
+          --cpp_out=${CMAKE_BINARY_DIR}/src
+          --plugin=protoc-gen-service=${CMAKE_SOURCE_DIR}/src/lib/api/service_compiler/plugin.py
+          --service_out=lang=cpp,make_file_name=${ABS_FIL}:${CMAKE_BINARY_DIR}/src
+          -I ${PROTOBUF_INCLUDE_DIRS} -I ${CMAKE_SOURCE_DIR}/src ${ABS_FIL}
+        DEPENDS ${ABS_FIL} ${TEMPLATES}
+                ${CMAKE_CURRENT_SOURCE_DIR}/api/service_compiler/plugin.py
        COMMENT "Running C++ service compiler on ${FIL}"
-        VERBATIM
-      )
+        VERBATIM)
    endif()
+
  endforeach()

  # remove some warnings from protobuf generated files
  if(WIN32)
-    set_source_files_properties(
-      ${${_ARGS_CXX_SOURCES}}
-      PROPERTIES
-        COMPILE_FLAGS "/wd4309 /wd4838"
-    )
+    set_source_files_properties(${${SRCS}} PROPERTIES COMPILE_FLAGS
+                                                      "/wd4309 /wd4838")
  else()
    set_source_files_properties(
-      ${${_ARGS_CXX_SOURCES}}
+      ${${SRCS}}
      PROPERTIES
-        COMPILE_FLAGS "-Wno-unused-parameter -Wno-effc++ -Wno-pedantic -Wno-sign-compare -Wno-uninitialized"
+        COMPILE_FLAGS
+        "-Wno-unused-parameter -Wno-effc++ -Wno-pedantic -Wno-sign-compare -Wno-uninitialized"
    )
  endif()

-  set(${_ARGS_CXX_SOURCES} ${${_ARGS_CXX_SOURCES}} PARENT_SCOPE)
-  set(${_ARGS_CXX_HEADERS} ${${_ARGS_CXX_HEADERS}} PARENT_SCOPE)
+  set(${SRCS}
+      ${${SRCS}}
+      PARENT_SCOPE)
+  set(${HDRS}
+      ${${HDRS}}
+      PARENT_SCOPE)
+
 endfunction()

@@ -1,83 +0,0 @@
-include_guard(GLOBAL)
-
-function(protobuf_generate_cpp)
-  set(_options)
-  set(_one_value_args
-    CXX_SOURCES
-    CXX_HEADERS
-    CXX_OUT_DIR
-    PROTOC
-  )
-  set(_multi_value_args
-    PROTO_INCLUDE_DIRS
-    PROTO_SOURCES
-  )
-  cmake_parse_arguments(_ARGS "${_options}" "${_one_value_args}" "${_multi_value_args}" ${ARGN})
-
-  if (NOT DEFINED _ARGS_PROTOC)
-    if (WIN32)
-      set(_ARGS_PROTOC "${EXTERNALS_BIN}/protoc.exe")
-    else()
-      set(_ARGS_PROTOC "${EXTERNALS_BIN}/protoc")
-    endif()
-  endif()
-  set(${_ARGS_CXX_SOURCES})
-  set(${_ARGS_CXX_HEADERS})
-
-  set(_proto_include_cli_args)
-  foreach(_dir ${_ARGS_PROTO_INCLUDE_DIRS})
-    list(APPEND _proto_include_cli_args -I ${_dir})
-  endforeach()
-
-  foreach(_fil ${_ARGS_PROTO_SOURCES})
-    get_filename_component(_abs_fil ${_fil} ABSOLUTE)
-    get_filename_component(_fil_we ${_fil} NAME_WE)
-    get_filename_component(_fil_dir ${_fil} DIRECTORY)
-    if(_fil_dir)
-      set(_fil_we "${_fil_dir}/${_fil_we}")
-    endif()
-    list(APPEND ${_ARGS_CXX_SOURCES} "${_ARGS_CXX_OUT_DIR}/${_fil_we}.pb.cc")
-    list(APPEND ${_ARGS_CXX_HEADERS} "${_ARGS_CXX_OUT_DIR}/${_fil_we}.pb.h")
-    message(STATUS "add_custom_command for ${_fil} -> ${_ARGS_CXX_OUT_DIR}/${_fil_we}.pb.h")
-    # TODO: Pass proto sources by relative path (so this fn can be called from
-    # arbitrary subdirs).
-    add_custom_command(
-      OUTPUT
-        "${_ARGS_CXX_OUT_DIR}/${_fil_we}.pb.cc"
-        "${_ARGS_CXX_OUT_DIR}/${_fil_we}.pb.h"
-      COMMAND
-        ${_ARGS_PROTOC}
-      ARGS
-        --cpp_out=${_ARGS_CXX_OUT_DIR} ${_proto_include_cli_args} ${_abs_fil}
-      DEPENDS ${_abs_fil}
-      COMMENT "Running C++ protocol buffer compiler on ${_fil}"
-      VERBATIM
-    )
-  endforeach()
-
-  set_source_files_properties(
-    ${${_ARGS_CXX_SOURCES}}
-    ${${_ARGS_CXX_HEADERS}}
-    PROPERTIES
-      GENERATED TRUE
-  )
-
-  # remove some warnings from protobuf generated files
-  if(WIN32)
-    set_source_files_properties(
-      ${${_ARGS_CXX_SOURCES}}
-      PROPERTIES
-        COMPILE_FLAGS "/wd4309 /wd4838"
-    )
-  else()
-    set_source_files_properties(
-      ${${_ARGS_CXX_SOURCES}}
-      PROPERTIES
-        COMPILE_FLAGS "-Wno-unused-parameter -Wno-effc++ -Wno-pedantic -Wno-sign-compare -Wno-uninitialized"
-    )
-  endif()
-
-  set(${_ARGS_CXX_SOURCES} ${${_ARGS_CXX_SOURCES}} PARENT_SCOPE)
-  set(${_ARGS_CXX_HEADERS} ${${_ARGS_CXX_HEADERS}} PARENT_SCOPE)
-endfunction()
-
@@ -1,61 +0,0 @@
-include_guard(GLOBAL)
-
-function(protobuf_generate_cpp_config)
-  set(_options)
-  set(_one_value_args
-    PROTO_SOURCES_HEADER
-    PROTO_DEFAULTS_HEADER
-  )
-  set(_multi_value_args
-    PROTO_SOURCES
-    PROTO_DEFAULTS
-  )
-  cmake_parse_arguments(_ARGS "${_options}" "${_one_value_args}" "${_multi_value_args}" ${ARGN})
-
-  # Configure the 'config_proto_files' header. It provides an array of strings populated with the
-  # files in PROTO_SOURCES. At runtime, we use this array to generate a special 'config' proto
-  # message with fields for each message defined in said files. A global instance of this 'config'
-  # proto is then constructed, and populated with [compile-time] defaults, and [runtime] overrides
-  # (i.e., in spark.conf). Calls to `bts::config::get<T>()` retrieve the value from the field
-  # associated with type `T` in this global instance.
-  message(STATUS "PROTO_SOURCES: ${_ARGS_PROTO_SOURCES}")
-  message(STATUS "PROTO_SOURCES_HEADER: ${_ARGS_PROTO_SOURCES_HEADER}")
-  message(STATUS "PROTO_DEFAULTS_HEADER: ${_ARGS_PROTO_DEFAULTS_HEADER}")
-  if (_ARGS_PROTO_SOURCES_HEADER)
-    set(PROTO_SOURCES)
-    set(PROTO_SOURCES_CXX_ARRAY_ITEMS "")
-    foreach(_proto_path ${_ARGS_PROTO_SOURCES})
-      list(APPEND PROTO_SOURCES ${_proto_path})
-      string(APPEND PROTO_SOURCES_CXX_ARRAY_ITEMS "\"${_proto_path}\",\n")
-    endforeach()
-    list(LENGTH PROTO_SOURCES PROTO_SOURCES_LENGTH)
-    message(STATUS "CONFIGURING PROTO_SOURCES_HEADER): ${_ARGS_PROTO_SOURCES_HEADER}")
-    configure_file(${CMAKE_SOURCE_DIR}/cmake/config_proto_files.h.in ${_ARGS_PROTO_SOURCES_HEADER})
-  endif()
-
-  # Generate static defaults. Ideally, this would be a single string, but we use an array because
-  # the compiler complains if the string is longer than 65k.
-  set(_cfg_defaults_str "inline const char * config_static_defaults[] = { ")
-
-  foreach(_cfg_file_path ${_ARGS_PROTO_DEFAULTS})
-    file(READ ${_cfg_file_path} _cfg_file_content)
-    string(REPLACE "\"" "\\\"" _cfg_file_content_escaped ${_cfg_file_content})
-    string(REPLACE "\n" ";" _cfg_lines ${_cfg_file_content_escaped})
-
-    foreach(_line ${_cfg_lines})
-      string(APPEND _cfg_defaults_str "\n  \"${_line}\",")
-    endforeach(_line)
-  endforeach(_cfg_file_path)
-
-  string(APPEND _cfg_defaults_str "};")
-  set(_old_cfg_defaults_str "")
-
-  if(EXISTS ${_ARGS_PROTO_DEFAULTS_HEADER})
-    file(READ ${_ARGS_PROTO_DEFAULTS_HEADER} _old_cfg_defaults_str)
-  endif()
-
-  if(NOT _cfg_defaults_str STREQUAL _old_cfg_defaults_str)
-    file(WRITE "${_ARGS_PROTO_DEFAULTS_HEADER}" "${_cfg_defaults_str}")
-  endif()
-endfunction()
-
@@ -10,7 +10,7 @@ function(protobuf_generate_fs)
    set(_ARGS_TARGET_NAME "${PROJECT_NAME}_generate_fs_protos")
  endif()
  if (NOT DEFINED _ARGS_PROTOC)
-    set(_ARGS_PROTOC "${EXTERNALS_BIN}/protoc")
+    set(_ARGS_PROTOC "${SPARK_BIN}/protoc")
  endif()

  set(_proto_include_cli_args)
@@ -27,8 +27,8 @@ function(protobuf_generate_fs)
  endforeach()

  set(_venv_exec ${CMAKE_SOURCE_DIR}/tools/venv_exec.sh)
-  set(_plugin_py ${CMAKE_SOURCE_DIR}/src/lib_spark/spark/api/service_compiler/plugin.py)
-  set(_template_fs ${CMAKE_SOURCE_DIR}/src/lib_spark/spark/api/service_compiler/templates/fs/fable.fs)
+  set(_plugin_py ${CMAKE_SOURCE_DIR}/src/lib/api/service_compiler/plugin.py)
+  set(_template_fs ${CMAKE_SOURCE_DIR}/src/lib/api/service_compiler/templates/fs/fable.fs)
  add_custom_command(
    OUTPUT ${_outputs}
    WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}
@@ -1,10 +0,0 @@
-include_guard(GLOBAL)
-
-macro(set_first_defined RESULT)
-  foreach(_arg ${ARGN})
-    if (DEFINED ${_arg})
-      set(${RESULT} ${${_arg}})
-      break()
-    endif()
-  endforeach()
-endmacro()
@@ -1,39 +0,0 @@
-include_guard(DIRECTORY)
-include(${CMAKE_SOURCE_DIR}/cmake/SymLink.cmake)
-
-function(sym_link_python_module)
-  set(_options)
-  set(_one_value_args LIB_NAME MODULE_NAME PLUGIN_NAME)
-  set(_multi_value_args)
-  cmake_parse_arguments(_ARGS "${_options}" "${_one_value_args}"
-                        "${_multi_value_args}" ${ARGN})
-  if(NOT DEFINED _ARGS_LIB_NAME)
-    message(
-      FATAL_ERROR "LIB_NAME must be specified for sym_link_python_module()")
-  endif()
-  set(_lib_name ${_ARGS_LIB_NAME})
-  set(_module_name ${_ARGS_MODULE_NAME})
-  if(NOT DEFINED _ARGS_MODULE_NAME)
-    set(_module_name ${_lib_name}_module)
-  endif()
-  set(_plugin_name ${_ARGS_PLUGIN_NAME})
-  set(_deps ${_lib_name} ${_module_name})
-  if(DEFINED _ARGS_PLUGIN_NAME)
-    list(APPEND _deps ${_plugin_name} copy_${_plugin_name})
-  endif()
-  add_dependencies(${PROJECT_NAME} ${_deps})
-  sym_link_local_lib(${_lib_name})
-  set(_module_suffix cpython-310-x86_64-linux-gnu.so)
-  sym_link_file(
-    SOURCE
-    ${CMAKE_BINARY_DIR}/src/lib_${_lib_name}/${_module_name}.${_module_suffix}
-    DEST
-    ${CMAKE_CURRENT_BINARY_DIR}/${_module_name}.${_module_suffix}
-    INSTALL_DEST
-    ${CMAKE_INSTALL_BINDIR}/)
-  if(DEFINED _ARGS_PLUGIN_NAME)
-    get_target_property(plugins ${_plugin_name} INTERFACE_PYTHON_SCRIPTS_DIR)
-    sym_link_dir(TARGET_NAME ${PROJECT_NAME}_link_${_plugin_name} SOURCE
-                 ${plugins} DEST ${CMAKE_CURRENT_BINARY_DIR}/${_plugin_name})
-  endif()
-endfunction()
@@ -1,11 +0,0 @@
-#pragma once
-
-#include <array>
-
-namespace bts::@PROJECT_NAME@::protobufs {
-
-inline std::array<const char *, @PROTO_SOURCES_LENGTH@> config_proto_files = {
-@PROTO_SOURCES_CXX_ARRAY_ITEMS@
-};
-
-} // namespace bts::@PROJECT_NAME@::protobufs
@@ -0,0 +1,24 @@
+cmake_minimum_required(VERSION 3.5)
+
+# Explicitly set policy to convert relative paths to absolute (although this is a default policy,
+# setting it explicitly prevents a compile-warning)
+cmake_policy(SET CMP0076 NEW)
+
+# add protos to the library to facilitate source_group
+target_sources(libspark PUBLIC ${Config_Protos} ${NonConfig_Protos} ${Service_Protos})
+
+function(assign_source_group)
+  foreach(_source IN ITEMS ${ARGN})
+    if(IS_ABSOLUTE "${_source}")
+      file(RELATIVE_PATH _source_rel "${CMAKE_CURRENT_SOURCE_DIR}" "${_source}")
+    else()
+      set(_source_rel "${_source}")
+    endif()
+    get_filename_component(_source_path "${_source_rel}" PATH)
+    string(REPLACE "/" "\\" _source_path_msvc "${_source_path}")
+    source_group("${_source_path_msvc}" FILES "${_source}")
+    set(SOURCE_FILES "${_source}")
+  endforeach()
+endfunction(assign_source_group)
+
+assign_source_group(${SparkLib_Sources} ${Config_Protos} ${NonConfig_Protos} ${Service_Protos})
@@ -1,31 +1,26 @@
 services:
  sparkd:
    volumes:
-      - ${SPARKD__DATA_DIR:-sparkd_data}:/home/bts/.local/share/bts/sparkd/data
+      - ${SPARKD__DATA_DIR:-sparkd_data}:/home/bts/sparkdata/data
    build:
-      dockerfile: src/app_sparkd/Dockerfile
+      dockerfile: src/server/sparkd/Dockerfile
      args:
        - IMAGE=${SPARKD__IMAGE}
        - SOURCE_DIR=${SPARKD__SOURCE_DIR}
        - CONFIG_DIR=${SPARKD__CONFIG_DIR}
-        - PYTHON_REQUIREMENTS=${SPARKD__PYTHON_REQUIREMENTS}
    shm_size: "${SPARKD__SHM_SIZE:-512mb}"
    command: >
      sh -c "
-        echo bts | sudo -S chown -R bts:bts /home/bts/.local/share/bts/sparkd/data
-        if [ -e "/home/bts/.local/share/bts/sparkd/data/logging.sparkd/log" ]; then
-          mv "/home/bts/.local/share/bts/sparkd/data/logging.sparkd/log" "/home/bts/.local/share/bts/sparkd/data/logging.sparkd/log_$(date '+%Y%m%d-%H%M')"
+        echo bts | sudo -S chown -R bts:bts /home/bts/sparkdata/data
+        if [ -e "/home/bts/sparkdata/data/logging.sparkd/log" ]; then
+          mv "/home/bts/sparkdata/data/logging.sparkd/log" "/home/bts/sparkdata/data/logging.sparkd/log_$(date '+%Y%m%d-%H%M')"
        fi
-        if [ -e "/home/bts/.local/share/bts/sparkd/conf/.env.local" ]; then
-          echo 'Sourcing /home/bts/.local/share/bts/sparkd/conf/.env.local'
+        if [ -e "/home/bts/sparkdata/conf/.env.local" ]; then
+          echo 'Sourcing /home/bts/sparkdata/conf/.env.local'
          set -a
-          . /home/bts/.local/share/bts/sparkd/conf/.env.local
+          . /home/bts/sparkdata/conf/.env.local
          set +a
        fi
-        if [ -d /home/bts/.local/share/bts/sparkd/conf/plugins ]; then
-          echo 'Copying /home/bts/.local/share/bts/sparkd/conf/plugins to runtime dir (/home/bts/bin)'
-          cp -r /home/bts/.local/share/bts/sparkd/conf/plugins /home/bts/bin/
-        fi
        ./sparkd run"
    env_file:
      - .env
@@ -38,37 +33,31 @@ services:
      resources:
        limits:
          memory: "${SPARKD__MEMORY:-4gb}"
-
  sparkd-symbols:
    volumes:
-      - ${SPARKD__DATA_DIR:-sparkd_data}:/home/bts/.local/share/bts/sparkd/data
-      - ${SPARKD__RUNTIME_CONF_DIR:-sparkd_conf}:/home/bts/.local/share/bts/sparkd/conf
+      - ${SPARKD__DATA_DIR:-sparkd_data}:/home/bts/sparkdata/data
+      - ${SPARKD__RUNTIME_CONF_DIR:-sparkd_conf}:/home/bts/sparkdata/conf
    build:
-      dockerfile: src/app_sparkd/Dockerfile
+      dockerfile: src/server/sparkd/Dockerfile
      args:
        - IMAGE=${SPARKD__IMAGE}
        - SOURCE_DIR=${SPARKD__SOURCE_DIR}
        - CONFIG_DIR=${SPARKD__CONFIG_DIR}
-        - PYTHON_REQUIREMENTS=${SPARKD__PYTHON_REQUIREMENTS}
    shm_size: "${SPARKD__SHM_SIZE:-512mb}"
    command: >
      sh -c "
-        echo bts | sudo -S chown -R bts:bts /home/bts/.local/share/bts/sparkd
-        if [ -e "/home/bts/.local/share/bts/sparkd/data/logging.sparkd/log" ]; then
-          mv "/home/bts/.local/share/bts/sparkd/data/logging.sparkd/log" "/home/bts/.local/share/bts/sparkd/data/logging.sparkd/log_$(date '+%Y%m%d-%H%M')"
+        echo bts | sudo -S chown -R bts:bts /home/bts/sparkdata/data
+        if [ -e "/home/bts/sparkdata/data/logging.sparkd/log" ]; then
+          mv "/home/bts/sparkdata/data/logging.sparkd/log" "/home/bts/sparkdata/data/logging.sparkd/log_$(date '+%Y%m%d-%H%M')"
        fi
-        ls -alh /home/bts/.local/share/bts/sparkd/conf/
-        cat /home/bts/.local/share/bts/sparkd/conf/.env.local
-        if [ -e "/home/bts/.local/share/bts/sparkd/conf/.env.local" ]; then
-          echo 'Sourcing /home/bts/.local/share/bts/sparkd/conf/.env.local'
+        ls -alh /home/bts/sparkdata/conf/
+        cat /home/bts/sparkdata/conf/.env.local
+        if [ -e "/home/bts/sparkdata/conf/.env.local" ]; then
+          echo 'Sourcing /home/bts/sparkdata/conf/.env.local'
          set -a
-          . /home/bts/.local/share/bts/sparkd/conf/.env.local
+          . /home/bts/sparkdata/conf/.env.local
          set +a
        fi
-        if [ -d /home/bts/.local/share/bts/sparkd/conf/plugins ]; then
-          echo 'Copying /home/bts/.local/share/bts/sparkd/conf/plugins to runtime dir (/home/bts/bin)'
-          cp -r /home/bts/.local/share/bts/sparkd/conf/plugins /home/bts/bin/
-        fi
        ./sparkd run"
    env_file:
      - .env
@@ -78,9 +67,6 @@ services:
      - "${SPARKD__QUICKBUS_PORT}:${SPARKD__QUICKBUS_PORT}"
      - "${SPARKD__SIM_FIX_PORT}:${SPARKD__SIM_FIX_PORT}"
    deploy:
-      restart_policy:
-        condition: any
-        delay: 3s
      resources:
        limits:
          memory: "${SPARKD__MEMORY:-4gb}"
@@ -91,10 +77,10 @@ services:
      - type: volume
        read_only: true
        source: sparkd_data
-        target: /home/bts/.local/share/bts/sparkd/data/logging.sparkd
+        target: /home/bts/sparkdata/data/logging.sparkd
        volume:
-          subpath: logging
-    command: ["tail", "-f", "/home/bts/.local/share/bts/sparkd/data/logging.sparkd/log"]
+          subpath: logging.sparkd
+    command: ["tail", "-f", "/home/bts/sparkdata/data/logging.sparkd/log"]

  sparkd-ping-test:
    depends_on:
@@ -109,89 +95,6 @@ services:
        RUN apt-get update && apt-get -y install iputils-ping
    command: [ "ping", "sparkd" ]

-  pubsub:
-    volumes:
-      - ${PUBSUB__DATA_DIR:-pubsub_data}:/home/bts/.local/share/bts/PubSubServer/data
-    build:
-      dockerfile: src/DataStreamSuite/PubSubServer/Dockerfile
-      args:
-        - SOURCE_DIR=${PUBSUB__SOURCE_DIR}
-        - CONFIG_DIR=${PUBSUB__CONFIG_DIR}
-    ports:
-      - 22019:22019
-    command: >
-      sh -c "
-        echo bts | sudo -S chown -R bts:bts /home/bts/.local/share/bts/PubSubServer
-        if [ -e "/home/bts/.local/share/bts/PubSubServer/data/logging.PubSubServer/log" ]; then
-          mv "/home/bts/.local/share/bts/PubSubServer/data/logging.PubSubServer/log" "/home/bts/.local/share/bts/PubSubServer/data/logging.PubSubServer/log_$(date '+%Y%m%d-%H%M')"
-        fi
-        if [ -e "/home/bts/.local/share/bts/PubSubServer/conf/.env" ]; then
-          echo 'Sourcing /home/bts/.local/share/bts/PubSubServer/conf/.env'
-          set -a
-          . /home/bts/.local/share/bts/PubSubServer/conf/.env
-          set +a
-        fi
-        if [ -e "/home/bts/.local/share/bts/PubSubServer/conf/.env.local" ]; then
-          echo 'Sourcing /home/bts/.local/share/bts/PubSubServer/conf/.env.local'
-          set -a
-          . /home/bts/.local/share/bts/PubSubServer/conf/.env.local
-          set +a
-        fi
-        ./PubSubServer"
-    env_file:
-      - .env
-      - ${PUBSUB__CONFIG_DIR}/.env
-
-  md-publisher:
-    volumes:
-      - ${MD_PUBLISHER__DATA_DIR:-md_publisher_data}:/home/bts/.local/share/bts/DataStreamPublisher/data
-      - historic_md:/home/bts/.local/share/bts/databento
-    build:
-      dockerfile: src/DataStreamSuite/DataStreamPublisher/Dockerfile
-      args:
-        - SOURCE_DIR=${MD_PUBLISHER__SOURCE_DIR}
-        - CONFIG_DIR=${MD_PUBLISHER__CONFIG_DIR}
-    command: >
-      sh -c "
-        echo bts | sudo -S chown -R bts:bts /home/bts/.local/share/bts/DataStreamPublisher
-        if [ -e "/home/bts/.local/share/bts/DataStreamPublisher/data/logging.DataStreamPublisher/log" ]; then
-          mv "/home/bts/.local/share/bts/DataStreamPublisher/data/logging.DataStreamPublisher/log" "/home/bts/.local/share/bts/DataStreamPublisher/data/logging.DataStreamPublisher/log_$(date '+%Y%m%d-%H%M')"
-        fi
-        if [ -e "/home/bts/.local/share/bts/DataStreamPublisher/conf/.env" ]; then
-          echo 'Sourcing /home/bts/.local/share/bts/DataStreamPublisher/conf/.env'
-          set -a
-          . /home/bts/.local/share/bts/DataStreamPublisher/conf/.env
-          set +a
-        fi
-        if [ -e "/home/bts/.local/share/bts/DataStreamPublisher/conf/.env.local" ]; then
-          echo 'Sourcing /home/bts/.local/share/bts/DataStreamPublisher/conf/.env.local'
-          set -a
-          . /home/bts/.local/share/bts/DataStreamPublisher/conf/.env.local
-          set +a
-        fi
-      dockerize -wait tcp://pubsub:22019 -timeout 10s ./DataStreamPublisher"
-    env_file:
-      - .env
-      - ${MD_PUBLISHER__CONFIG_DIR}/.env
-
-  api-server:
-    depends_on:
-      - sparkd
-    ports:
-      - 8008:8008
-      - 9443:9443
-    build:
-      dockerfile: src/client/spark_api_server/Dockerfile
-    command: [ "dockerize", "-wait", "tcp://sparkd:8000", "-timeout", "10s", "./spark_api_server", "run" ]
-    environment:
-      - ASPNETCORE_ENVIRONMENT=Staging
-
 volumes:
  sparkd_data:
    name: "sparkd_${SPARKD__CONFIG_PORT:-8000}"
-  pubsub_data:
-    name: "pubsub_22019"
-  md_publisher_data:
-    name: "md_publisher_22019"
-  historic_md:
-    name: "historic_md"
@@ -124,8 +124,8 @@ service OrderService {
 ```
 ### C++ Example
 ```cpp
-#include "spark/api/public/ServiceContext.h"
-#include "spark/api/services/OrderService.client.pb.h"
+#include "lib/api/public/ServiceContext.h"
+#include "lib/api/services/OrderService.client.pb.h"
 using namespace bts::api;
 ServiceContext ctx;
 auto orders = ctx.locate<orderman::OrderService>();
@@ -29,9 +29,9 @@ namespace sample {
 ```

 ```cpp
-#include "spark/api/public/ServiceContext.h"
-#include "spark/domain_model/Security.h"
-#include "spark/api/services/MarketDataService.client.pb.h"
+#include "lib/api/public/ServiceContext.h"
+#include "spark/lib/domain_model/Security.h"
+#include "lib/api/services/MarketDataService.client.pb.h"
 #include <thread>

 using namespace bts::api;
@@ -0,0 +1,6 @@
+{
+  "sdk": {
+    "version": "6.0.200",
+    "rollForward": "latestMajor"
+  }
+}
@@ -16,16 +16,11 @@ jmespath==1.0.1
 lazy-object-proxy==1.6.0
 MarkupSafe==3.0.2
 mccabe==0.6.1
-numpy==2.2.6
 platformdirs==2.3.0
 protobuf==4.21.12
 pybars3==0.9.7
-pybind11==2.13.6
-pybind11-stubgen==2.5.3
-pybit==5.11.0
 pycodestyle==2.7.0
 pycparser==2.20
-pycryptodome==3.23.0
 pylint==2.10.2
 PyMeta3==0.5.1
 python-dateutil==2.9.0.post0
@@ -36,7 +31,6 @@ six==1.17.0
 termcolor==1.1.0
 toml==0.10.2
 urllib3==2.0.3
-websocket-client==1.8.0
 Werkzeug==3.1.3
 wrapt==1.12.1
 yapf==0.31.0
@@ -1,164 +1,103 @@
-Microsoft Visual Studio Solution File, Format Version 12.00
-# Visual Studio Version 17
-VisualStudioVersion = 17.3.32922.545
-MinimumVisualStudioVersion = 15.0.26124.0
-Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "spark_api", "src\client\spark_api\spark_api.csproj", "{DC9A8975-8211-4FC9-A9BA-C8B9D26AD481}"
-EndProject
-Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Spark", "src\client\spark\Spark.csproj", "{9A4D4D07-1AB6-4B2B-B111-A4F34D6C267E}"
-EndProject
-Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Spark.Console", "src\client\Spark.Console\Spark.Console.csproj", "{387566C6-C6F6-412C-A30C-9855BBCB4816}"
-EndProject
-Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "spark_core", "src\client\spark_core\spark_core.csproj", "{6E449EB6-20AC-4FA6-B8D9-10F4BB7E1FEE}"
-EndProject
-Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "spark_sdk", "src\client\spark_sdk\spark_sdk.csproj", "{6963C314-83C4-4667-B0D0-C2F7AE208C29}"
-EndProject
-Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "spark_test", "src\client\spark_test\spark_test.csproj", "{ECD78CC5-138F-4809-A764-2358C2AC2E82}"
-EndProject
-Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "spark_websocket", "src\client\spark_websocket\spark_websocket.csproj", "{3201359D-C071-4089-9EDE-B04D759F1508}"
-EndProject
-Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "spark_api_server", "src\client\spark_api_server\spark_api_server.csproj", "{2B8C5D01-FF78-4BFA-B332-5E430586DAF9}"
-EndProject
-Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "DataStreamCore", "src\DataStreamSuite\DataStreamCore\DataStreamCore.csproj", "{4E4F491F-C688-43C4-88DA-AC92BC5FA1B8}"
-EndProject
-Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "DataStream", "src\DataStreamSuite\DataStream\DataStream.csproj", "{2AC54ADC-3246-427E-9CA9-FEA2DBCB9432}"
-EndProject
-Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "SharedStreams", "src\DataStreamSuite\SharedStreams\SharedStreams.csproj", "{4E96E5CC-7791-40B3-A8A1-C34FBB736B3B}"
-EndProject
-Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "PubSub", "src\DataStreamSuite\PubSub\PubSub.csproj", "{51C40BA9-F226-4A35-939D-1ED6F5FF7299}"
-EndProject
-Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "src", "src", "{4564E677-343A-46CB-AF02-AFC4E38BF94B}"
-EndProject
-Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "client", "client", "{56A8A70A-DA03-4CED-A144-EC10F1379FD8}"
-EndProject
-Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "DataStreamSuite", "DataStreamSuite", "{915EF238-CA0D-4E73-AF50-88CB3F29C7AD}"
-EndProject
-Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Bts.Extensions.Hosting", "src\DataStreamSuite\Bts.Extensions.Hosting\Bts.Extensions.Hosting.csproj", "{4E0E51AA-7B59-4808-B7CC-9EAEDF397460}"
-EndProject
-Global
-	GlobalSection(SolutionConfigurationPlatforms) = preSolution
-		Debug|x64 = Debug|x64
-		MinSizeRel|x64 = MinSizeRel|x64
-		Release|x64 = Release|x64
-		RelWithDebInfo|x64 = RelWithDebInfo|x64
-	EndGlobalSection
-	GlobalSection(ProjectConfigurationPlatforms) = postSolution
-		{DC9A8975-8211-4FC9-A9BA-C8B9D26AD481}.Debug|x64.ActiveCfg = Debug|x64
-		{DC9A8975-8211-4FC9-A9BA-C8B9D26AD481}.Debug|x64.Build.0 = Debug|x64
-		{DC9A8975-8211-4FC9-A9BA-C8B9D26AD481}.MinSizeRel|x64.ActiveCfg = Release|x64
-		{DC9A8975-8211-4FC9-A9BA-C8B9D26AD481}.MinSizeRel|x64.Build.0 = Release|x64
-		{DC9A8975-8211-4FC9-A9BA-C8B9D26AD481}.Release|x64.ActiveCfg = Release|x64
-		{DC9A8975-8211-4FC9-A9BA-C8B9D26AD481}.Release|x64.Build.0 = Release|x64
-		{DC9A8975-8211-4FC9-A9BA-C8B9D26AD481}.RelWithDebInfo|x64.ActiveCfg = Release|x64
-		{DC9A8975-8211-4FC9-A9BA-C8B9D26AD481}.RelWithDebInfo|x64.Build.0 = Release|x64
-		{9A4D4D07-1AB6-4B2B-B111-A4F34D6C267E}.Debug|x64.ActiveCfg = Debug|x64
-		{9A4D4D07-1AB6-4B2B-B111-A4F34D6C267E}.Debug|x64.Build.0 = Debug|x64
-		{9A4D4D07-1AB6-4B2B-B111-A4F34D6C267E}.MinSizeRel|x64.ActiveCfg = Release|x64
-		{9A4D4D07-1AB6-4B2B-B111-A4F34D6C267E}.MinSizeRel|x64.Build.0 = Release|x64
-		{9A4D4D07-1AB6-4B2B-B111-A4F34D6C267E}.Release|x64.ActiveCfg = Release|x64
-		{9A4D4D07-1AB6-4B2B-B111-A4F34D6C267E}.Release|x64.Build.0 = Release|x64
-		{9A4D4D07-1AB6-4B2B-B111-A4F34D6C267E}.RelWithDebInfo|x64.ActiveCfg = Release|x64
-		{9A4D4D07-1AB6-4B2B-B111-A4F34D6C267E}.RelWithDebInfo|x64.Build.0 = Release|x64
-		{387566C6-C6F6-412C-A30C-9855BBCB4816}.Debug|x64.ActiveCfg = Debug|x64
-		{387566C6-C6F6-412C-A30C-9855BBCB4816}.Debug|x64.Build.0 = Debug|x64
-		{387566C6-C6F6-412C-A30C-9855BBCB4816}.MinSizeRel|x64.ActiveCfg = Debug|x64
-		{387566C6-C6F6-412C-A30C-9855BBCB4816}.MinSizeRel|x64.Build.0 = Debug|x64
-		{387566C6-C6F6-412C-A30C-9855BBCB4816}.Release|x64.ActiveCfg = Release|x64
-		{387566C6-C6F6-412C-A30C-9855BBCB4816}.Release|x64.Build.0 = Release|x64
-		{387566C6-C6F6-412C-A30C-9855BBCB4816}.RelWithDebInfo|x64.ActiveCfg = Release|x64
-		{387566C6-C6F6-412C-A30C-9855BBCB4816}.RelWithDebInfo|x64.Build.0 = Release|x64
-		{6E449EB6-20AC-4FA6-B8D9-10F4BB7E1FEE}.Debug|x64.ActiveCfg = Debug|x64
-		{6E449EB6-20AC-4FA6-B8D9-10F4BB7E1FEE}.Debug|x64.Build.0 = Debug|x64
-		{6E449EB6-20AC-4FA6-B8D9-10F4BB7E1FEE}.MinSizeRel|x64.ActiveCfg = Release|x64
-		{6E449EB6-20AC-4FA6-B8D9-10F4BB7E1FEE}.MinSizeRel|x64.Build.0 = Release|x64
-		{6E449EB6-20AC-4FA6-B8D9-10F4BB7E1FEE}.Release|x64.ActiveCfg = Release|x64
-		{6E449EB6-20AC-4FA6-B8D9-10F4BB7E1FEE}.Release|x64.Build.0 = Release|x64
-		{6E449EB6-20AC-4FA6-B8D9-10F4BB7E1FEE}.RelWithDebInfo|x64.ActiveCfg = Release|x64
-		{6E449EB6-20AC-4FA6-B8D9-10F4BB7E1FEE}.RelWithDebInfo|x64.Build.0 = Release|x64
-		{6963C314-83C4-4667-B0D0-C2F7AE208C29}.Debug|x64.ActiveCfg = Debug|x64
-		{6963C314-83C4-4667-B0D0-C2F7AE208C29}.Debug|x64.Build.0 = Debug|x64
-		{6963C314-83C4-4667-B0D0-C2F7AE208C29}.MinSizeRel|x64.ActiveCfg = Release|x64
-		{6963C314-83C4-4667-B0D0-C2F7AE208C29}.MinSizeRel|x64.Build.0 = Release|x64
-		{6963C314-83C4-4667-B0D0-C2F7AE208C29}.Release|x64.ActiveCfg = Release|x64
-		{6963C314-83C4-4667-B0D0-C2F7AE208C29}.Release|x64.Build.0 = Release|x64
-		{6963C314-83C4-4667-B0D0-C2F7AE208C29}.RelWithDebInfo|x64.ActiveCfg = Release|x64
-		{6963C314-83C4-4667-B0D0-C2F7AE208C29}.RelWithDebInfo|x64.Build.0 = Release|x64
-		{ECD78CC5-138F-4809-A764-2358C2AC2E82}.Debug|x64.ActiveCfg = Debug|x64
-		{ECD78CC5-138F-4809-A764-2358C2AC2E82}.Debug|x64.Build.0 = Debug|x64
-		{ECD78CC5-138F-4809-A764-2358C2AC2E82}.MinSizeRel|x64.ActiveCfg = Debug|x64
-		{ECD78CC5-138F-4809-A764-2358C2AC2E82}.MinSizeRel|x64.Build.0 = Debug|x64
-		{ECD78CC5-138F-4809-A764-2358C2AC2E82}.Release|x64.ActiveCfg = Release|x64
-		{ECD78CC5-138F-4809-A764-2358C2AC2E82}.Release|x64.Build.0 = Release|x64
-		{ECD78CC5-138F-4809-A764-2358C2AC2E82}.RelWithDebInfo|x64.ActiveCfg = Release|x64
-		{ECD78CC5-138F-4809-A764-2358C2AC2E82}.RelWithDebInfo|x64.Build.0 = Release|x64
-		{3201359D-C071-4089-9EDE-B04D759F1508}.Debug|x64.ActiveCfg = Debug|x64
-		{3201359D-C071-4089-9EDE-B04D759F1508}.Debug|x64.Build.0 = Debug|x64
-		{3201359D-C071-4089-9EDE-B04D759F1508}.MinSizeRel|x64.ActiveCfg = Debug|x64
-		{3201359D-C071-4089-9EDE-B04D759F1508}.MinSizeRel|x64.Build.0 = Debug|x64
-		{3201359D-C071-4089-9EDE-B04D759F1508}.Release|x64.ActiveCfg = Release|x64
-		{3201359D-C071-4089-9EDE-B04D759F1508}.Release|x64.Build.0 = Release|x64
-		{3201359D-C071-4089-9EDE-B04D759F1508}.RelWithDebInfo|x64.ActiveCfg = Release|x64
-		{3201359D-C071-4089-9EDE-B04D759F1508}.RelWithDebInfo|x64.Build.0 = Release|x64
-		{2B8C5D01-FF78-4BFA-B332-5E430586DAF9}.Debug|x64.ActiveCfg = Debug|x64
-		{2B8C5D01-FF78-4BFA-B332-5E430586DAF9}.Debug|x64.Build.0 = Debug|x64
-		{2B8C5D01-FF78-4BFA-B332-5E430586DAF9}.MinSizeRel|x64.ActiveCfg = Debug|x64
-		{2B8C5D01-FF78-4BFA-B332-5E430586DAF9}.MinSizeRel|x64.Build.0 = Debug|x64
-		{2B8C5D01-FF78-4BFA-B332-5E430586DAF9}.Release|x64.ActiveCfg = Release|x64
-		{2B8C5D01-FF78-4BFA-B332-5E430586DAF9}.Release|x64.Build.0 = Release|x64
-		{2B8C5D01-FF78-4BFA-B332-5E430586DAF9}.RelWithDebInfo|x64.ActiveCfg = Release|x64
-		{2B8C5D01-FF78-4BFA-B332-5E430586DAF9}.RelWithDebInfo|x64.Build.0 = Release|x64
-		{4E4F491F-C688-43C4-88DA-AC92BC5FA1B8}.Debug|x64.ActiveCfg = Debug|Any CPU
-		{4E4F491F-C688-43C4-88DA-AC92BC5FA1B8}.Debug|x64.Build.0 = Debug|Any CPU
-		{4E4F491F-C688-43C4-88DA-AC92BC5FA1B8}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
-		{4E4F491F-C688-43C4-88DA-AC92BC5FA1B8}.MinSizeRel|x64.Build.0 = Debug|Any CPU
-		{4E4F491F-C688-43C4-88DA-AC92BC5FA1B8}.Release|x64.ActiveCfg = Release|Any CPU
-		{4E4F491F-C688-43C4-88DA-AC92BC5FA1B8}.Release|x64.Build.0 = Release|Any CPU
-		{4E4F491F-C688-43C4-88DA-AC92BC5FA1B8}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
-		{4E4F491F-C688-43C4-88DA-AC92BC5FA1B8}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
-		{2AC54ADC-3246-427E-9CA9-FEA2DBCB9432}.Debug|x64.ActiveCfg = Debug|Any CPU
-		{2AC54ADC-3246-427E-9CA9-FEA2DBCB9432}.Debug|x64.Build.0 = Debug|Any CPU
-		{2AC54ADC-3246-427E-9CA9-FEA2DBCB9432}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
-		{2AC54ADC-3246-427E-9CA9-FEA2DBCB9432}.MinSizeRel|x64.Build.0 = Debug|Any CPU
-		{2AC54ADC-3246-427E-9CA9-FEA2DBCB9432}.Release|x64.ActiveCfg = Release|Any CPU
-		{2AC54ADC-3246-427E-9CA9-FEA2DBCB9432}.Release|x64.Build.0 = Release|Any CPU
-		{2AC54ADC-3246-427E-9CA9-FEA2DBCB9432}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
-		{2AC54ADC-3246-427E-9CA9-FEA2DBCB9432}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
-		{4E96E5CC-7791-40B3-A8A1-C34FBB736B3B}.Debug|x64.ActiveCfg = Debug|Any CPU
-		{4E96E5CC-7791-40B3-A8A1-C34FBB736B3B}.Debug|x64.Build.0 = Debug|Any CPU
-		{4E96E5CC-7791-40B3-A8A1-C34FBB736B3B}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
-		{4E96E5CC-7791-40B3-A8A1-C34FBB736B3B}.MinSizeRel|x64.Build.0 = Debug|Any CPU
-		{4E96E5CC-7791-40B3-A8A1-C34FBB736B3B}.Release|x64.ActiveCfg = Release|Any CPU
-		{4E96E5CC-7791-40B3-A8A1-C34FBB736B3B}.Release|x64.Build.0 = Release|Any CPU
-		{4E96E5CC-7791-40B3-A8A1-C34FBB736B3B}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
-		{4E96E5CC-7791-40B3-A8A1-C34FBB736B3B}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
-		{51C40BA9-F226-4A35-939D-1ED6F5FF7299}.Debug|x64.ActiveCfg = Debug|Any CPU
-		{51C40BA9-F226-4A35-939D-1ED6F5FF7299}.Debug|x64.Build.0 = Debug|Any CPU
-		{51C40BA9-F226-4A35-939D-1ED6F5FF7299}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
-		{51C40BA9-F226-4A35-939D-1ED6F5FF7299}.MinSizeRel|x64.Build.0 = Debug|Any CPU
-		{51C40BA9-F226-4A35-939D-1ED6F5FF7299}.Release|x64.ActiveCfg = Release|Any CPU
-		{51C40BA9-F226-4A35-939D-1ED6F5FF7299}.Release|x64.Build.0 = Release|Any CPU
-		{51C40BA9-F226-4A35-939D-1ED6F5FF7299}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
-		{51C40BA9-F226-4A35-939D-1ED6F5FF7299}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
-		{4E0E51AA-7B59-4808-B7CC-9EAEDF397460}.Debug|x64.ActiveCfg = Debug|Any CPU
-		{4E0E51AA-7B59-4808-B7CC-9EAEDF397460}.Debug|x64.Build.0 = Debug|Any CPU
-		{4E0E51AA-7B59-4808-B7CC-9EAEDF397460}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
-		{4E0E51AA-7B59-4808-B7CC-9EAEDF397460}.MinSizeRel|x64.Build.0 = Debug|Any CPU
-		{4E0E51AA-7B59-4808-B7CC-9EAEDF397460}.Release|x64.ActiveCfg = Release|Any CPU
-		{4E0E51AA-7B59-4808-B7CC-9EAEDF397460}.Release|x64.Build.0 = Release|Any CPU
-		{4E0E51AA-7B59-4808-B7CC-9EAEDF397460}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
-		{4E0E51AA-7B59-4808-B7CC-9EAEDF397460}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
-	EndGlobalSection
-	GlobalSection(SolutionProperties) = preSolution
-		HideSolutionNode = FALSE
-	EndGlobalSection
-	GlobalSection(ExtensibilityGlobals) = postSolution
-		SolutionGuid = {422D1620-0166-4C26-9A81-CA8A2EB1AB58}
-	EndGlobalSection
-	GlobalSection(Performance) = preSolution
-		HasPerformanceSessions = true
-	EndGlobalSection
-	GlobalSection(NestedProjects) = preSolution
-		{56A8A70A-DA03-4CED-A144-EC10F1379FD8} = {4564E677-343A-46CB-AF02-AFC4E38BF94B}
-		{915EF238-CA0D-4E73-AF50-88CB3F29C7AD} = {4564E677-343A-46CB-AF02-AFC4E38BF94B}
-		{4E0E51AA-7B59-4808-B7CC-9EAEDF397460} = {915EF238-CA0D-4E73-AF50-88CB3F29C7AD}
-	EndGlobalSection
-EndGlobal
+Microsoft Visual Studio Solution File, Format Version 12.00
+# Visual Studio Version 17
+VisualStudioVersion = 17.3.32922.545
+MinimumVisualStudioVersion = 15.0.26124.0
+Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "spark_api", "src\client\spark_api\spark_api.csproj", "{DC9A8975-8211-4FC9-A9BA-C8B9D26AD481}"
+EndProject
+Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Spark", "src\client\spark\Spark.csproj", "{9A4D4D07-1AB6-4B2B-B111-A4F34D6C267E}"
+EndProject
+Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Spark.Console", "src\client\Spark.Console\Spark.Console.csproj", "{387566C6-C6F6-412C-A30C-9855BBCB4816}"
+EndProject
+Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "spark_core", "src\client\spark_core\spark_core.csproj", "{6E449EB6-20AC-4FA6-B8D9-10F4BB7E1FEE}"
+EndProject
+Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "spark_sdk", "src\client\spark_sdk\spark_sdk.csproj", "{6963C314-83C4-4667-B0D0-C2F7AE208C29}"
+EndProject
+Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "spark_test", "src\client\spark_test\spark_test.csproj", "{ECD78CC5-138F-4809-A764-2358C2AC2E82}"
+EndProject
+Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "spark_websocket", "src\client\spark_websocket\spark_websocket.csproj", "{3201359D-C071-4089-9EDE-B04D759F1508}"
+EndProject
+Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "spark_api_server", "src\client\spark_api_server\spark_api_server.csproj", "{2B8C5D01-FF78-4BFA-B332-5E430586DAF9}"
+EndProject
+Global
+	GlobalSection(SolutionConfigurationPlatforms) = preSolution
+		Debug|x64 = Debug|x64
+		MinSizeRel|x64 = MinSizeRel|x64
+		Release|x64 = Release|x64
+		RelWithDebInfo|x64 = RelWithDebInfo|x64
+	EndGlobalSection
+	GlobalSection(ProjectConfigurationPlatforms) = postSolution
+		{DC9A8975-8211-4FC9-A9BA-C8B9D26AD481}.Debug|x64.ActiveCfg = Debug|x64
+		{DC9A8975-8211-4FC9-A9BA-C8B9D26AD481}.Debug|x64.Build.0 = Debug|x64
+		{DC9A8975-8211-4FC9-A9BA-C8B9D26AD481}.MinSizeRel|x64.ActiveCfg = Release|x64
+		{DC9A8975-8211-4FC9-A9BA-C8B9D26AD481}.MinSizeRel|x64.Build.0 = Release|x64
+		{DC9A8975-8211-4FC9-A9BA-C8B9D26AD481}.Release|x64.ActiveCfg = Release|x64
+		{DC9A8975-8211-4FC9-A9BA-C8B9D26AD481}.Release|x64.Build.0 = Release|x64
+		{DC9A8975-8211-4FC9-A9BA-C8B9D26AD481}.RelWithDebInfo|x64.ActiveCfg = Release|x64
+		{DC9A8975-8211-4FC9-A9BA-C8B9D26AD481}.RelWithDebInfo|x64.Build.0 = Release|x64
+		{9A4D4D07-1AB6-4B2B-B111-A4F34D6C267E}.Debug|x64.ActiveCfg = Debug|x64
+		{9A4D4D07-1AB6-4B2B-B111-A4F34D6C267E}.Debug|x64.Build.0 = Debug|x64
+		{9A4D4D07-1AB6-4B2B-B111-A4F34D6C267E}.MinSizeRel|x64.ActiveCfg = Release|x64
+		{9A4D4D07-1AB6-4B2B-B111-A4F34D6C267E}.MinSizeRel|x64.Build.0 = Release|x64
+		{9A4D4D07-1AB6-4B2B-B111-A4F34D6C267E}.Release|x64.ActiveCfg = Release|x64
+		{9A4D4D07-1AB6-4B2B-B111-A4F34D6C267E}.Release|x64.Build.0 = Release|x64
+		{9A4D4D07-1AB6-4B2B-B111-A4F34D6C267E}.RelWithDebInfo|x64.ActiveCfg = Release|x64
+		{9A4D4D07-1AB6-4B2B-B111-A4F34D6C267E}.RelWithDebInfo|x64.Build.0 = Release|x64
+		{387566C6-C6F6-412C-A30C-9855BBCB4816}.Debug|x64.ActiveCfg = Debug|x64
+		{387566C6-C6F6-412C-A30C-9855BBCB4816}.Debug|x64.Build.0 = Debug|x64
+		{387566C6-C6F6-412C-A30C-9855BBCB4816}.MinSizeRel|x64.ActiveCfg = Debug|x64
+		{387566C6-C6F6-412C-A30C-9855BBCB4816}.MinSizeRel|x64.Build.0 = Debug|x64
+		{387566C6-C6F6-412C-A30C-9855BBCB4816}.Release|x64.ActiveCfg = Release|x64
+		{387566C6-C6F6-412C-A30C-9855BBCB4816}.Release|x64.Build.0 = Release|x64
+		{387566C6-C6F6-412C-A30C-9855BBCB4816}.RelWithDebInfo|x64.ActiveCfg = Release|x64
+		{387566C6-C6F6-412C-A30C-9855BBCB4816}.RelWithDebInfo|x64.Build.0 = Release|x64
+		{6E449EB6-20AC-4FA6-B8D9-10F4BB7E1FEE}.Debug|x64.ActiveCfg = Debug|x64
+		{6E449EB6-20AC-4FA6-B8D9-10F4BB7E1FEE}.Debug|x64.Build.0 = Debug|x64
+		{6E449EB6-20AC-4FA6-B8D9-10F4BB7E1FEE}.MinSizeRel|x64.ActiveCfg = Release|x64
+		{6E449EB6-20AC-4FA6-B8D9-10F4BB7E1FEE}.MinSizeRel|x64.Build.0 = Release|x64
+		{6E449EB6-20AC-4FA6-B8D9-10F4BB7E1FEE}.Release|x64.ActiveCfg = Release|x64
+		{6E449EB6-20AC-4FA6-B8D9-10F4BB7E1FEE}.Release|x64.Build.0 = Release|x64
+		{6E449EB6-20AC-4FA6-B8D9-10F4BB7E1FEE}.RelWithDebInfo|x64.ActiveCfg = Release|x64
+		{6E449EB6-20AC-4FA6-B8D9-10F4BB7E1FEE}.RelWithDebInfo|x64.Build.0 = Release|x64
+		{6963C314-83C4-4667-B0D0-C2F7AE208C29}.Debug|x64.ActiveCfg = Debug|x64
+		{6963C314-83C4-4667-B0D0-C2F7AE208C29}.Debug|x64.Build.0 = Debug|x64
+		{6963C314-83C4-4667-B0D0-C2F7AE208C29}.MinSizeRel|x64.ActiveCfg = Release|x64
+		{6963C314-83C4-4667-B0D0-C2F7AE208C29}.MinSizeRel|x64.Build.0 = Release|x64
+		{6963C314-83C4-4667-B0D0-C2F7AE208C29}.Release|x64.ActiveCfg = Release|x64
+		{6963C314-83C4-4667-B0D0-C2F7AE208C29}.Release|x64.Build.0 = Release|x64
+		{6963C314-83C4-4667-B0D0-C2F7AE208C29}.RelWithDebInfo|x64.ActiveCfg = Release|x64
+		{6963C314-83C4-4667-B0D0-C2F7AE208C29}.RelWithDebInfo|x64.Build.0 = Release|x64
+		{ECD78CC5-138F-4809-A764-2358C2AC2E82}.Debug|x64.ActiveCfg = Debug|x64
+		{ECD78CC5-138F-4809-A764-2358C2AC2E82}.Debug|x64.Build.0 = Debug|x64
+		{ECD78CC5-138F-4809-A764-2358C2AC2E82}.MinSizeRel|x64.ActiveCfg = Debug|x64
+		{ECD78CC5-138F-4809-A764-2358C2AC2E82}.MinSizeRel|x64.Build.0 = Debug|x64
+		{ECD78CC5-138F-4809-A764-2358C2AC2E82}.Release|x64.ActiveCfg = Release|x64
+		{ECD78CC5-138F-4809-A764-2358C2AC2E82}.Release|x64.Build.0 = Release|x64
+		{ECD78CC5-138F-4809-A764-2358C2AC2E82}.RelWithDebInfo|x64.ActiveCfg = Release|x64
+		{ECD78CC5-138F-4809-A764-2358C2AC2E82}.RelWithDebInfo|x64.Build.0 = Release|x64
+		{3201359D-C071-4089-9EDE-B04D759F1508}.Debug|x64.ActiveCfg = Debug|x64
+		{3201359D-C071-4089-9EDE-B04D759F1508}.Debug|x64.Build.0 = Debug|x64
+		{3201359D-C071-4089-9EDE-B04D759F1508}.MinSizeRel|x64.ActiveCfg = Debug|x64
+		{3201359D-C071-4089-9EDE-B04D759F1508}.MinSizeRel|x64.Build.0 = Debug|x64
+		{3201359D-C071-4089-9EDE-B04D759F1508}.Release|x64.ActiveCfg = Release|x64
+		{3201359D-C071-4089-9EDE-B04D759F1508}.Release|x64.Build.0 = Release|x64
+		{3201359D-C071-4089-9EDE-B04D759F1508}.RelWithDebInfo|x64.ActiveCfg = Release|x64
+		{3201359D-C071-4089-9EDE-B04D759F1508}.RelWithDebInfo|x64.Build.0 = Release|x64
+		{2B8C5D01-FF78-4BFA-B332-5E430586DAF9}.Debug|x64.ActiveCfg = Debug|x64
+		{2B8C5D01-FF78-4BFA-B332-5E430586DAF9}.Debug|x64.Build.0 = Debug|x64
+		{2B8C5D01-FF78-4BFA-B332-5E430586DAF9}.MinSizeRel|x64.ActiveCfg = Debug|x64
+		{2B8C5D01-FF78-4BFA-B332-5E430586DAF9}.MinSizeRel|x64.Build.0 = Debug|x64
+		{2B8C5D01-FF78-4BFA-B332-5E430586DAF9}.Release|x64.ActiveCfg = Release|x64
+		{2B8C5D01-FF78-4BFA-B332-5E430586DAF9}.Release|x64.Build.0 = Release|x64
+		{2B8C5D01-FF78-4BFA-B332-5E430586DAF9}.RelWithDebInfo|x64.ActiveCfg = Release|x64
+		{2B8C5D01-FF78-4BFA-B332-5E430586DAF9}.RelWithDebInfo|x64.Build.0 = Release|x64
+	EndGlobalSection
+	GlobalSection(SolutionProperties) = preSolution
+		HideSolutionNode = FALSE
+	EndGlobalSection
+	GlobalSection(ExtensibilityGlobals) = postSolution
+		SolutionGuid = {422D1620-0166-4C26-9A81-CA8A2EB1AB58}
+	EndGlobalSection
+	GlobalSection(Performance) = preSolution
+		HasPerformanceSessions = true
+	EndGlobalSection
+EndGlobal
@@ -8,6 +8,7 @@
   public int Integer;
   public double Real;
   public string Identifier;
+   public DateTime DateTime; 
   public ImmutableList<IArithmeticExpression> Arguments;
   public ImmutableList<Assignment> Assignments;
   public IArithmeticExpression Argument;
@@ -1,7 +1,7 @@
 <Project Sdk="Microsoft.NET.Sdk">

  <PropertyGroup>
-    <TargetFramework>net8.0</TargetFramework>
+    <TargetFramework>net7.0</TargetFramework>
    <ImplicitUsings>enable</ImplicitUsings>
    <Nullable>enable</Nullable>
  </PropertyGroup>
@@ -1,7 +1,7 @@
 <Project Sdk="Microsoft.NET.Sdk">

  <PropertyGroup>
-    <TargetFramework>net8.0</TargetFramework>
+    <TargetFramework>net7.0</TargetFramework>
    <ImplicitUsings>enable</ImplicitUsings>
    <Nullable>enable</Nullable>
  </PropertyGroup>
@@ -1,7 +1,7 @@
 <Project Sdk="Microsoft.NET.Sdk">

  <PropertyGroup>
-    <TargetFramework>net8.0</TargetFramework>
+    <TargetFramework>net7.0</TargetFramework>
    <ImplicitUsings>enable</ImplicitUsings>
    <Nullable>enable</Nullable>
  </PropertyGroup>
@@ -1,6 +1,4 @@
 using System;
-using System.Runtime.Versioning;
-
 using Microsoft.Win32;

 namespace BCG
@@ -12,7 +10,6 @@ namespace BCG
         rk.SetValue (name, value.ToString ());
      }

-      [SupportedOSPlatform("windows")]
      public static double getValue (RegistryKey rk, string name)
      {
         object v = rk.GetValue (name);
@@ -1,11 +1,19 @@
 namespace Bts.BooleanExpressions;

+using bts;
+
 class BooleanExpression : IBooleanExpression {
   readonly Func<IDictionary<string, object?>, bool> evaluate;

   public BooleanExpression(Func<IDictionary<string, object?>, bool> evaluate) =>
      this.evaluate = evaluate;

-   public bool Evaluate(IDictionary<string, object?> env) =>
-      evaluate(env);
+   public bool Evaluate(IDictionary<string, object?> env) {
+      try {
+         return evaluate(env);
+      } catch (Exception ex) {
+         Log.assert(false, $"BooleanExpression.Evaluate failed: {ex}");
+         throw;
+      }
+   }
 }
@@ -47,6 +47,11 @@ IsOfType                 '/'

 %%

+{StringConstant} {
+   yylval.String = yytext.Substring(1, yytext.Length - 2).Replace(@"\'", "'");
+   return (int)Token.STRING_CONSTANT;
+} 
+
 {BooleanConstant} {
   yylval.Boolean = yytext.ToLower() switch {
      "true" => true,
@@ -57,11 +62,6 @@ IsOfType                 '/'
   return (int)Token.BOOLEAN_CONSTANT;
 }

-{StringConstant} {
-   yylval.String = yytext.Substring(1, yytext.Length - 2).Replace(@"\'", "'");
-   return (int)Token.STRING_CONSTANT;
-} 
-
 {NullConstant} {
   yylval.Null = DBNull.Value;
   return (int)Token.NULL_CONSTANT;
@@ -1,14 +1,14 @@
 <Project Sdk="Microsoft.NET.Sdk">

  <PropertyGroup>
-    <TargetFramework>net8.0</TargetFramework>
+    <TargetFramework>net7.0</TargetFramework>
    <ImplicitUsings>enable</ImplicitUsings>
    <Nullable>enable</Nullable>
  </PropertyGroup>

  <ItemGroup>
    <PackageReference Include="Google.Protobuf" Version="3.24.3" />
-    <PackageReference Include="YaccLexTools" Version="1.2.3" />
+    <PackageReference Include="YaccLexTools" Version="1.2.0" />
  </ItemGroup>

  <ItemGroup>
@@ -24,6 +24,8 @@ namespace QUT.GplexBuffers
        public BufferException(string message) : base(message) { }
        public BufferException(string message, Exception innerException)
            : base(message, innerException) { }
+        protected BufferException(SerializationInfo info, StreamingContext context)
+            : base(info, context) { }
    }

    internal abstract class ScanBuff
@@ -58,10 +60,12 @@ namespace QUT.GplexBuffers
            return new BuildBuffer(source);
        }

+#if (!BYTEMODE)
        public static ScanBuff GetBuffer(Stream source, int fallbackCodePage)
        {
            return new BuildBuffer(source, fallbackCodePage);
        }
+#endif // !BYTEMODE
 #endif // !NOFILES
    }

@@ -349,15 +353,17 @@ namespace QUT.GplexBuffers
        {
            FileStream fStrm = (stream as FileStream);
            if (fStrm != null) FileName = fStrm.Name;
-            NextBlk = BlockReaderFactory.Get(stream);
+            NextBlk = BlockReaderFactory.Raw(stream);
        }

+#if (!BYTEMODE)
        public BuildBuffer(Stream stream, int fallbackCodePage)
        {
            FileStream fStrm = (stream as FileStream);
            if (fStrm != null) FileName = fStrm.Name;
            NextBlk = BlockReaderFactory.Get(stream, fallbackCodePage);
        }
+#endif

        /// <summary>
        /// Marks a conservative lower bound for the buffer,
@@ -428,19 +434,43 @@ namespace QUT.GplexBuffers
    //
    internal static class BlockReaderFactory
    {
+        public static BlockReader Raw(Stream stream)
+        {
+            return delegate(char[] block, int index, int number)
+            {
+                byte[] b = new byte[number];
+                int count = stream.Read(b, 0, number);
+                int i = 0;
+                int j = index;
+                for (; i < count; i++, j++)
+                    block[j] = (char)b[i];
+                return count;
+            };
+        }

-        public static BlockReader Get(Stream stream, int fallbackCodePage = -2)
+#if (!BYTEMODE)
+        public static BlockReader Get(Stream stream, int fallbackCodePage)
        {
            Encoding encoding;
            int preamble = Preamble(stream);

            if (preamble != 0)  // There is a valid BOM here!
                encoding = Encoding.GetEncoding(preamble);
-            else if (fallbackCodePage >= 0) // Anything but "guess"
+            else if (fallbackCodePage == -1) // Fallback is "raw" bytes
+                return Raw(stream);
+            else if (fallbackCodePage != -2) // Anything but "guess"
                encoding = Encoding.GetEncoding(fallbackCodePage);
            else // This is the "guess" option
-                encoding = GuessEncoding(stream);
-
+            {
+                int guess = new Guesser(stream).GuessCodePage();
+                stream.Seek(0, SeekOrigin.Begin);
+                if (guess == -1) // ==> this is a 7-bit file
+                    encoding = Encoding.ASCII;
+                else if (guess == 65001)
+                    encoding = Encoding.UTF8;
+                else             // ==> use the machine default
+                    encoding = Encoding.Default;
+            }
            StreamReader reader = new StreamReader(stream, encoding);
            return reader.Read;
        }
@@ -458,14 +488,6 @@ namespace QUT.GplexBuffers
            int b2 = stream.ReadByte();
            if (b0 == 0xef && b1 == 0xbb && b2 == 0xbf)
                return 65001; // UTF8
-
-            int b3 = stream.ReadByte();
-            // Check for UTF-32 BOM (Byte Order Mark)
-            if (b0 == 0x00 && b1 == 0x00 && b2 == 0xFE && b3 == 0xFF)
-                return 12001; // UTF-32BE
-            if (b0 == 0xFF && b1 == 0xFE && b2 == 0x00 && b3 == 0x00)
-                return 12000; // UTF-32LE
-
            //
            // There is no unicode preamble, so we
            // return denoter for the machine default.
@@ -473,150 +495,7 @@ namespace QUT.GplexBuffers
            stream.Seek(0, SeekOrigin.Begin);
            return 0;
        }
-
-        private static Encoding GuessEncoding(Stream stream)
-        {
-            stream.Seek(0, SeekOrigin.Begin);
-            if (IsUtf16(stream, out Encoding encoding))
-            {
-                stream.Seek(0, SeekOrigin.Begin);
-                return encoding;
-            }
-
-            stream.Seek(0, SeekOrigin.Begin);
-            if (IsUtf8(stream))
-            {
-                stream.Seek(0, SeekOrigin.Begin);
-                return Encoding.UTF8;
-            }
-
-            stream.Seek(0, SeekOrigin.Begin);
-            if (IsAscii(stream))
-            {
-                stream.Seek(0, SeekOrigin.Begin);
-                return Encoding.ASCII;
-            }
-
-#if !NET462_OR_GREATER
-            stream.Seek(0, SeekOrigin.Begin);
-            if (IsLatin1(stream))
-            {
-                stream.Seek(0, SeekOrigin.Begin);
-                return Encoding.Latin1;
-            }
-#endif
-            stream.Seek(0, SeekOrigin.Begin);
-            return Encoding.Default;
-        }
-
-        private static bool IsUtf8(Stream stream)
-        {
-            int b;
-            while ((b = stream.ReadByte()) != -1)
-            {
-                if ((b & 0x80) == 0) // 0xxxxxxx
-                {
-                    continue;
-                }
-                else if ((b & 0xE0) == 0xC0) // 110xxxxx 10xxxxxx
-                {
-                    if (!IsValidUtf8ContinuationByte(stream)) return false;
-                }
-                else if ((b & 0xF0) == 0xE0) // 1110xxxx 10xxxxxx 10xxxxxx
-                {
-                    if (!IsValidUtf8ContinuationByte(stream) || !IsValidUtf8ContinuationByte(stream)) return false;
-                }
-                else if ((b & 0xF8) == 0xF0) // 11110xxx 10xxxxxx 10xxxxxx 10xxxxxx
-                {
-                    if (!IsValidUtf8ContinuationByte(stream) || !IsValidUtf8ContinuationByte(stream) || !IsValidUtf8ContinuationByte(stream)) return false;
-                }
-                else
-                {
-                    return false;
-                }
-            }
-            return true;
-        }
-
-        private static bool IsValidUtf8ContinuationByte(Stream stream)
-        {
-            int b = stream.ReadByte();
-            return b != -1 && (b & 0xC0) == 0x80; // 10xxxxxx
-        }
-
-        private static bool IsUtf16(Stream stream, out Encoding encoding)
-        {
-            encoding = Encoding.Default;
-
-            if (stream.Length >= 2)
-            {
-                // Read the first two bytes
-                int b0 = stream.ReadByte();
-                int b1 = stream.ReadByte();
-
-                // Check for UTF-16 BOM (Byte Order Mark)
-                if (b0 == 0xFF && b1 == 0xFE)
-                {
-                    encoding = Encoding.Unicode;
-                    return true; // UTF-16LE
-                }
-                if (b0 == 0xFE && b1 == 0xFF)
-                {
-                    encoding = Encoding.BigEndianUnicode;
-                    return true; // UTF-16BE
-                }
-
-                // Reset the stream position
-                stream.Seek(0, SeekOrigin.Begin);
-
-                // Check for common UTF-16 patterns
-                byte[] buffer = new byte[2];
-                while (stream.Read(buffer, 0, 2) == 2)
-                {
-                    if (buffer[0] == 0x00 && buffer[1] != 0x00)
-                    {
-                        encoding = Encoding.BigEndianUnicode;
-                        return true; // Likely UTF-16BE
-                    }
-                    if (buffer[0] != 0x00 && buffer[1] == 0x00)
-                    {
-                        encoding = Encoding.Unicode;
-                        return true; // Likely UTF-16LE
-                    }
-                }
-            }
-
-            return false;
-        }
-
-        private static bool IsAscii(Stream stream)
-        {
-            int b;
-            while ((b = stream.ReadByte()) != -1)
-            {
-                // Latin1 includes byte values from 0x00 to 0xFF
-                if (b > 0x7F)
-                {
-                    return false;
-                }
-            }
-            return true;
-        }
-
-        private static bool IsLatin1(Stream stream)
-        {
-            int b;
-            while ((b = stream.ReadByte()) != -1)
-            {
-                // Latin1 includes byte values from 0x00 to 0xFF
-                if (b > 0xFF)
-                {
-                    return false;
-                }
-            }
-            return true;
-        }
-
+#endif // !BYTEMODE
    }
 #endif // !NOFILES
    #endregion Buffer classes
@@ -661,6 +540,180 @@ namespace QUT.GplexBuffers
            return 0;
        }
    }
+#region guesser
+#if (!BYTEMODE)
+    // ==============================================================
+    // ============          Encoding Guesser           =============
+    // ==============================================================
+
+    /// <summary>
+    /// This class provides a simple finite state automaton that
+    /// scans the file looking for (1) valid UTF-8 byte patterns,
+    /// (2) bytes >= 0x80 which are not part of a UTF-8 sequence.
+    /// The method then guesses whether it is UTF-8 or maybe some 
+    /// local machine default encoding.  This works well for the
+    /// various Latin encodings.
+    /// </summary>
+    internal class Guesser
+    {
+        ScanBuff buffer;
+
+        public int GuessCodePage() { return Scan(); }
+
+        const int maxAccept = 10;
+        const int initial = 0;
+        const int eofNum = 0;
+        const int goStart = -1;
+        const int INITIAL = 0;
+        const int EndToken = 0;
+
+        #region user code
+        /* 
+         *  Reads the bytes of a file to determine if it is 
+         *  UTF-8 or a single-byte code page file.
+         */
+        public long utfX;
+        public long uppr;
+        #endregion user code
+
+        int state;
+        int currentStart = startState[0];
+        int code;
+
+        #region ScannerTables
+        static int[] startState = new int[] { 11, 0 };
+
+        #region CharacterMap
+        static sbyte[] map = new sbyte[256] {
+/*     '\0' */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 
+/*   '\x10' */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 
+/*   '\x20' */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 
+/*      '0' */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 
+/*      '@' */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 
+/*      'P' */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 
+/*      '`' */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 
+/*      'p' */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 
+/*   '\x80' */ 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 
+/*   '\x90' */ 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 
+/*   '\xA0' */ 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 
+/*   '\xB0' */ 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 
+/*   '\xC0' */ 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 
+/*   '\xD0' */ 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 
+/*   '\xE0' */ 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 
+/*   '\xF0' */ 4, 4, 4, 4, 4, 4, 4, 4, 5, 5, 5, 5, 5, 5, 5, 5 };
+        #endregion
+
+        static sbyte[][] nextState = new sbyte[][] {
+            new sbyte[] {0, 0, 0, 0, 0, 0},
+            new sbyte[] {-1, -1, 10, -1, -1, -1},
+            new sbyte[] {-1, -1, -1, -1, -1, -1},
+            new sbyte[] {-1, -1, 8, -1, -1, -1},
+            new sbyte[] {-1, -1, 5, -1, -1, -1},
+            new sbyte[] {-1, -1, 6, -1, -1, -1},
+            new sbyte[] {-1, -1, 7, -1, -1, -1},
+            null,
+            new sbyte[] {-1, -1, 9, -1, -1, -1},
+            null,
+            null,
+            new sbyte[] {-1, 1, 2, 3, 4, 2}
+        };
+
+
+        [SuppressMessage("Microsoft.Performance", "CA1810:InitializeReferenceTypeStaticFieldsInline")]
+        // Reason for suppression: cannot have self-reference in array initializer.
+        static Guesser()
+        {
+            nextState[7] = nextState[2];
+            nextState[9] = nextState[2];
+            nextState[10] = nextState[2];
+        }
+
+        int NextState()
+        {
+            if (code == ScanBuff.EndOfFile)
+                return eofNum;
+            else
+                return nextState[state][map[code]];
+        }
+        #endregion
+
+        public Guesser(System.IO.Stream file) { SetSource(file); }
+
+        public void SetSource(System.IO.Stream source)
+        {
+            this.buffer = new BuildBuffer(source);
+            code = buffer.Read();
+        }
+
+        int Scan()
+        {
+            for (; ; )
+            {
+                int next;
+                state = currentStart;
+                while ((next = NextState()) == goStart)
+                    code = buffer.Read();
+
+                state = next;
+                code = buffer.Read();
+
+                while ((next = NextState()) > eofNum)
+                {
+                    state = next;
+                    code = buffer.Read();
+                }
+                if (state <= maxAccept)
+                {
+                    #region ActionSwitch
+#pragma warning disable 162
+                    switch (state)
+                    {
+                        case eofNum:
+                            switch (currentStart)
+                            {
+                                case 11:
+                                    if (utfX == 0 && uppr == 0) return -1; /* raw ascii */
+                                    else if (uppr * 10 > utfX) return 0;   /* default code page */
+                                    else return 65001;                     /* UTF-8 encoding */
+                                    break;
+                            }
+                            return EndToken;
+                        case 1: // Recognized '{Upper128}',	Shortest string "\xC0"
+                        case 2: // Recognized '{Upper128}',	Shortest string "\x80"
+                        case 3: // Recognized '{Upper128}',	Shortest string "\xE0"
+                        case 4: // Recognized '{Upper128}',	Shortest string "\xF0"
+                            uppr++;
+                            break;
+                        case 5: // Recognized '{Utf8pfx4}{Utf8cont}',	Shortest string "\xF0\x80"
+                            uppr += 2;
+                            break;
+                        case 6: // Recognized '{Utf8pfx4}{Utf8cont}{2}',	Shortest string "\xF0\x80\x80"
+                            uppr += 3;
+                            break;
+                        case 7: // Recognized '{Utf8pfx4}{Utf8cont}{3}',	Shortest string "\xF0\x80\x80\x80"
+                            utfX += 3;
+                            break;
+                        case 8: // Recognized '{Utf8pfx3}{Utf8cont}',	Shortest string "\xE0\x80"
+                            uppr += 2;
+                            break;
+                        case 9: // Recognized '{Utf8pfx3}{Utf8cont}{2}',	Shortest string "\xE0\x80\x80"
+                            utfX += 2;
+                            break;
+                        case 10: // Recognized '{Utf8pfx2}{Utf8cont}',	Shortest string "\xC0\x80"
+                            utfX++;
+                            break;
+                        default:
+                            break;
+                    }
+#pragma warning restore 162
+                    #endregion
+                }
+            }
+        }
+    } // end class Guesser
+    
+#endif // !BYTEMODE
+#endregion
 #endif // !NOFILES

 // End of code copied from embedded resource
@@ -1,7 +1,7 @@
 <Project Sdk="Microsoft.NET.Sdk">

  <PropertyGroup>
-    <TargetFramework>net8.0</TargetFramework>
+    <TargetFramework>net7.0</TargetFramework>
    <ImplicitUsings>enable</ImplicitUsings>
    <Nullable>enable</Nullable>
  </PropertyGroup>
@@ -1,7 +1,7 @@
 <Project Sdk="Microsoft.NET.Sdk">

  <PropertyGroup>
-    <TargetFramework>net8.0</TargetFramework>
+    <TargetFramework>net7.0</TargetFramework>
    <ImplicitUsings>enable</ImplicitUsings>
    <Nullable>enable</Nullable>
  </PropertyGroup>
@@ -1,13 +0,0 @@
-<Project Sdk="Microsoft.NET.Sdk">
-
-  <PropertyGroup>
-    <TargetFramework>net8.0</TargetFramework>
-    <ImplicitUsings>enable</ImplicitUsings>
-    <Nullable>enable</Nullable>
-  </PropertyGroup>
-
-  <ItemGroup>
-    <PackageReference Include="Microsoft.Extensions.Hosting" Version="10.0.0" />
-  </ItemGroup>
-
-</Project>
@@ -1,71 +0,0 @@
-namespace Bts.Extensions.Hosting;
-
-using Microsoft.Extensions.Configuration;
-using Microsoft.Extensions.Hosting;
-using Microsoft.Extensions.Logging;
-
-using System.Diagnostics;
-
-public static class HostBuilderExtensions {
-   public static IHostBuilder CreateBtsBuilder(string[] args) {
-      // Obtain calling assembly name here, rather than in the lambda where it is actually needed,
-      // because want the name of the assembly that called this function, not the assembly that will eventually invoke
-      // the lambda...that will be the Microsoft.Extensions.Hosting assembly.
-      // Get the name of the assembly that called this function. That will act as the "appsettings"
-      // json file prefix
-      var exeName = GetCallingAssemblyName();
-      return Host.CreateDefaultBuilder(args)
-         .ConfigureAppConfiguration((context, configBuilder) =>
-            ConfigureApp(configBuilder, context.HostingEnvironment.EnvironmentName, exeName));
-   }
-
-   public static void ConfigureApp(IConfigurationBuilder configBuilder, string? environment = null, string? prefix = null) {
-      var baseFileName = prefix is null ? "appsettings" : $"{prefix}.appsettings";
-      AddJsonFile($"{baseFileName}.json");
-      if (environment is not null) {
-         AddJsonFile($"{baseFileName}.{environment}.json");
-      }
-
-      var cfg = configBuilder.Build().GetSection("config")?.Value;
-      if (!SetBasePath(cfg)) {
-         AddJsonFile(cfg);
-      }
-
-      bool SetBasePath(string? path) {
-         if (!Directory.Exists(path)) {
-            return false;
-         }
-         configBuilder.SetBasePath(path);
-         return true;
-      }
-
-      bool AddJsonFile(string? path) {
-         if (!File.Exists(path)) {
-            return false;
-         }
-         configBuilder.AddJsonFile(path);
-         return true;
-      }
-   }
-
-   public static void ConfigureLogging(ILoggingBuilder loggingBuilder, ILoggerProvider loggerProvider) =>
-      loggingBuilder.AddFilter("Microsoft", LogLevel.Warning)
-         .AddFilter("System", LogLevel.Warning)
-         .AddFilter("NToastNotify", LogLevel.Warning)
-         .AddProvider(loggerProvider)
-         .AddSimpleConsole(opts => {
-            opts.SingleLine = true;
-            opts.TimestampFormat = "yyyy-MM-dd HH:mm:ss.ffffff|";
-         })
-         .AddDebug();
-
-   public static string GetCallingAssemblyName() =>
-      new StackTrace()
-         .GetFrame(2)
-         ?.GetMethod()
-         ?.Module
-         ?.Assembly
-         ?.GetName()
-         ?.Name
-         ?? throw new InvalidOperationException("Failed to determine name of calling assembly (calling method is null)");
-}
@@ -1,9 +0,0 @@
-<Project Sdk="Microsoft.NET.Sdk">
-
-  <PropertyGroup>
-    <TargetFramework>net8.0</TargetFramework>
-    <ImplicitUsings>enable</ImplicitUsings>
-    <Nullable>enable</Nullable>
-  </PropertyGroup>
-
-</Project>
@@ -1,28 +0,0 @@
-namespace Bts.Net;
-
-using System.Linq;
-using System.Net;
-
-/// <summary>
-/// Provides facilities to compare hosts for equality.
-/// <para>
-/// This class doesn't implement IEqualityComparer because a consistent hash code cannot be
-/// guarenteed for equivalent host strings. The equals impl relies on Dns.GetHostAddresses, which
-/// is non-determinstic (implementations that depends on mutable properties face a similar problem).
-/// </para>
-/// </summary>
-public class HostComparer {
-   public static bool Equals(string? x, string? y) {
-      if (x is null) {
-         return y is null;
-      }
-      if (y is null) {
-         return false;
-      }
-      return Dns.GetHostAddresses(x).Intersect(
-         Dns.GetHostAddresses(y), IpAddressComparer.Instance).Any();
-   }
-
-   public static bool Equals(IPHostEntry x, IPHostEntry y) =>
-      x.AddressList.Intersect(y.AddressList, IpAddressComparer.Instance).Any();
-}
@@ -1,30 +0,0 @@
-namespace Bts.Net;
-
-using System.Diagnostics.CodeAnalysis;
-using System.Net;
-
-/// <summary>
-/// Compares ip addresses. Unlike the default comparer, v4 and equivalent v6 addresses are
-/// considered equal.
-/// </summary>
-public class IpAddressComparer : IEqualityComparer<IPAddress> {
-   public static IEqualityComparer<IPAddress> Instance { get; } = new IpAddressComparer();
-
-   public bool Equals(IPAddress? x, IPAddress? y) {
-      if (x is null) {
-         return y is null;
-      }
-      if (y is null) {
-         return false;
-      }
-      return x.Equals(y) ||
-         (x.IsIPv4MappedToIPv6 && x.MapToIPv4().Equals(y)) ||
-         (y.IsIPv4MappedToIPv6 && y.MapToIPv4().Equals(x));
-   }
-
-   public int GetHashCode([DisallowNull] IPAddress address) =>
-      address.IsIPv4MappedToIPv6
-         // Return consistent hash code for v4 and equivalent v6
-         ? address.MapToIPv4().GetHashCode()
-         : address.GetHashCode();
-}
@@ -1,33 +0,0 @@
-namespace Bts.Net;
-
-using System;
-using System.Net;
-using System.Text.Json;
-using System.Text.Json.Serialization;
-
-public class IpHostEntryJsonConverter : JsonConverter<IPHostEntry> {
-   public override IPHostEntry? Read(ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options) {
-      throw new NotImplementedException();
-   }
-
-   public override void Write(Utf8JsonWriter writer, IPHostEntry value, JsonSerializerOptions options) {
-      writer.WriteStartObject();
-      // HostName
-      writer.WriteString(nameof(value.HostName), value.HostName);
-      // Aliases
-      writer.WritePropertyName("Aliases");
-      writer.WriteStartArray();
-      foreach (var alias in value.Aliases) {
-         writer.WriteStringValue(alias);
-      }
-      writer.WriteEndArray();
-      // Addresses
-      writer.WritePropertyName("Addresses");
-      writer.WriteStartArray();
-      foreach (var address in value.AddressList) {
-         writer.WriteStringValue(address.ToString());
-      }
-      writer.WriteEndArray();
-      writer.WriteEndObject();
-   }
-}
@@ -1,39 +0,0 @@
-namespace Bts.Net;
-
-using System;
-using System.IO;
-using System.Net;
-using System.Text;
-using System.Text.Json;
-
-/// <summary>
-/// This class seriailzes IpHostEntry to json (string). A common buffer is used to reduce memory
-/// allocations on repeated calls to <see cref="Serialize"/>. For that reason, this class is NOT
-/// thread safe.
-/// </summary>
-public class IpHostEntryJsonSerializer {
-   readonly MemoryStream buffer = new();
-   readonly IpHostEntryJsonConverter converter;
-
-   public IpHostEntryJsonSerializer(IpHostEntryJsonConverter? converter = null) =>
-      this.converter = converter ?? new() { };
-
-   public string? Serialize(IPHostEntry dp, JsonSerializerOptions? opts = null) {
-      opts ??= JsonSerializerOptions.Default;
-      buffer.Position = 0;
-      buffer.SetLength(0);
-      using (var jsonWriter = new Utf8JsonWriter(buffer)) {
-         converter.Write(jsonWriter, dp, opts);
-      }
-      var bytes = buffer.GetBuffer();
-      var span = new ArraySegment<byte>(bytes, 0, (int)buffer.Length);
-      return Encoding.UTF8.GetString(span);
-   }
-
-   public void Serialize(IPHostEntry dp, Utf8JsonWriter jsonWriter, JsonSerializerOptions? opts = null) {
-      opts ??= JsonSerializerOptions.Default;
-      buffer.Position = 0;
-      buffer.SetLength(0);
-      converter.Write(jsonWriter, dp, opts);
-   }
-}
@@ -1,94 +0,0 @@
-namespace Bts.Net;
-
-using System.Diagnostics.CodeAnalysis;
-using System.Net;
-using System.Text.RegularExpressions;
-
-public static partial class IpUtils {
-   static readonly Regex ipv4Regex = new(string.Join('.', @"(((0|1)\d)|(2\d)|\d)?\d"));
-
-   // public static void ParseUri(string str, out string? scheme, out string? host, out string? port, out string? path, out string? queryParams) {
-   //    if (Uri.TryCreate(str, UriKind.Absolute, out var uri)) {
-   //    }
-   // }
-
-   // public static bool AreEndpointsEqual(Uri? x, Uri? y) {
-   //    if (x is null) {
-   //       return y is null;
-   //    }
-   //    if (y is null) {
-   //       return false;
-   //    }
-   //    if (x.IsAbsoluteUri != y.IsAbsoluteUri) {
-   //       return false;
-   //    }
-   //    if (x.IsAbsoluteUri) {
-   //       if (x.Scheme != y.Scheme) {
-   //          return false;
-   //       }
-   //    }
-   //    string? xHost = null;
-   // }
-
-   // static string ResolveHostIpv4(Uri uri) {
-   //    // switch (uri.HostNameType) {
-   //    //    case UriHostNameType.IPv4:
-   //    //       return uri.Host;
-   //    //    case UriHostNameType.Dns:
-   //    //       break;
-   //    //    case UriHostNameType.IPv6:
-   //    //       var addresses = Dns.GetHostAddresses(uri.Host);
-   //    //       foreach (var address in addresses) {
-   //    //          if (address.IsIPv4MappedToIPv6) {
-   //    //             return address.MapToIPv4().Address;
-   //    //          }
-   //    //       }
-   //    //       break;
-   //    // }
-   //    var hostAddresses = Dns.GetHostAddresses(uri.Host);
-   //    foreach (IPAddress address in hostAddresses) {
-   //       if (address.Equals(remoteEndPoint.Address) ||
-   //           (address.IsIPv4MappedToIPv6 && address.MapToIPv4().Equals(remoteEndPoint.Address)) ||
-   //           (remoteEndPoint.Address.IsIPv4MappedToIPv6 && remoteEndPoint.Address.MapToIPv4().Equals(address))) {
-   //          isMatch = true;
-   //          break;
-   //       }
-   //    }
-   // }
-
-   // public static bool AreEndpointsEqual(string? x, string? y) {
-   //    if (x == y) {
-   //       return true;
-   //    }
-   //    if (x is null) {
-   //       return y is null;
-   //    }
-   //    if (y is null) {
-   //       return false;
-   //    }
-   //    var xUri = new Uri(x);
-   //    var yUri = new Uri(y);
-   //    // return xUri.Equals(yUri);
-   // }
-
-   public static bool IsIpv4(string host) {
-      host = removeSchemaFromUrl(host);
-      host = removePortFromUrl(host);
-      return ipv4Regex.IsMatch(host);
-   }
-
-   static string removeSchemaFromUrl(string url) {
-      var i = url.IndexOf("://");
-      return i < 0 ? url : url.Substring(i + 3);
-   }
-
-   static string removePortFromUrl(string url) {
-      var match = MyRegex().Match(url);
-      return match.Success
-         ? string.Concat(url.AsSpan(0, match.Index), url.AsSpan(match.Index + match.Length))
-         : url;
-   }
-
-   [GeneratedRegex(":\\d{1,4}")]
-   private static partial Regex MyRegex();
-}
@@ -1,17 +1,20 @@
 <Project Sdk="Microsoft.NET.Sdk">

  <PropertyGroup>
-    <TargetFramework>net8.0</TargetFramework>
+    <TargetFramework>net7.0</TargetFramework>
    <ImplicitUsings>enable</ImplicitUsings>
    <Nullable>enable</Nullable>
  </PropertyGroup>

  <ItemGroup>
+    <PackageReference Include="Google.Protobuf" Version="3.24.3" />
    <PackageReference Include="YaccLexTools" Version="1.2.2" />
  </ItemGroup>

  <ItemGroup>
+    <ProjectReference Include="..\DataStreamCore\DataStreamCore.csproj" />
    <ProjectReference Include="..\ExpressionCore\ExpressionCore.csproj" />
+    <ProjectReference Include="..\..\client\spark_core\spark_core.csproj" />
  </ItemGroup>

  <ItemGroup Label="ColumnMapExpression">
@@ -1,11 +0,0 @@
-namespace bts.BlueOrb {
-   using BCG;
-
-   [DataStreamUserRespresentation("AddColumnsStream")]
-   public class AddColumnsStream: ViewStream {
-      public AddColumnsStream(ViewStream parent_str, string add_col_expr) {
-         ViewStream rs = new BCG.AddColumnsStream(parent_str, add_col_expr);
-         mySetRealSource(rs);
-      }
-   }
-}
@@ -1,118 +0,0 @@
-namespace Bts.DataStream;
-
-using BCG;
-
-using bts.collections;
-using bts.utils;
-
-using Bts.DataStreamCore;
-
-using Spark.Core.Collections;
-
-using System;
-using System.Collections;
-using System.Collections.Generic;
-using System.Linq;
-
-public class BlueAddIndexColumnTable : IBlueRowCollection {
-   readonly IBlueTableSchema schema;
-   readonly IList<object?>[] columnValues;
-   readonly int count;
-
-   BlueAddIndexColumnTable(IBlueTableSchema schema, IList<object?>[] columnValues, int? count = null) {
-      this.schema = schema;
-      this.columnValues = columnValues;
-      this.count = count ?? columnValues[0].Count;
-   }
-
-   IBlueTableSchema IBlueRowCollection.schema => schema;
-
-   public int Count => count;
-
-   public IBlueRow this[int id] =>
-      id > 0 && id <= count
-         ? GetRow(id - 1)
-         : throw new KeyNotFoundException();
-
-   public static BlueAddIndexColumnTable Create(string tableName, ColumnInfo[] columnInfos, IList<object?>[] columnValues) {
-      if (columnInfos.Any(c => c.colName == "id")) {
-         throw new ArgumentException("'id' column already exists", nameof(columnInfos));
-      }
-      if (columnInfos.Length != columnValues.Length) {
-         throw new ArgumentException("columnValues length differs from columnInfos length", nameof(columnValues));
-      }
-      int count = columnValues.FirstOrDefault()?.Count ?? 0;
-      var origCols = columnInfos;
-      var origColValues = columnValues;
-      columnInfos = new ColumnInfo[origCols.Length + 1];
-      columnInfos[0] = new ColumnInfo("id", 0, typeof(int));
-      columnValues = new IList<object?>[origCols.Length + 1];
-      columnValues[0] = new StepList<int>(1, 1, count).WrapSelector(id => (object?)id, null);
-      for (int i = 1; i < columnInfos.Length; i++) {
-         columnInfos[i] = new ColumnInfo(origCols[i - 1].colName, i, origCols[i - 1].colType);
-         columnValues[i] = origColValues[i - 1];
-      }
-      var schema = new BlueTableSchema(columnInfos, tableName);
-      return new(schema, columnValues, count);
-   }
-
-   public static BlueAddIndexColumnTable Create(IBlueTableSchema schema, IList<object?>[] columnValues) {
-      int count = columnValues.FirstOrDefault()?.Count ?? 0;
-      int? idColIdx = schema.TryResolveIndices("id");
-      var idColValues = new StepList<int>(1, 1, count).WrapSelector(id => (object?)id, null);
-      var columnInfos = schema.columnInfo;
-      if (idColIdx.HasValue) {
-         // Replace existing id column. columnValues may or may not contain the old ids (if its
-         // length is one less column than that of columnInfos, it doesn't contain them).
-         switch (columnValues.Length - schema.columnInfo.Length) {
-            case 0:
-               columnValues[idColIdx.Value] = idColValues;
-               break;
-            case -1:
-               columnValues = ArrayHelper.InsertExact(columnValues, idColIdx.Value, true, idColValues);
-               break;
-            default:
-               throw new ArgumentException("ColumnInfos length disagrees with columnValues length");
-         }
-      } else {
-         if (columnValues.Length != columnInfos.Length) {
-            throw new ArgumentException("ColumnInfos length disagrees with columnValues length");
-         }
-         var origColInfos = columnInfos;
-         columnInfos = new ColumnInfo[columnInfos.Length + 1];
-         columnInfos[0] = new("id", 0, typeof(int));
-         for (int i = 1; i < columnInfos.Length; i++) {
-            var c = origColInfos[i - 1];
-            columnInfos[i] = new(c.colName, c.colNo + 1, c.colType);
-         }
-         columnValues = ArrayHelper.InsertExact(columnValues, 0, true, idColValues);
-      }
-      return new(new BlueTableSchema(schema.tableName, columnInfos), columnValues);
-   }
-
-   public bool containsID(int id) =>
-      id > 0 && id <= count;
-
-   public IList<object?> GetColumn(int index) =>
-      columnValues[index];
-
-   public IBlueRow GetRow(int index) =>
-      new BlueRowColumnWrapper(schema, index, columnValues);
-
-   public int IndexOf(IBlueRow item) {
-      int id = item.Id();
-      return id > 0 && id <= count && BlueRowEqualityComparer.Equals(GetRow(id - 1), item)
-         ? id - 1
-         : -1;
-   }
-
-   public IEnumerator<IBlueRow> GetEnumerator() {
-      int count = Count;
-      for (int i = 0; i < count; i++) {
-         yield return GetRow(i);
-      }
-   }
-
-   IEnumerator IEnumerable.GetEnumerator() =>
-      GetEnumerator();
-}
@@ -1,56 +0,0 @@
-namespace Bts.DataStream;
-
-using BCG;
-
-using Bts.DataStreamCore;
-
-using mbts.utils;
-
-using System.Collections;
-using System.Collections.Generic;
-using System.Linq;
-
-public class BlueIndexedColumnTable : IBlueRowCollection {
-   readonly IBlueTableSchema schema;
-   readonly IList<object?>[] columnValues;
-   readonly Dictionary<int, int> rowIdToIndex;
-
-   public BlueIndexedColumnTable(IBlueTableSchema schema, IList<object?>[] columnValues) {
-      this.schema = schema;
-      this.columnValues = columnValues;
-      int idColumnIndex = schema.ResolveIndices("id");
-      rowIdToIndex = columnValues[idColumnIndex]
-         .Select((idObj, idx) => new KeyValuePair<int, int>(ExceptionAssert.Argument.Is<int>(idObj), idx))
-         .ToDictionary();
-   }
-
-   public IBlueRow this[int id] =>
-      rowIdToIndex.TryGetValue(id, out var idx)
-         ? GetRow(idx)
-         : throw new KeyNotFoundException();
-
-   IBlueTableSchema IBlueRowCollection.schema => schema;
-
-   public int Count => rowIdToIndex.Count;
-
-   public bool containsID(int id) =>
-      rowIdToIndex.ContainsKey(id);
-
-   public IEnumerator<IBlueRow> GetEnumerator() {
-      int count = Count;
-      for (int i = 0; i < count; i++) {
-         yield return GetRow(i);
-      }
-   }
-
-   public IBlueRow GetRow(int index) =>
-      new BlueRowColumnWrapper(schema, index, columnValues);
-
-   public int IndexOf(IBlueRow item) =>
-      rowIdToIndex.TryGetValue(item.Id(), out var idx) && BlueRowEqualityComparer.Equals(GetRow(idx), item)
-         ? idx
-         : -1;
-
-   IEnumerator IEnumerable.GetEnumerator() =>
-      GetEnumerator();
-}
@@ -1,21 +0,0 @@
-namespace Bts.DataStream;
-
-using BCG;
-
-using System.Collections.Generic;
-
-public class BlueRowColumnWrapper : IBlueRow {
-   readonly int index;
-   readonly IList<object?>[] columns;
-
-   public BlueRowColumnWrapper(IBlueTableSchema schema, int index, IList<object?>[] columns) {
-      this.schema = schema;
-      this.index = index;
-      this.columns = columns;
-   }
-
-   public object? this[int index] =>
-      columns[index][this.index];
-
-   public IBlueTableSchema schema { get; }
-}
@@ -1,18 +0,0 @@
-namespace bts.BlueOrb {
-   using System;
-
-   using BCG;
-
-   [DataStreamUserRespresentation("CSVReaderStream")]
-   public class CSVReaderStream : ViewStream {
-      public CSVReaderStream(string file_name)
-         : this(file_name, false) { }
-      public CSVReaderStream(string file_name, bool guess_schema) {
-         ViewStream rs = new BCG.CSVReaderStream(file_name);
-         if (guess_schema) {
-            rs = new SchemaGuesserStream(rs);
-         }
-         mySetRealSource(rs);
-      }
-   }
-}
@@ -1,13 +0,0 @@
-namespace bts.BlueOrb {
-   using BCG;
-
-   using System;
-
-   [DataStreamUserRespresentation("CSVSaverStream")]
-   public class CSVSaverStream: ViewStream {
-      public CSVSaverStream(ViewStream parent_str, string file_name) {
-         BCG.CSVSaverStream rs = new BCG.CSVSaverStream(parent_str, file_name, true, Int32.MaxValue, false);
-         mySetRealSource(rs);
-      }
-   }
-}
@@ -1,46 +0,0 @@
-using System;
-
-namespace BCG {
-   [DataStreamUserRespresentation("CountStream")]
-   public class CountStream : ViewStream {
-      private DataPacket myLastFiredDataPacket;
-      private IBlueTableSchema mySchema = new BlueTableSchema("count",
-         new ColumnInfo("id", 0, typeof(int)),
-         new ColumnInfo("count", 1, typeof(int)));
-      private int myId = 0;
-      private ViewStream myVS;
-
-      public CountStream(ViewStream vs) {
-         myVS = vs;
-      }
-
-      protected override void myOnFirstSubscriber() {
-         myVS.onChange += new ChangeDelegate(myHandleChanges);
-      }
-      protected override void myOnLastSubscriber() {
-         myVS.onChange -= new ChangeDelegate(myHandleChanges);
-      }
-
-
-      private void myHandleChanges(object newValue) {
-         myFireEvent(
-            myLastFiredDataPacket =
-               myLastFiredDataPacket == null ?
-                  new DataPacket(myGetRows(newValue, ++myId), myGetRows(newValue, myId),
-                  new BlueTable(mySchema))
-                  :
-                  new DataPacket(myGetRows(newValue, ++myId), myGetRows(newValue, myId),
-                  myLastFiredDataPacket.newRows)
-            );
-      }
-
-      private IBlueRowCollection myGetRows(object dp, int id) {
-         BlueTable retVal = new BlueTable(mySchema);
-         retVal.addRow
-            (new BlueRow(mySchema, id, (dp as DataPacket).newRows.Count));
-         return retVal;
-      }
-
-
-   }
-}
@@ -1,156 +0,0 @@
-namespace bts.BlueOrb {
-   using BCG;
-   using System;
-   using System.Text.RegularExpressions;
-
-   [DataStreamUserRespresentation("FilterStream")]
-   public class FilterStream : ViewStream {
-      public class RegExpFilter : IStreamFilter {
-         private Regex[] myRegExps;
-         private string[] myCols;
-
-         public RegExpFilter(string col, string regExp)
-            : this(new string[] { col }, new string[] { regExp }) { }
-
-         public RegExpFilter(string[] cols, string[] regExps) {
-            myCols = cols;
-            myRegExps = new Regex[regExps.Length];
-            for (int i = 0; i < regExps.Length; i++)
-               myRegExps[i] = new Regex(regExps[i]);
-         }
-
-         #region IStreamFilter Members
-         public bool filter(IBlueRow dr) {
-            bool rv = true;
-            for (int i = 0; i < myCols.Length; i++) {
-               rv &= myRegExps[i].IsMatch(dr[myCols[i]] is DBNull ? "" : dr[myCols[i]].ToString());
-               if (!rv)
-                  return rv;
-            }
-            return rv;
-         }
-         #endregion
-      }
-      private BlueTable myLastFiredRows;
-      private IBlueRowCollection myLastReceivedRows;
-      private IStreamFilter myFilter;
-      private ViewStream myParentStream;
-      private string myFilterString;
-      private bool myUseNewFilter = false;// Configuration.get<DataStreamConfig>().UseNewFilter;
-
-      #region constructors
-      public delegate bool OnFilter(IBlueRow r);
-      private class OnFilterWrapper : IStreamFilter {
-         public OnFilterWrapper(OnFilter f) { this.f = f; }
-
-         public bool filter(IBlueRow dr) { return f(dr); }
-
-         private OnFilter f;
-      }
-      public FilterStream(ViewStream vs, OnFilter f)
-         : this(vs, new OnFilterWrapper(f)) { }
-
-
-      public FilterStream(ViewStream vs, IStreamFilter filter) {
-         {
-            myParentStream = vs;
-            myFilter = filter;
-            myFilterString = filter.ToString();
-            if (filter is IDynamicStreamFilter)
-               (filter as IDynamicStreamFilter).onFilterChange += new FilterChangeDelegate(myHandleFilterChange);
-         }
-      }
-
-      public FilterStream(ViewStream vs, string filterExpression)
-         : this(vs, new BooleanExpressionFilter(filterExpression)) {
-         myFilterString = filterExpression;
-      }
-
-      public FilterStream(ViewStream vs, string col, string regExp)
-         : this(vs, new RegExpFilter(col, regExp)) {
-         myFilterString = col + " , " + regExp;
-      }
-
-      public FilterStream(ViewStream vs)
-         : this(vs, new BinaryFilter(true)) {
-         myFilterString = true.ToString();
-      }
-      #endregion
-
-      #region overridden methods
-      protected override void myOnFirstSubscriber() { myParentStream.onChange += new ChangeDelegate(myHandleCallbacks); }
-
-      protected override void myOnLastSubscriber() {
-         myLastFiredRows = null;
-         myParentStream.onChange -= new ChangeDelegate(myHandleCallbacks);
-      }
-
-      protected override void myFireEvent(object o) {
-         DataPacket dp = o as DataPacket;
-         if (myLastValue == null || dp.addedRows.Count + dp.deletedRows.Count > 0)
-            base.myFireEvent(o);
-      }
-      #endregion
-
-      private void myHandleCallbacks(object newValue) {
-         DataPacket retVal;
-         DataPacket arrivedPacket = newValue as DataPacket;
-         myLastReceivedRows = arrivedPacket.newRows;
-         if (myLastFiredRows != null) {
-            retVal = new DataPacket(myLastFiredRows, new BlueTable(myLastFiredRows.schema),
-               new BlueTable(myLastFiredRows.schema));
-            foreach (IBlueRow r in arrivedPacket.deletedRows)
-               if (myLastFiredRows.containsID((int)r["id"])) {
-                  myLastFiredRows.deleteRow(r);
-                  (retVal.deletedRows as BlueTable).addRow(r);
-               }
-            foreach (IBlueRow r in arrivedPacket.addedRows)
-               if (myFilter.filter(r)) {
-                  myLastFiredRows.addRow(r);
-                  (retVal.addedRows as BlueTable).addRow(r);
-               }
-         } else {
-            myLastFiredRows = new BlueTable(arrivedPacket.schema);
-            retVal = new DataPacket(myLastFiredRows, myLastFiredRows, new BlueTable(myLastFiredRows.schema));
-            foreach (IBlueRow r in arrivedPacket.newRows)
-               if (myFilter.filter(r))
-                  myLastFiredRows.addRow(r);
-         }
-         myFireEvent(retVal);
-      }
-
-      public IStreamFilter filter {
-         set {
-            {
-               myFilter = value;
-               if (numberOfSubscribers > 0) {
-                  if (myLastReceivedRows != null) {
-                     BlueTable nRows = new BlueTable(myLastReceivedRows.schema);
-                     foreach (IBlueRow r in myLastReceivedRows)
-                        if (myFilter.filter(r))
-                           nRows.addRow(r);
-                     IBlueRowCollection dRows = new BlueTable(nRows.schema),
-                        aRows = new BlueTable(nRows.schema);
-                     BlueDataUtility.computeDifferences(nRows, myLastFiredRows, out dRows, out aRows);
-                     myLastFiredRows = nRows;
-                     myFireEvent(new DataPacket(nRows, aRows, dRows));
-                  }
-               }
-            }
-         }
-         get {            
-               return myFilter;
-         }
-      }
-
-      //public string filterString { set { filter = new BooleanExpressionFilter(value); } }
-
-      public override string ToString() {
-         return GetType().ToString() + " " + myFilterString;
-      }
-
-      private void myHandleFilterChange() {
-         filter = myFilter;
-      }
-   }
-}
@@ -1,127 +0,0 @@
-using System;
-using System.Collections.Generic;
-using System.Diagnostics;
-using System.Linq;
-using System.Text.Json;
-
-using bts;
-//using bts.SilverPine;
-using BCG;
-
-namespace bts.BlueOrb {
-   [DataStreamUserRespresentation("GroupByStream")]
-   public class GroupByStream : ViewStream {
-
-      public interface INonIncGroupByEvaluator {
-         // grouped_rows
-         IBlueRowCollection evaluate(IDictionary<int, List<IBlueRow>> grouped_rows);
-         string[] get_group_by_cols();
-      }
-
-      private class DelNonIncGroupByEvaluator : INonIncGroupByEvaluator {
-         Func<string[]> _groupby_cols_func;
-         Func<List<IBlueRow>, List<IBlueRow>> _evaluate_func;
-         public DelNonIncGroupByEvaluator(Func<string[]> groupby_cols_func,
-                                          Func<List<IBlueRow>, List<IBlueRow>> evaluate_func) {
-            _groupby_cols_func = groupby_cols_func;
-            _evaluate_func = evaluate_func;
-         }
-         public IBlueRowCollection evaluate(IDictionary<int, List<IBlueRow>> grouped_rows) {
-            BlueTable rv = null;
-            foreach (var kv in grouped_rows) {
-               List<IBlueRow> g_rows = kv.Value;
-               List<IBlueRow> out_rows = _evaluate_func.Invoke(g_rows);
-               if (out_rows != null && out_rows.Count > 0) {
-                  rv = rv ?? new BlueTable(out_rows.ElementAt(0).schema);
-               }
-               if (rv != null)
-                  rv.addRows(out_rows);
-            }
-            return rv;
-         }
-         public string[] get_group_by_cols() {
-            return _groupby_cols_func();
-         }
-
-      }
-      private ViewStream _main_stream;
-      private INonIncGroupByEvaluator _groupby_func;
-      private BlueRowHasher _row_hasher;
-      private string[] _groupby_cols;
-      private Stopwatch _timer = new();
-      private int _n_total_fires = 0, _n_expensive_fires = 0;
-
-      public GroupByStream(ViewStream main_stream, INonIncGroupByEvaluator groupby_func) {
-         _main_stream = main_stream;
-         _groupby_func = groupby_func;
-         _groupby_cols = _groupby_func.get_group_by_cols();
-         _row_hasher = new BlueRowHasher(_groupby_cols == null ? 0 : _groupby_cols.Length);
-      }
-
-      public GroupByStream(ViewStream main_stream, Func<string[]> groupby_cols_func,
-                                 Func<List<IBlueRow>, List<IBlueRow>> evaluate_func)
-                                 : this(main_stream, new DelNonIncGroupByEvaluator(groupby_cols_func, evaluate_func)) { }
-
-      public GroupByStream(ViewStream parent_str, string groupby_cols, string standard_group_by_text)
-      : this(parent_str, groupby_cols, standard_group_by_text, String.Empty) { }
-
-      // this is the main constructor that does everything
-      public GroupByStream(ViewStream parent_str, string groupby_cols, string standard_group_by_ops, string optional_sort_col) {
-         MultiRowOpHelper std_group_by = MultiRowOpHelper.parse_ops(standard_group_by_ops);
-         Func<string[]> groupby_cols_func = () => {
-            if (String.IsNullOrEmpty(groupby_cols))
-               return null;
-            return groupby_cols.Split(',').Select(x => x.Trim()).ToArray();
-         };
-         Func<List<IBlueRow>, List<IBlueRow>> evaluate_func = (group_rows) => {
-            if (!String.IsNullOrEmpty(optional_sort_col))
-               group_rows.Sort((r1, r2) => BlueDataUtility.compare_rows(r2, r1, optional_sort_col));
-            // this si where the group by aggregation happens
-            IBlueRow rv = MultiRowOpHelper.group_by_compute(group_rows, std_group_by);
-            if (rv == null)
-               return new List<IBlueRow>();
-            return new IBlueRow[] { rv }.ToList();
-         };
-         ViewStream rs = new GroupByStream(parent_str, groupby_cols_func, evaluate_func);
-         mySetRealSource(rs);
-      }
-      public GroupByStream(ViewStream parent_str, string standard_group_by_text) :
-      this(parent_str, String.Empty, standard_group_by_text) {
-      }
-
-
-      protected override void myOnFirstSubscriber() { _main_stream.onChange += _main_stream_onchange; }
-
-      protected override void myOnLastSubscriber() { _main_stream.onChange -= _main_stream_onchange; }
-
-      private void _main_stream_onchange(object nv) {
-         DataPacket dp = nv as DataPacket;
-         ///grouping together
-         _timer.Restart();
-         IDictionary<int, List<IBlueRow>> grouped_rows = (_groupby_cols == null || _groupby_cols.Length == 0) ?
-         new Dictionary<int, List<IBlueRow>>() { { 0, dp.newRows.to_list() } } :
-         BlueDataUtility.group_together(dp.newRows, _groupby_cols, ref _row_hasher);
-         _timer.Stop();
-         var groupElapsedMs = _timer.ElapsedMilliseconds;
-         if (groupElapsedMs >= 50) {
-            Log.DataFlow.info($"Time to group {dp.newRows.Count} rows into {grouped_rows.Count} groups: {_timer.ElapsedMilliseconds}ms");
-         }
-         // Log.log(bts.Log.Level.Info, bts.log.Subsystem.gen, String.Format("Time to group {0} rows into {1} groups = {2}", dp.newRows.Count, grouped_rows.Count, _timer.lastIntervalTime));
-         /// reducing
-         _timer.Restart();
-         IBlueRowCollection total_state = _groupby_func.evaluate(grouped_rows);
-         if (total_state == null)
-            // mayeb returna n empty table ?
-            return;
-         _timer.Stop();
-         var applyElapsedMs = _timer.ElapsedMilliseconds;
-         _n_total_fires++;
-         if (applyElapsedMs >= 50) {
-            _n_expensive_fires++;
-            Log.DataFlow.info(
-               $"Time to evaluate {grouped_rows.Count} groups {_timer.ElapsedMilliseconds}ms, ratio {_n_expensive_fires}/{_n_total_fires}");
-         }
-         myFireEvent(new DataPacket(total_state, total_state, myLastValue == null ? new BlueTable(total_state.schema) : (myLastValue as DataPacket).newRows));
-      }
-   }
-}
@@ -1,196 +0,0 @@
-namespace bts.BlueOrb {
-   using BCG;
-   using RowHash = System.Int32;
-   using System;
-   using System.Collections.Generic;
-   using System.Linq;
-   using System.Text;
-   using System.Threading.Tasks;
-   using System.Collections;
-   using System.Data.Entity.Core.Mapping;
-
-   [DataStreamUserRespresentation("JoinStream")]
-   public class SimpleJoinStream : ViewStream {
-      public class JoinBookkeepingData {
-         // initialized in cons
-         private readonly List<string> _join_col_names;
-         private readonly bool _is_left;
-         private readonly BlueRowHasher _hasher;
-         // initialized on first udpate         
-         private IBlueTableSchema _schema;
-         private int[] _join_cols;
-         private IDictionary<RowHash, List<IBlueRow>> _hash_row_map = new Dictionary<RowHash, List<IBlueRow>>();
-
-         public JoinBookkeepingData(ViewStream vs, List<string> join_cols, bool is_left, BlueRowHasher row_hasher) {
-            _join_col_names = join_cols.Where(c => !String.IsNullOrEmpty(c)).ToList();
-            _is_left = is_left;
-            _hasher = row_hasher;
-         }
-
-         public IBlueTableSchema schema => _schema;
-
-         public IDictionary<RowHash, List<IBlueRow>> rows => _hash_row_map;
-
-         public List<IBlueRow> get_matching_rows(int hash) {
-            List<IBlueRow> rv = null;
-            _hash_row_map.TryGetValue(hash, out rv);
-            return rv;
-         }
-
-         private RowHash _get_hash(IBlueRow r) {
-            if (_join_cols == null || _join_cols.Length == 0) {
-               return 0;
-            }
-            return _hasher.get_hash(r, _join_cols);
-         }
-
-         public void update_data(DataPacket dp) {
-            if (!ready) {
-               _schema = dp.newRows.schema;
-               _join_cols = _join_col_names.Count == 0 ? new int[] { } : _join_col_names.Select(n => dp.schema[n].colNo).ToArray();
-            }
-            _hash_row_map.Clear();
-            foreach (IBlueRow r in dp.newRows) {
-               int hash = _get_hash(r);
-               List<IBlueRow> hash_rows = null;
-               if (!_hash_row_map.TryGetValue(hash, out hash_rows)) {
-                  hash_rows = new List<IBlueRow>();
-                  _hash_row_map.Add(hash, hash_rows);
-               }
-               hash_rows.Add(r);
-            }
-         }
-
-         public bool ready {
-            get {
-               return _schema != null;
-            }
-         }
-
-      }
-
-      private ViewStream _ls, _rs;
-      private JoinBookkeepingData _ldata, _rdata;
-      private ViewStreamDatabase _vsd;
-      private IBlueTableSchema _schema = null;
-
-      public SimpleJoinStream(ViewStream ls, string ljc, ViewStream rs, string rjc) :
-         this(ViewStreamDatabase.instance, ls, ljc, rs, rjc) { }
-
-      public SimpleJoinStream(ViewStreamDatabase vsd, ViewStream ls, string ljc, ViewStream rs, string rjc) {
-         _ls = ls;
-         _rs = rs;
-         List<string> lj_col_list = ljc.Split(',').Where(x => !String.IsNullOrEmpty(x)).ToList();
-         List<string> rj_col_list = rjc.Split(',').Where(x => !String.IsNullOrEmpty(x)).ToList();
-         Log.assert(lj_col_list.Count == rj_col_list.Count);
-         int n_cols = lj_col_list.Count;
-         BlueRowHasher hasher = new BlueRowHasher(n_cols);
-         _ldata = new JoinBookkeepingData(ls, lj_col_list, true, hasher);
-         _rdata = new JoinBookkeepingData(rs, rj_col_list, false, hasher);
-      }
-
-      protected override void myOnFirstSubscriber() {
-         _ls.onChange += _ls_onchange;
-         _rs.onChange += _rs_onchange;
-      }
-
-      private void _ls_onchange(object dp_obj) {
-         DataPacket dp = dp_obj as DataPacket;
-         _ldata.update_data(dp);
-         _form_and_fire();
-      }
-
-      private void _rs_onchange(object dp_obj) {
-         DataPacket dp = dp_obj as DataPacket;
-         _rdata.update_data(dp);
-         _form_and_fire();
-      }
-
-      protected override void myOnLastSubscriber() {
-         _ls.onChange -= _ls_onchange;
-         _rs.onChange -= _rs_onchange;
-      }
-
-      private void _build_schema() {
-         if (_schema != null)
-            return;
-         List<ColumnInfo> cis = new List<ColumnInfo>();
-         cis.Add(new ColumnInfo("id", 0, TypeCode.Int32));
-         foreach (ColumnInfo ci in _ldata.schema.columnInfo) {
-            string col_name = _rdata.schema.contains_column(ci.colName) ? $"left_{ci.colName}" : ci.colName;
-            cis.Add(new ColumnInfo(col_name, cis.Count, ci.colType));
-         }
-         foreach (ColumnInfo ci in _rdata.schema.columnInfo) {
-            string col_name = _ldata.schema.contains_column(ci.colName) ? $"right_{ci.colName}" : ci.colName;
-            cis.Add(new ColumnInfo(col_name, cis.Count, ci.colType));
-         }
-         _schema = new BlueTableSchema($"{_ldata.schema.tableName}_join_{_rdata.schema.tableName}", cis.ToArray());
-      }
-
-      private void _form_and_fire() {
-         if (!_ldata.ready || !_rdata.ready)
-            return;
-         _build_schema();
-         List<object[]> res_rows = new List<object[]>();
-
-         Action<object[], IBlueRow, bool> copy_row = (rval, r, is_left) => {
-            int offset = 1;
-            JoinBookkeepingData sdata = _ldata;
-            if (is_left) {
-               offset = 1;
-               sdata = _ldata;
-            } else {
-               sdata = _rdata;
-               offset = 1 + _ldata.schema.columnInfo.Length;
-            }
-            for (int i = 0; i < sdata.schema.columnInfo.Length; i++) {
-               rval[offset + i] = r[i];
-            }
-         };
-         Func<int, IBlueRow, object[]> build_row_array = (id, l_row) => {
-            object[] rv = new object[_schema.columnNames.Length];
-            for (int i = 0; i < _schema.columnNames.Length; i++) {
-               rv[i] = DBNull.Value;
-            }
-            rv[0] = id;
-            copy_row(rv, l_row, true);
-            return rv;
-         };
-         // this is where the join happens
-         foreach (var kv in _ldata.rows) {
-            RowHash hash = kv.Key;
-            List<IBlueRow> l_rows = kv.Value;
-            List<IBlueRow> r_rows = _rdata.get_matching_rows(hash);
-
-            foreach (IBlueRow lr in l_rows) {
-               if (r_rows != null) {
-                  foreach (IBlueRow r in r_rows) {
-                     object[] rowvals = build_row_array(res_rows.Count, lr);
-                     copy_row(rowvals, r, false);
-                     res_rows.Add(rowvals);
-                  }
-               } else {
-                  object[] rowvals = build_row_array(res_rows.Count, lr);
-                  res_rows.Add(rowvals);
-               }
-            }
-         }
-         BlueTable rv = new BlueTable(_schema);
-         foreach (object[] rval in res_rows) {
-            rv.addRow(new BlueRow(_schema, rval));
-         }
-         var dp = BlueDataUtility.build_dp(rv, myLastValue as DataPacket);
-         myFireEvent(dp);
-      }
-   }
-
-   [DataStreamUserRespresentation("FullJoinStream")]
-   public class FullJoinStream : ViewStream {
-      public FullJoinStream(ViewStream l_str, string ljcs, ViewStream r_str, string rjcs) {
-         ViewStream js = new SimpleJoinStream(l_str, ljcs, r_str, rjcs);
-         ViewStream rs = new bts.BlueOrb.FilterStream(js, r => !DBNull.Value.Equals(r["right_id"]));
-         mySetRealSource(rs);
-      }
-   }
-
-}
@@ -1,28 +0,0 @@
-//namespace bts.Blueorb {
-//   using BCG;
-
-//   public class JsonSaverStream : ViewStream {
-//      private ViewStream _stream;
-//      private string _file_name;
-//      public JsonSaverStream(ViewStream vs, string file_name) {
-//         _stream = vs;
-//         _file_name = file_name;
-//      }
-
-//      protected override void myOnFirstSubscriber() {
-//         _stream.onChange += _on_data;
-//      }
-
-//      private void _on_data(object dp_obj) {
-//         if (dp_obj == null) {
-//            return;
-//         }
-//         DataPacket dp = dp_obj as DataPacket;
-//         dp.newRows.write_to_json_file(dp.newRows, _file_name);
-//      }
-
-//      protected override void myOnLastSubscriber() {
-//         _stream.onChange -= _on_data;
-//      }
-//   }
-//}
@@ -1,91 +0,0 @@
-using System;
-using System.Collections;
-using System.Collections.Generic;
-
-using Amazon.Runtime.Internal.Util;
-
-using BCG;
-using BCG.Utilities;
-
-using bts;
-
-namespace bts.BlueOrb{
-   public class LiteralStream : ViewStream {
-
-      public static List<Tuple<ColumnInfo, object>> parse_columns(string key_value_pairs) {
-         List<Tuple<ColumnInfo, object>> rv = new List<Tuple<ColumnInfo, object>>();
-         string[] pairs = key_value_pairs.Split(' ');
-         foreach (string p in pairs) {
-            string[] kvp = p.Split(':');
-            string col_name = kvp[0];
-            string val = kvp[1];
-            var tup = BlueDataUtility.guess_type(val);
-            var ci = new ColumnInfo(col_name, rv.Count, tup.Item1);
-            rv.Add(Tuple.Create(ci, tup.Item2));
-         }
-         return rv;
-      }
-
-      //literal("table1", "x:123, y:1.001, z:abc;x:123, y:1.001, z:abc")
-      // col_name: col_value ... ; col_name: col_value
-      public LiteralStream(string table_name, string rows) {
-         Func<string, int, IBlueRow> get_row = (json_row, id) => {
-            Dictionary<string, object> map = new Dictionary<string, object>();
-            string[] toks = json_row.Split(',', StringSplitOptions.TrimEntries);
-            //Logger.assert(col_vals.Length == 2);
-            foreach (string t in toks) {
-               var col_vals = t.Split(':', StringSplitOptions.TrimEntries);
-               string col = col_vals[0];
-               string val = col_vals[1];
-               map.Add(col, val);
-            }
-            map["id"] = id;
-            return new BlueRow(map);
-         };
-         string[] row_arr = rows.Split(';', StringSplitOptions.TrimEntries);
-         BlueTable table = null;
-         foreach (string json_string in row_arr) {
-            IBlueRow r = //BlueDataUtility.convert_json_to_row(json_string, table == null ? 0 : table.Count);
-            get_row(json_string, table == null ? 0 : table.Count);
-            table = table ?? new BlueTable(r.schema);
-            table.addRow(r);
-         }
-         ViewStream str = new BlueTableStream(table);
-         str = new SchemaGuesserStream(str);
-         mySetRealSource(str);
-      }
-
-      public LiteralStream(string tableName, string schema, string value) {
-         string[] columns = schema.Split(' ');
-
-
-         ColumnInfo[] ci = new ColumnInfo[columns.Length + 1];
-         ci[0] = new ColumnInfo("id", 0, typeof(int));
-         int n = 1;
-         foreach (string c in columns) {
-            string[] columnAndType = c.Split('/');
-            if (columnAndType.Length == 1)
-               ci[n] = new ColumnInfo(c, n, typeof(double));
-            else
-               ci[n] = new ColumnInfo(columnAndType[0], n, Type.GetType("System." + columnAndType[1], true));
-            n++;
-         }
-
-
-         BlueTable t = new BlueTable(new BlueTableSchema(tableName, ci));
-         string[] rows = value.Split(';');
-         n = 0;
-         foreach (string r in rows)
-            try {
-               if (r == string.Empty)
-                  continue;
-               t.addRow(
-                  BlueDataUtility.createRowFromCSVFileRow(r, t.schema, ++n));
-            } catch (Exception e) {
-               throw (new Exception("Error occured at line: " + n.ToString() + " " + e.Message));
-            }
-
-         mySetRealSource(new BlueRowCollectionStaticStream(t));
-      }
-   }
-}
@@ -1,89 +0,0 @@
-using System;
-
-using BCG.Utilities.General;
-using BCG;
-using bts;
-
-namespace bts.BlueOrb {
-   [DataStreamUserRespresentation("MillisecondStream")]
-   public class MillisecondStream : ViewStream, IWorker {
-      private DateTime _last_fire_time = DateTime.MinValue;
-      private readonly TimeSpan _period;
-      private readonly int _size;
-      private readonly ViewStreamDatabase _vsd;
-      private IBlueTableSchema _schema;
-      private bts.CircularBufferTable _table;
-
-      public MillisecondStream(int interval_msecs, int size)
-      : this(ViewStreamDatabase.instance, interval_msecs, size) {
-      }
-
-      public MillisecondStream(ViewStreamDatabase vsd, int interval_msecs, int size) {
-         _vsd = vsd;
-         _period = TimeSpan.FromMilliseconds(interval_msecs);
-         _size = size;
-         _schema = BlueDataUtility.form_schema("Milliseconds",
-               Tuple.Create("id", TypeCode.Int32),
-               Tuple.Create("time", TypeCode.DateTime),
-               Tuple.Create("display_time", TypeCode.String));
-         _table = new CircularBufferTable(_schema, _size);
-      }
-
-      protected override void myOnFirstSubscriber() {
-         _vsd.addWorker(this);
-      }
-
-      protected override void myOnLastSubscriber() {
-         _vsd.removeWorker(this);
-      }
-
-      private int _id = 0;
-      public void doWork() {
-         if (DateTime.Now - _last_fire_time >= _period) {
-            _last_fire_time = DateTime.Now;
-            IBlueRow nr = new BlueRow(_schema, _id++, _last_fire_time,
-            _last_fire_time.ToString("yyyy-MM-dd HH:mm:ss.fff"));
-            _table = _table.Add(nr);
-            var dp = BlueDataUtility.build_dp_inc(_table, myLastValue as DataPacket);
-            myFireEvent(dp);
-         }
-      }
-   }
-   internal class MSecondStream : TimeUnitStream {
-      public MSecondStream(int intervalInms, int size)
-         : base(DateTime.Now, new TimeSpan(0, 0, 0, 0, intervalInms), size, ViewStreamDatabase.instance) { }
-
-      protected override DateTime myRoundTime(DateTime dt) { return dt; }
-   }
-
-   internal class DisplayableMillisecondStream : ViewStream, IStreamMapper {
-      private IBlueTableSchema mySchema = new BlueTableSchema("ms",
-                                                               new ColumnInfo("id", 0, typeof(int)),
-                                                               new ColumnInfo("time", 1, typeof(string)));
-      public DisplayableMillisecondStream(int intervalInMs, int sz) {
-         mySetRealSource(new MapStream(new MSecondStream(intervalInMs, sz), this));
-      }
-      #region IStreamMapper Members
-
-      public IBlueTableSchema getOutputSchema(IBlueTableSchema inputSchema) {
-         return mySchema;
-      }
-
-      #endregion
-
-      #region IDynamicStreamMapper Members
-
-      public IBlueRow map(IBlueRow row) {
-         DateTime time = row.get_datetime("time");
-         var time_txt = time.ToString("yyyy-MM-dd HH:mm:ss.fff");
-         return new BlueRow(mySchema, row["id"], time_txt);
-      }
-
-      #endregion
-
-      private string myToString(DateTime dt) {
-         string sep = ":";
-         return dt.Hour + sep + dt.Minute + sep + dt.Second + sep + dt.Millisecond;
-      }
-   }
-}
@@ -1,169 +0,0 @@
-namespace bts.BlueOrb {
-   using BCG;
-
-   using System;
-   using System.Collections.Generic;
-   using System.Linq;
-   //using MathNet.Numerics.Statistics;
-
-
-   public enum MultiRowOp {
-      Mean,
-      Median,
-      Mode,
-      Range,
-      StdDev,
-      Max,
-      Min,
-      First,
-      Last,
-      Sum,
-      EdgeDiff,
-      EdgeDiffPercentage,
-   }
-   public class MultiRowOpHelper {
-      // input + op
-      public readonly List<Tuple<string, MultiRowOp>> operations;
-
-      public static MultiRowOpHelper parse_ops(string text) {
-         // text is inp_col: op, inp_col: op, inp_col:op
-         // ex: "open:First, close:Last, high:Max, low:Min, volume:Sum")
-         List<Tuple<string, MultiRowOp>> rv = new List<Tuple<string, MultiRowOp>>();
-         string[] calcs = text.Split(',');
-         foreach (string c in calcs) {
-            string[] toks = c.Split(':');
-            string inp_col = toks[0].Trim();
-            string op_txt = toks[1].Trim();
-            MultiRowOp op = Enum.Parse<MultiRowOp>(op_txt, true);
-            rv.Add(Tuple.Create(inp_col, op));
-         }
-         return new MultiRowOpHelper(rv);
-      }
-
-      public MultiRowOpHelper(List<Tuple<string, MultiRowOp>> operations_) {
-         operations = operations_;
-      }
-
-      // aggregates multiple rows into one - ie does group_by compute
-      public static IBlueRow group_by_compute(List<IBlueRow> rows, MultiRowOpHelper parameters) {
-         if (rows.Count() == 0) {
-            return null;
-         }
-
-         IBlueRow ir = rows.ElementAt(0);
-         IBlueTableSchema in_schema = ir.schema;
-         IBlueTableSchema out_schema = in_schema;
-         IDictionary<string, Tuple<string, MultiRowOp>> out_col_map =
-            new Dictionary<string, Tuple<string, MultiRowOp>>();
-         foreach (var t in parameters.operations) {
-            string out_col_name = $"{t.Item1}_{t.Item2}".ToLower();
-            out_schema = out_schema.add_column(TypeCode.Double, out_col_name);
-            out_col_map[out_col_name] = t;
-         }
-         MutableBlueRow or = new MutableBlueRow(out_schema, ir);
-         foreach (var kv in out_col_map) {
-            string out_col_name = kv.Key;
-            string in_col_name = kv.Value.Item1;
-            MultiRowOp op = kv.Value.Item2;
-            IEnumerable<double> col_vals = BlueDataUtility.get_colvalues<double>(rows, in_col_name);
-            double comp_res = compute_op(col_vals, op);
-            or.setValue(out_col_name, comp_res);
-         }
-         return or;
-      }
-
-      // aggregates multiple rows into one, used by MultiRowRollingMathStream
-      public static IBlueRow group_by_compute(List<IBlueRow> rows, int start_idx, int n_rows, MultiRowOpHelper parameters) {
-         if (rows.Count() == 0) {
-            return null;
-         }
-         IBlueRow ir = rows.ElementAt(start_idx);
-         IBlueTableSchema in_schema = ir.schema;
-         IBlueTableSchema out_schema = in_schema;
-         // out_col vs inp_cal + type
-         IDictionary<string, Tuple<string, MultiRowOp>> out_col_map =
-            new Dictionary<string, Tuple<string, MultiRowOp>>();
-         foreach (var t in parameters.operations) {
-            string out_col_name = $"{t.Item1}_{t.Item2}".ToLower();
-            out_schema = out_schema.add_column(TypeCode.Double, out_col_name);
-            out_col_map[out_col_name] = t;
-         }
-         MutableBlueRow or = new MutableBlueRow(out_schema, ir);
-         foreach (var kv in out_col_map) {
-            string out_col_name = kv.Key;
-            string in_col_name = kv.Value.Item1;
-            MultiRowOp op = kv.Value.Item2;
-            IEnumerable<double> col_vals = BlueDataUtility.get_colvalues<double>(rows, start_idx, n_rows, in_col_name);
-            double comp_res = compute_op(col_vals, op);
-            or.setValue(out_col_name, comp_res);
-            or.setValue("id", ir["id"]);
-         }
-         return or;
-      }
-
-      // rolling_rows are the rolling window of rows. the 0th row is the one thats gettin
-      // calculated. rows 1 - n-1 are its window rows. the schema is based on the operations.
-
-      public IBlueRow rolling_calc(List<IBlueRow> rolling_rows) {
-         if (rolling_rows.Count() == 0) {
-            return null;
-         }
-         IBlueRow ir = rolling_rows.ElementAt(0);
-         IBlueTableSchema in_schema = ir.schema;
-         IBlueTableSchema out_schema = in_schema;
-         IDictionary<string, Tuple<string, MultiRowOp>> out_col_map =
-            new Dictionary<string, Tuple<string, MultiRowOp>>();
-         foreach (var t in operations) {
-            string out_col_name = $"{t.Item1}_{t.Item2}".ToLower();
-            out_schema = out_schema.add_column(TypeCode.Double, out_col_name);
-            out_col_map[out_col_name] = t;
-         }
-         MutableBlueRow or = new MutableBlueRow(out_schema, ir);
-         foreach (var kv in out_col_map) {
-            string out_col_name = kv.Key;
-            string in_col_name = kv.Value.Item1;
-            MultiRowOp op = kv.Value.Item2;
-            IEnumerable<double> col_vals = BlueDataUtility.get_colvalues<double>(rolling_rows, in_col_name);
-            double comp_res = compute_op(col_vals, op);
-            or.setValue(out_col_name, comp_res);
-         }
-         return or;
-      }
-
-      public static double compute_op(IEnumerable<double> col_vals, MultiRowOp op) {
-         switch (op) {
-            case MultiRowOp.Max:
-               return col_vals.Max<double>();
-            case MultiRowOp.Min:
-               return col_vals.Min<double>();
-            case MultiRowOp.Mean:
-               return MathNet.Numerics.Statistics.Statistics.Mean(col_vals);
-            case MultiRowOp.Sum:
-               return col_vals.Sum();
-            case MultiRowOp.Median:
-               return MathNet.Numerics.Statistics.Statistics.Median(col_vals);
-            //case MultiRowOp.Mode:
-            //   return MathNet.Numerics.Statistics.Statistics.(col_vals);
-            case MultiRowOp.First:
-               return col_vals.First();
-            case MultiRowOp.Last:
-               return col_vals.Last();
-            case MultiRowOp.StdDev:
-               return MathNet.Numerics.Statistics.Statistics.StandardDeviation(col_vals);
-            case MultiRowOp.EdgeDiff:
-               return col_vals.First() - col_vals.Last();
-            case MultiRowOp.EdgeDiffPercentage: {
-                  double first = col_vals.First();
-                  double last = col_vals.Last();
-                  return ((first - last) / last) * 100.0;
-                  return col_vals.First() - col_vals.Last();
-               }
-            default:
-               // TODO
-               Log.assert(false);
-               return Double.NaN;
-         }
-      }
-
-   }
-}
@@ -1,40 +0,0 @@
-namespace bts.BlueOrb {
-   using BCG;
-
-   using System.Collections.Generic;
-   using System.Net;
-
-   [DataStreamUserRespresentation("MultiRowRollingMathStream")]
-   public class MultiRowRollingMathStream : ViewStream {
-      private ViewStream _sorted_parent_str;
-      private MultiRowOpHelper _op;
-      private int _rolling_window_sz;
-      public MultiRowRollingMathStream(ViewStream parent_str, string op_txt, string sort_exp, int rolling_window_sz) {
-         _sorted_parent_str = new SortStream(parent_str, sort_exp);
-         _op = MultiRowOpHelper.parse_ops(op_txt);
-         _rolling_window_sz = rolling_window_sz;
-      }
-
-      protected override void myOnFirstSubscriber() {
-         _sorted_parent_str.onChange += _parent_str_onchange;
-      }      
-
-      protected override void myOnLastSubscriber() {
-         _sorted_parent_str.onChange -= _parent_str_onchange;
-      }
-
-      private void _parent_str_onchange(object dp_obj) {
-         DataPacket dp = dp_obj as DataPacket;
-         var rows = dp.newRows.to_list();
-         BlueTable out_table = null;
-         for (int i = 0; i < rows.Count; i++) {
-            var out_r = MultiRowOpHelper.group_by_compute(rows, i, _rolling_window_sz, _op);
-            if (out_table == null)
-               out_table = new BlueTable(out_r.schema);
-            out_table.addRow(out_r);
-         }            
-         var dp_out = BlueDataUtility.build_dp_inc(out_table, myLastValue as DataPacket, true);
-         myFireEvent(dp_out);
-      }
-   }
-}
@@ -1,29 +0,0 @@
-namespace Bts.DataStream.Parquet;
-
-using bts.collections;
-
-using System;
-using System.Collections.Generic;
-
-public interface IParquetColumnReader : IDisposable {
-   IList<object?> Read();
-   IList<object?> Read(int? start, int? stop);
-}
-
-public interface IParquetColumnReader<TLogical, TPhysical> : IParquetColumnReader {
-   new IList<TLogical?> Read();
-   new IList<TLogical?> Read(int? start, int? stop);
-
-   IList<object?> IParquetColumnReader.Read() =>
-      Read().WrapSelector(v => (object?)v, null);
-
-   IList<object?> IParquetColumnReader.Read(int? start, int? stop) =>
-      Read(start, stop).WrapSelector(v => (object?)v, null);
-}
-
-public interface IParquetColumnReader<T> : IParquetColumnReader<T, T> {
-   new IList<T?> Read();
-
-   IList<object?> IParquetColumnReader.Read() =>
-      Read().WrapSelector(v => (object?)v, null);
-}
@@ -1,507 +0,0 @@
-namespace Bts.DataStream.Parquet;
-
-using bts;
-using bts.collections;
-
-using mbts.utils;
-
-using ParquetSharp;
-
-using Spark.Core.Collections;
-
-using System;
-using System.Collections.Generic;
-
-public static class ParquetAdapter {
-   public static IParquetColumnReader GetColumnReader(this RowGroupReader rowGroupReader, int index, Type? colType = null) {
-      int rowCount = checked((int)rowGroupReader.MetaData.NumRows);
-      var colReader = rowGroupReader.Column(index);
-      return getColumnReader(colReader, rowCount, colType: colType);
-   }
-
-   public static IList<object?>[] ReadColumns(this ParquetFileReader fileReader) {
-      var schema = fileReader.FileMetaData.Schema.ToBlueSchema("parquetReader");
-      var colInfos = schema.columnInfo;
-      var colTypes = Array.ConvertAll(colInfos, c => c.colType);
-      var colValues = new IList<object?>[fileReader.FileMetaData.NumColumns];
-      int rowCount = checked((int)fileReader.FileMetaData.NumRows);
-      for (int i = 0; i < colValues.Length; i++) {
-         colValues[i] = new List<object?>(rowCount);
-      }
-      int rowGroupCount = fileReader.FileMetaData.NumRowGroups;
-      for (int rowGroup = 0; rowGroup < rowGroupCount; ++rowGroup) {
-         using var rowGroupReader = fileReader.RowGroup(rowGroup);
-         rowGroupReader.ReadColumnsTo(ref colValues, colTypes);
-      }
-      return colValues;
-   }
-
-   public static IList<object?>[] ReadColumns(this ParquetFileReader fileReader, int? start, int? stop) {
-      if (!start.HasValue && !stop.HasValue) {
-         return ReadColumns(fileReader);
-      }
-      var schema = fileReader.FileMetaData.ToBlueSchema("parquetReader");
-      var colInfos = schema.columnInfo;
-      var colTypes = Array.ConvertAll(colInfos, c => c.colType);
-      var colValues = new IList<object?>[fileReader.FileMetaData.NumColumns];
-      int rowCount = checked((int)fileReader.FileMetaData.NumRows);
-      start ??= 0;
-      stop ??= rowCount;
-      var sliceLength = stop.Value - start.Value;
-      for (int i = 0; i < colValues.Length; i++) {
-         colValues[i] = new List<object?>(sliceLength);
-      }
-      int numRowGroups = fileReader.FileMetaData.NumRowGroups;
-      int? numRowsPerGroup = null;
-      int rowIdx = 0;
-      for (int rowGroup = 0; rowGroup < numRowGroups; ++rowGroup) {
-         if (stop.Value <= rowIdx) {
-            break;
-         }
-         using var rowGroupReader = fileReader.RowGroup(rowGroup);
-         int numRowsInGroup = checked((int)rowGroupReader.MetaData.NumRows);
-         numRowsPerGroup ??= numRowsInGroup;
-         if (rowGroup < numRowGroups - 1) {
-            Log.assert(numRowsInGroup == numRowsPerGroup.Value);
-         } else {
-            Log.assert(numRowsInGroup <= numRowsPerGroup.Value);
-         }
-         int startIdx = start.Value - rowIdx;
-         int stopIdx = stop.Value - rowIdx;
-         rowIdx += numRowsInGroup;
-         if (startIdx >= numRowsInGroup) {
-            continue;
-         }
-         startIdx = Math.Max(0, startIdx);
-         stopIdx = Math.Min(stopIdx, numRowsInGroup);
-         int n = rowGroupReader.ReadColumnsTo(ref colValues, colTypes, startIdx, stopIdx);
-         Log.assert(n == (stopIdx - startIdx));
-      }
-      return colValues;
-   }
-
-   public static int ReadColumnsTo(this RowGroupReader rowGroupReader, ref IList<object?>[] columnValues, Type[] colTypes) {
-      int rowCount = checked((int)rowGroupReader.MetaData.NumRows);
-      for (int colIdx = rowGroupReader.MetaData.NumColumns - 1; colIdx > -1; colIdx--) {
-         using var colReader = rowGroupReader.GetColumnReader(colIdx, colTypes[colIdx]);
-         columnValues[colIdx].AddRange(colReader.Read());
-      }
-      return rowCount;
-   }
-
-   public static int ReadColumnsTo(this RowGroupReader rowGroupReader, ref IList<object?>[] columnValues, Type[] colTypes, int? start, int? stop) {
-      if (!start.HasValue && !stop.HasValue) {
-         return ReadColumnsTo(rowGroupReader, ref columnValues, colTypes);
-      }
-      int numRows = checked((int)rowGroupReader.MetaData.NumRows);
-      if (start.HasValue && start.Value >= numRows) {
-         return 0;
-      }
-      int? readRowCount = null;
-      for (int colIdx = rowGroupReader.MetaData.NumColumns - 1; colIdx > -1; colIdx--) {
-         using var colReader = rowGroupReader.GetColumnReader(colIdx, colTypes[colIdx]);
-         var colVals = colReader.Read(start, stop);
-         readRowCount ??= colVals.Count;
-         Log.assert(readRowCount == colVals.Count);
-         columnValues[colIdx].AddRange(colVals);
-      }
-      return readRowCount ?? throw new ArgumentException("Row group has no columns", nameof(rowGroupReader));
-   }
-
-   static IParquetColumnReader getColumnReader(ColumnReader reader, int bufferSize = 4096, Type? colType = null) {
-      var desc = reader.ColumnDescriptor;
-      var logicalType = desc.LogicalType;
-      return logicalType.Type switch {
-         LogicalTypeEnum.Int => IntReader(),
-         LogicalTypeEnum.String => new ParquetColumnReader<string?>(reader, bufferSize),
-         LogicalTypeEnum.Date => new ParquetColumnReader<DateTime?>(reader, bufferSize),
-         LogicalTypeEnum.Decimal => DecimalReader(),
-         LogicalTypeEnum.Timestamp => TimestampReader(),
-         LogicalTypeEnum.None => DefaultReader(),
-         _ => throw new NotImplementedException(),
-      };
-
-      IParquetColumnReader DefaultReader() {
-         return desc.PhysicalType switch {
-            PhysicalType.Boolean => new ParquetColumnReader<bool?>(reader, bufferSize),
-            PhysicalType.Int32 => new ParquetColumnReader<int?>(reader, bufferSize),
-            PhysicalType.Int64 => new ParquetColumnReader<long?>(reader, bufferSize),
-            PhysicalType.Float => new ParquetColumnReader<float?>(reader, bufferSize),
-            PhysicalType.Double => new ParquetColumnReader<double?>(reader, bufferSize),
-            _ => throw new NotImplementedException(),
-         };
-      }
-
-      IParquetColumnReader TimestampReader() {
-         var tlt = ExceptionAssert.of_type<TimestampLogicalType>(logicalType);
-         return new ParquetColumnReader<DateTimeNanos?, DateTime?>(reader, dtn => dtn?.DateTime, bufferSize);
-      }
-
-      IParquetColumnReader IntReader() {
-         var ilt = ExceptionAssert.of_type<IntLogicalType>(logicalType);
-         if (!ilt.IsSigned) {
-            if (colType is null) {
-               return ilt.BitWidth switch {
-                  8 => new ParquetColumnReader<byte?>(reader, bufferSize),
-                  16 => new ParquetColumnReader<ushort?>(reader, bufferSize),
-                  32 => new ParquetColumnReader<uint?>(reader, bufferSize),
-                  64 => new ParquetColumnReader<ulong?>(reader, bufferSize),
-                  _ => throw new NotImplementedException(),
-               };
-            } else if (colType == typeof(byte)) {
-               return ilt.BitWidth switch {
-                  8 => new ParquetColumnReader<byte?>(reader, bufferSize),
-                  16 => new ParquetColumnReader<ushort?, byte?>(reader, v => v.HasValue ? checked((byte)v.Value) : null, bufferSize),
-                  32 => new ParquetColumnReader<uint?, ushort?>(reader, v => v.HasValue ? checked((byte)v.Value) : null, bufferSize),
-                  64 => new ParquetColumnReader<ulong?, ushort?>(reader, v => v.HasValue ? checked((byte)v.Value) : null, bufferSize),
-                  _ => throw new NotImplementedException(),
-               };
-            } else if (colType == typeof(ushort)) {
-               return ilt.BitWidth switch {
-                  8 => new ParquetColumnReader<byte?, ushort?>(reader, v => v.HasValue ? v.Value : null, bufferSize),
-                  16 => new ParquetColumnReader<ushort?>(reader, bufferSize),
-                  32 => new ParquetColumnReader<uint?, ushort?>(reader, v => v.HasValue ? checked((ushort)v.Value) : null, bufferSize),
-                  64 => new ParquetColumnReader<ulong?, ushort?>(reader, v => v.HasValue ? checked((ushort)v.Value) : null, bufferSize),
-                  _ => throw new NotImplementedException(),
-               };
-            } else if (colType == typeof(uint)) {
-               return ilt.BitWidth switch {
-                  8 => new ParquetColumnReader<byte?, uint?>(reader, v => v.HasValue ? v.Value : null, bufferSize),
-                  16 => new ParquetColumnReader<ushort?, uint?>(reader, v => v.HasValue ? v.Value : null, bufferSize),
-                  32 => new ParquetColumnReader<uint?>(reader, bufferSize),
-                  64 => new ParquetColumnReader<ulong?, uint?>(reader, v => v.HasValue ? checked((uint)v.Value) : null, bufferSize),
-                  _ => throw new NotImplementedException(),
-               };
-            } else if (colType == typeof(ulong)) {
-               return ilt.BitWidth switch {
-                  8 => new ParquetColumnReader<byte?, ulong?>(reader, v => v.HasValue ? v.Value : null, bufferSize),
-                  16 => new ParquetColumnReader<ushort?, ulong?>(reader, v => v.HasValue ? v.Value : null, bufferSize),
-                  32 => new ParquetColumnReader<uint?, ulong?>(reader, v => v.HasValue ? v.Value : null, bufferSize),
-                  64 => new ParquetColumnReader<ulong?>(reader, bufferSize),
-                  _ => throw new NotImplementedException(),
-               };
-            }
-            throw new NotImplementedException($"Expected signed int column type, but got {colType}");
-         } else {
-            if (colType is null) {
-               return ilt.BitWidth switch {
-                  8 => new ParquetColumnReader<sbyte?>(reader, bufferSize),
-                  16 => new ParquetColumnReader<short?>(reader, bufferSize),
-                  32 => new ParquetColumnReader<int?>(reader, bufferSize),
-                  64 => new ParquetColumnReader<long?>(reader, bufferSize),
-                  _ => throw new NotImplementedException(),
-               };
-            } else if (colType == typeof(sbyte)) {
-               return ilt.BitWidth switch {
-                  8 => new ParquetColumnReader<sbyte?>(reader, bufferSize),
-                  16 => new ParquetColumnReader<short?, sbyte?>(reader, v => v.HasValue ? checked((sbyte)v.Value) : null, bufferSize),
-                  32 => new ParquetColumnReader<int?, sbyte?>(reader, v => v.HasValue ? checked((sbyte)v.Value) : null, bufferSize),
-                  64 => new ParquetColumnReader<long?, sbyte?>(reader, v => v.HasValue ? checked((sbyte)v.Value) : null, bufferSize),
-                  _ => throw new NotImplementedException(),
-               };
-            } else if (colType == typeof(short)) {
-               return ilt.BitWidth switch {
-                  8 => new ParquetColumnReader<char?, short?>(reader, v => v.HasValue ? checked((short)v.Value) : null, bufferSize),
-                  16 => new ParquetColumnReader<short?>(reader, bufferSize),
-                  32 => new ParquetColumnReader<int?, short?>(reader, v => v.HasValue ? checked((short)v.Value) : null, bufferSize),
-                  64 => new ParquetColumnReader<long?, short?>(reader, v => v.HasValue ? checked((short)v.Value) : null, bufferSize),
-                  _ => throw new NotImplementedException(),
-               };
-            } else if (colType == typeof(int)) {
-               return ilt.BitWidth switch {
-                  8 => new ParquetColumnReader<char?, int?>(reader, v => v.HasValue ? v.Value : null, bufferSize),
-                  16 => new ParquetColumnReader<short?, int?>(reader, v => v.HasValue ? v.Value : null, bufferSize),
-                  32 => new ParquetColumnReader<int?>(reader, bufferSize),
-                  64 => new ParquetColumnReader<long?, int?>(reader, v => v.HasValue ? checked((int)v.Value) : null, bufferSize),
-                  _ => throw new NotImplementedException(),
-               };
-            } else if (colType == typeof(long)) {
-               return ilt.BitWidth switch {
-                  8 => new ParquetColumnReader<char?, long?>(reader, v => v.HasValue ? v.Value : null, bufferSize),
-                  16 => new ParquetColumnReader<short?, long?>(reader, v => v.HasValue ? v.Value : null, bufferSize),
-                  32 => new ParquetColumnReader<int?, long?>(reader, v => v.HasValue ? v.Value : null, bufferSize),
-                  64 => new ParquetColumnReader<long?>(reader, bufferSize),
-                  _ => throw new NotImplementedException(),
-               };
-            }
-            throw new NotImplementedException($"Expected unsigned int column type, but got {colType}");
-         }
-      }
-
-      IParquetColumnReader DecimalReader() {
-         var dlt = ExceptionAssert.of_type<DecimalLogicalType>(logicalType);
-         return new ParquetColumnReader<double?>(reader, bufferSize);
-      }
-   }
-
-   public static IColumnWriter GetColumnWriter(TypeCode typeCode, ColumnWriter writer, int bufferSize = 4096) {
-      return typeCode switch {
-         TypeCode.Boolean => new ColumnWriter<bool?>(writer, bufferSize),
-         TypeCode.SByte => new ColumnWriter<sbyte?>(writer, bufferSize, GetInt8Converter((IntLogicalType)writer.ColumnDescriptor.LogicalType)),
-         TypeCode.Int16 => new ColumnWriter<short?>(writer, bufferSize, GetInt16Converter((IntLogicalType)writer.ColumnDescriptor.LogicalType)),
-         TypeCode.Int32 => new ColumnWriter<int?>(writer, bufferSize, GetInt32Converter((IntLogicalType)writer.ColumnDescriptor.LogicalType)),
-         TypeCode.Int64 => new ColumnWriter<long?>(writer, bufferSize, GetInt64Converter((IntLogicalType)writer.ColumnDescriptor.LogicalType)),
-         TypeCode.Byte => new ColumnWriter<byte?>(writer, bufferSize, GetUInt8Converter((IntLogicalType)writer.ColumnDescriptor.LogicalType)),
-         TypeCode.UInt16 => new ColumnWriter<ushort?>(writer, bufferSize, GetUInt16Converter((IntLogicalType)writer.ColumnDescriptor.LogicalType)),
-         TypeCode.UInt32 => new ColumnWriter<uint?>(writer, bufferSize, GetUInt32Converter((IntLogicalType)writer.ColumnDescriptor.LogicalType)),
-         TypeCode.UInt64 => new ColumnWriter<ulong?>(writer, bufferSize, GetUInt64Converter((IntLogicalType)writer.ColumnDescriptor.LogicalType)),
-         TypeCode.Single => new ColumnWriter<float?>(writer, bufferSize),
-         TypeCode.Double => new ColumnWriter<double?>(writer, bufferSize),
-         TypeCode.DateTime => new ColumnWriter<DateTimeNanos?>(writer, bufferSize, ConvertDateTimeObj),
-         TypeCode.String => new ColumnWriter<string?>(writer, bufferSize),
-         _ => throw new NotSupportedException($"Unsupported column TypeCode {typeCode}"),
-      };
-
-      static DateTimeNanos? ConvertDateTimeObj(object? obj) {
-         if (obj is DateTime dt) {
-            return new(dt);
-         }
-         if (obj is DateTimeNanos dtn) {
-            return dtn;
-         }
-         throw new InvalidCastException($"Unable to convert object of type {obj?.GetType()} to DateTimeNanos");
-      }
-   }
-
-   public static Func<object?, sbyte?> GetInt8Converter(IntLogicalType logicalType) =>
-      GetInt8Converter(logicalType.IsSigned, logicalType.BitWidth);
-
-   public static Func<object?, sbyte?> GetInt8Converter(bool isSigned, int bitWidth) =>
-      isSigned
-         ? bitWidth switch {
-            8 => obj => obj as sbyte?,
-            16 => obj => obj is short v ? checked((sbyte)v) : null,
-            32 => obj => obj is int v ? checked((sbyte)v) : null,
-            64 => obj => obj is long v ? checked((sbyte)v) : null,
-            _ => throw new NotImplementedException(),
-         }
-         : bitWidth switch {
-            8 => obj => obj is byte v ? checked((sbyte)v) : null,
-            16 => obj => obj is ushort v ? checked((sbyte)v) : null,
-            32 => obj => obj is uint v ? checked((sbyte)v) : null,
-            64 => obj => obj is ulong v ? checked((sbyte)v) : null,
-            _ => throw new NotImplementedException(),
-         };
-
-   public static Func<object?, short?> GetInt16Converter(IntLogicalType logicalType) =>
-      GetInt16Converter(logicalType.IsSigned, logicalType.BitWidth);
-
-   public static Func<object?, short?> GetInt16Converter(bool isSigned, int bitWidth) =>
-      isSigned
-         ? bitWidth switch {
-            8 => obj => obj is sbyte v ? v : null,
-            16 => obj => obj as short?,
-            32 => obj => obj is int v ? checked((short)v) : null,
-            64 => obj => obj is long v ? checked((short)v) : null,
-            _ => throw new NotImplementedException(),
-         }
-         : bitWidth switch {
-            8 => obj => obj is byte v ? v : null,
-            16 => obj => obj is ushort v ? checked((short)v) : null,
-            32 => obj => obj is uint v ? checked((short)v) : null,
-            64 => obj => obj is ulong v ? checked((short)v) : null,
-            _ => throw new NotImplementedException(),
-         };
-
-   public static Func<object?, int?> GetInt32Converter(IntLogicalType logicalType) =>
-      GetInt32Converter(logicalType.IsSigned, logicalType.BitWidth);
-
-   public static Func<object?, int?> GetInt32Converter(bool isSigned, int bitWidth) =>
-      isSigned
-         ? bitWidth switch {
-            8 => obj => obj is sbyte v ? v : null,
-            16 => obj => obj is short v ? v : null,
-            32 => obj => obj as int?,
-            64 => obj => obj is long v ? checked((int)v) : null,
-            _ => throw new NotImplementedException(),
-         }
-         : bitWidth switch {
-            8 => obj => obj is byte v ? v : null,
-            16 => obj => obj is ushort v ? v : null,
-            32 => obj => obj is uint v ? checked((int)v) : null,
-            64 => obj => obj is ulong v ? checked((int)v) : null,
-            _ => throw new NotImplementedException(),
-         };
-
-   public static Func<object?, long?> GetInt64Converter(IntLogicalType logicalType) =>
-      GetInt64Converter(logicalType.IsSigned, logicalType.BitWidth);
-
-   public static Func<object?, long?> GetInt64Converter(bool isSigned, int bitWidth) =>
-      isSigned
-         ? bitWidth switch {
-            8 => obj => obj is sbyte v ? v : null,
-            16 => obj => obj is short v ? v : null,
-            32 => obj => obj is int v ? v : null,
-            64 => obj => obj as long?,
-            _ => throw new NotImplementedException(),
-         }
-         : bitWidth switch {
-            8 => obj => obj is byte v ? v : null,
-            16 => obj => obj is ushort v ? v : null,
-            32 => obj => obj is uint v ? v : null,
-            64 => obj => obj is ulong v ? checked((long)v) : null,
-            _ => throw new NotImplementedException(),
-         };
-
-   public static Func<object?, byte?> GetUInt8Converter(IntLogicalType logicalType) =>
-      GetUInt8Converter(logicalType.IsSigned, logicalType.BitWidth);
-
-   public static Func<object?, byte?> GetUInt8Converter(bool isSigned, int bitWidth) =>
-      isSigned
-         ? bitWidth switch {
-            8 => obj => obj is sbyte v ? checked((byte)v) : null,
-            16 => obj => obj is short v ? checked((byte)v) : null,
-            32 => obj => obj is int v ? checked((byte)v) : null,
-            64 => obj => obj is long v ? checked((byte)v) : null,
-            _ => throw new NotImplementedException(),
-         }
-         : bitWidth switch {
-            8 => obj => obj as byte?,
-            16 => obj => obj is ushort v ? checked((byte)v) : null,
-            32 => obj => obj is uint v ? checked((byte)v) : null,
-            64 => obj => obj is ulong v ? checked((byte)v) : null,
-            _ => throw new NotImplementedException(),
-         };
-
-   public static Func<object?, ushort?> GetUInt16Converter(IntLogicalType logicalType) =>
-      GetUInt16Converter(logicalType.IsSigned, logicalType.BitWidth);
-
-   public static Func<object?, ushort?> GetUInt16Converter(bool isSigned, int bitWidth) =>
-      isSigned
-         ? bitWidth switch {
-            8 => obj => obj is sbyte v ? checked((ushort)v) : null,
-            16 => obj => obj is short v ? checked((ushort)v) : null,
-            32 => obj => obj is int v ? checked((ushort)v) : null,
-            64 => obj => obj is long v ? checked((ushort)v) : null,
-            _ => throw new NotImplementedException(),
-         }
-         : bitWidth switch {
-            8 => obj => obj is byte v ? v : null,
-            16 => obj => obj is ushort v ? v : null,
-            32 => obj => obj is uint v ? checked((ushort)v) : null,
-            64 => obj => obj is ulong v ? checked((ushort)v) : null,
-            _ => throw new NotImplementedException(),
-         };
-
-   public static Func<object?, uint?> GetUInt32Converter(IntLogicalType logicalType) =>
-      GetUInt32Converter(logicalType.IsSigned, logicalType.BitWidth);
-
-   public static Func<object?, uint?> GetUInt32Converter(bool isSigned, int bitWidth) =>
-      isSigned
-         ? bitWidth switch {
-            8 => obj => obj is sbyte v ? checked((uint)v) : null,
-            16 => obj => obj is short v ? checked((uint)v) : null,
-            32 => obj => obj is int v ? checked((uint)v) : null,
-            64 => obj => obj is long v ? checked((uint)v) : null,
-            _ => throw new NotImplementedException(),
-         }
-         : bitWidth switch {
-            8 => obj => obj is byte v ? v : null,
-            16 => obj => obj is ushort v ? v : null,
-            32 => obj => obj is uint v ? v : null,
-            64 => obj => obj is ulong v ? checked((uint)v) : null,
-            _ => throw new NotImplementedException(),
-         };
-
-   public static Func<object?, ulong?> GetUInt64Converter(IntLogicalType logicalType) =>
-      GetUInt64Converter(logicalType.IsSigned, logicalType.BitWidth);
-
-   public static Func<object?, ulong?> GetUInt64Converter(bool isSigned, int bitWidth) =>
-      isSigned
-         ? bitWidth switch {
-            8 => obj => obj is sbyte v ? checked((ulong)v) : null,
-            16 => obj => obj is short v ? checked((ulong)v) : null,
-            32 => obj => obj is int v ? checked((ulong)v) : null,
-            64 => obj => obj is long v ? checked((ulong)v) : null,
-            _ => throw new NotImplementedException(),
-         }
-         : bitWidth switch {
-            8 => obj => obj is byte v ? v : null,
-            16 => obj => obj is ushort v ? v : null,
-            32 => obj => obj is uint v ? v : null,
-            64 => obj => obj is ulong v ? v : null,
-            _ => throw new NotImplementedException(),
-         };
-
-   public static string ColumnEnumTypeKey(string colName) =>
-      $"{colName}.EnumType";
-
-   public static KeyValuePair<string, string> ColumnEnumTypeEntry(string colName, Type enumType) {
-      if (enumType.IsNullable()) {
-         enumType = enumType.GetGenericArguments()[0];
-      }
-      if (!enumType.IsEnum) {
-         throw new ArgumentException($"Must be enum type (actually {enumType})", nameof(enumType));
-      }
-      return new(ColumnEnumTypeKey(colName), enumType.ToString());
-   }
-
-   public static Type? GetColumnEnumType(IReadOnlyDictionary<string, string>? keyValueMetadata, string columnName) {
-      if (keyValueMetadata is not null) {
-         var key = ColumnEnumTypeKey(columnName);
-         if (keyValueMetadata.TryGetValue(key, out var typeName)) {
-            return Type.GetType(typeName);
-         }
-      }
-      return null;
-   }
-
-   public interface IColumnWriter : IDisposable {
-      void Write(IReadOnlyCollection<object?> values);
-   }
-
-   public interface IColumnWriter<T> : IColumnWriter {
-      void Write(IReadOnlyCollection<T?> values);
-   }
-
-   class ColumnWriter<T> : IColumnWriter<T> {
-      readonly T?[] buffer;
-      readonly LogicalColumnWriter<T> writer;
-      readonly Func<object?, T?> convert;
-
-      public ColumnWriter(ColumnWriter writer, int bufferSize)
-         : this(writer, new T?[bufferSize]) { }
-
-      public ColumnWriter(ColumnWriter writer, int bufferSize, Func<object?, T?> convert)
-         : this(writer, new T?[bufferSize], convert) { }
-
-      public ColumnWriter(ColumnWriter writer, T?[] buffer)
-         : this(writer, buffer, obj => (T?)obj) { }
-
-      public ColumnWriter(ColumnWriter writer, T?[] buffer, Func<object?, T?> convert) {
-         this.buffer = buffer;
-         this.writer = writer.LogicalWriter<T>();
-         this.convert = convert;
-      }
-
-      public void Write(IReadOnlyCollection<T?> values) {
-         if (buffer.Length == 0) {
-            throw new InvalidOperationException("Cannot write because buffer length is 0");
-         }
-         var count = values.Count;
-         var nPages = count / buffer.Length;
-         using var en = values.GetEnumerator();
-         for (int pg = 0; pg < nPages; pg++) {
-            for (int i = 0; i < buffer.Length; i++) {
-               Log.assert(en.MoveNext(), "Enumeration failed before reaching expected count");
-               buffer[i] = en.Current;
-            }
-            writer.WriteBatch(buffer!);
-         }
-         var remainder = count - nPages * buffer.Length;
-         if (remainder != 0) {
-            for (int i = 0; i < remainder; i++) {
-               Log.assert(en.MoveNext(), "Enumeration failed before reaching expected count");
-               buffer[i] = en.Current;
-            }
-            writer.WriteBatch(buffer!, 0, remainder);
-         }
-      }
-
-      public void Dispose() =>
-         writer.Dispose();
-
-      void IColumnWriter.Write(IReadOnlyCollection<object?> values) =>
-         Write(ReadOnlySelectorCollection.Create(values, convert));
-   }
-}
@@ -1,111 +0,0 @@
-namespace Bts.DataStream.Parquet;
-
-using bts;
-using bts.collections;
-
-using ParquetSharp;
-
-using System;
-using System.Collections.Generic;
-
-class ParquetColumnReader<T> : IParquetColumnReader<T> {
-   protected readonly LogicalColumnReader<T> reader;
-   protected readonly T?[] buffer;
-
-   public ParquetColumnReader(ColumnReader reader, int bufferSize)
-      : this(reader, new T?[bufferSize]) { }
-
-   protected ParquetColumnReader(ColumnReader reader, T?[] buffer) {
-      this.buffer = buffer;
-      this.reader = reader.LogicalReader<T>();
-   }
-
-   ~ParquetColumnReader() =>
-      Dispose();
-
-   public IList<T?> Read() {
-      if (buffer.Length == 0) {
-         throw new InvalidOperationException("Cannot write because buffer length is 0");
-      }
-      int n = reader.ReadBatch(buffer!);
-      return buffer.WrapSubRange(0, n);
-   }
-
-   public IList<T?> Read(int? start, int? stop) {
-      if (buffer.Length == 0) {
-         throw new InvalidOperationException("Cannot write because buffer length is 0");
-      }
-      start ??= 0;
-      stop ??= buffer.Length;
-      int length = stop.Value - start.Value;
-      if (length <= 0) {
-         return new List<T?>(0);
-      }
-      if (start.Value > 0) {
-         var nSkipped = checked((int)reader.Skip(start.Value));
-         Log.assert(nSkipped >= 0 && nSkipped <= start.Value);
-         if (nSkipped < start.Value) {
-            return new List<T?>(0);
-         }
-      }
-      int n = reader.ReadBatch(buffer!, 0, length);
-      return buffer.WrapSubRange(0, n);
-   }
-
-   public void Dispose() {
-      GC.SuppressFinalize(this);
-      reader.Dispose();
-   }
-}
-
-class ParquetColumnReader<TIn, TOut> : IParquetColumnReader<TOut> {
-   readonly Func<TIn, TOut?> convert;
-   readonly LogicalColumnReader<TIn> reader;
-   readonly TIn[] buffer;
-
-   public ParquetColumnReader(ColumnReader reader, Func<TIn, TOut?> convert, int bufferSize)
-      : this(reader, convert, new TIn[bufferSize]) { }
-
-   public ParquetColumnReader(ColumnReader reader, Func<TIn, TOut?> convert, TIn[] buffer) {
-      this.convert = convert;
-      this.reader = reader.LogicalReader<TIn>();
-      this.buffer = buffer;
-   }
-
-   ~ParquetColumnReader() =>
-      Dispose();
-
-   public IList<TOut?> Read() {
-      if (buffer.Length == 0) {
-         throw new InvalidOperationException("Cannot write because buffer length is 0");
-      }
-      int n = reader.ReadBatch(buffer!);
-      return buffer.WrapSelector(convert, null).WrapSubRange(0, n);
-   }
-
-   public IList<TOut?> Read(int? start, int? stop) {
-      if (buffer.Length == 0) {
-         throw new InvalidOperationException("Cannot write because buffer length is 0");
-      }
-      start ??= 0;
-      stop ??= buffer.Length;
-      int length = stop.Value - start.Value;
-      if (length <= 0) {
-         return new List<TOut?>(0);
-      }
-      if (start.Value > 0) {
-         var nSkipped = checked((int)reader.Skip(start.Value));
-         Log.assert(nSkipped >= 0 && nSkipped <= start.Value);
-         if (nSkipped < start.Value) {
-            return new List<TOut?>(0);
-         }
-      }
-      int n = reader.ReadBatch(buffer!, 0, length);
-      return buffer.WrapSelector(convert, null).WrapSubRange(0, n);
-   }
-
-   public void Dispose() {
-      GC.SuppressFinalize(this);
-      reader.Dispose();
-   }
-}
@@ -1,159 +0,0 @@
-namespace Bts.DataStream.Parquet;
-
-using BCG;
-
-using bts;
-using bts.collections;
-
-using Microsoft.Scripting.Utils;
-
-using ParquetSharp;
-
-using System;
-using System.Collections.Generic;
-using System.Linq;
-
-public static class ParquetExtensions {
-   public static int RowCount(this FileMetaData fileMetaData) =>
-      checked((int)fileMetaData.NumRows);
-
-   public static IBlueTableSchema ToBlueSchema(this SchemaDescriptor schemaDescriptor, string tableName = "parquet", IReadOnlyDictionary<string, string>? keyValueMetadata = null) {
-      var colInfos = new ColumnInfo[schemaDescriptor.NumColumns];
-      for (int i = 0; i < colInfos.Length; i++) {
-         colInfos[i] = schemaDescriptor.BlueColumn(i, keyValueMetadata);
-      }
-      return new BlueTableSchema(tableName, colInfos);
-   }
-
-   public static IBlueTableSchema ToBlueSchema(this FileMetaData fileMetaData, string tableName = "parquet") =>
-      ToBlueSchema(fileMetaData.Schema, tableName, fileMetaData.KeyValueMetadata);
-
-   public static ColumnInfo BlueColumn(this SchemaDescriptor schemaDescriptor, int index, IReadOnlyDictionary<string, string>? keyValueMetadata = null) {
-      var c = schemaDescriptor.Column(index);
-      var lt = c.LogicalType;
-      if (lt.Type == LogicalTypeEnum.Enum) {
-         var enumType = ParquetAdapter.GetColumnEnumType(keyValueMetadata, c.Name);
-         if (enumType is not null) {
-            return new(c.Name, index, enumType);
-         }
-      }
-      return new(c.Name, index, getTypeCode(c.LogicalType, c.PhysicalType));
-   }
-
-   public static IBlueRowCollection ReadBlueTable(this ParquetFileReader fileReader, int? start = null, int? stop = null) {
-      var schema = fileReader.FileMetaData.ToBlueSchema("parquetReader");
-      var colInfos = schema.columnInfo;
-      IBlueRowCollection? rv;
-      if (!schema.columnInfo.Any(c => c.colName == "id")) {
-         rv = BlueAddIndexColumnTable.Create(schema, fileReader.ReadColumns(start, stop));
-      } else {
-         rv = new BlueIndexedColumnTable(schema, fileReader.ReadColumns(start, stop));
-      }
-      Log.assert(!schema.columnInfo.Any(c => c is null));
-      return rv;
-   }
-
-   public static (Column[] Column, IReadOnlyDictionary<string, string> Meta) ToParquetSchema(this IBlueTableSchema schema) {
-      var blueColumns = schema.columnInfo;
-      var parquetColumns = new Column[blueColumns.Length];
-      var parquetMeta = new Dictionary<string, string>();
-      KeyValuePair<string, string>[] parquetColMeta;
-      for (int i = 0; i < blueColumns.Length; i++) {
-         (parquetColumns[i], parquetColMeta) = ToParquetColumn(blueColumns[i]);
-         parquetMeta.AddRange(parquetColMeta);
-      }
-      return (parquetColumns, parquetMeta);
-   }
-
-   public static (Column Column, KeyValuePair<string, string>[] Meta) ToParquetColumn(this ColumnInfo columnInfo) {
-      var colType = columnInfo.colType;
-      bool isNullable = colType.IsNullable();
-      if (isNullable) {
-         colType = colType.GetGenericArguments()[0];
-      }
-      KeyValuePair<string, string>[] meta;
-      TypeCode typeCode;
-      var colName = columnInfo.colName;
-      if (colType.IsEnum) {
-         typeCode = colType.GetEnumUnderlyingType().GetTypeCode();
-         meta = new[] { ParquetAdapter.ColumnEnumTypeEntry(columnInfo.colName, colType) };
-      } else {
-         typeCode = colType.GetTypeCode();
-         meta = Array.Empty<KeyValuePair<string, string>>();
-      }
-      return (ToParquetColumn(colName, typeCode), meta);
-
-      static Column ToParquetColumn(string colName, TypeCode typeCode) =>
-         typeCode switch {
-            TypeCode.Boolean => new Column<bool?>(colName),
-            TypeCode.Byte => new Column<byte?>(colName),
-            TypeCode.Int16 => new Column<short?>(colName),
-            TypeCode.Int32 => new Column<int?>(colName),
-            TypeCode.Int64 => new Column<long?>(colName),
-            TypeCode.UInt16 => new Column<ushort?>(colName),
-            TypeCode.UInt32 => new Column<uint?>(colName),
-            TypeCode.UInt64 => new Column<ulong?>(colName),
-            TypeCode.Single => new Column<float?>(colName),
-            TypeCode.Double => new Column<double?>(colName),
-            TypeCode.DateTime => new Column<DateTimeNanos?>(colName),
-            TypeCode.String => new Column<string?>(colName),
-            _ => throw new NotSupportedException($"Unsupported column TypeCode {typeCode}"),
-         };
-   }
-
-   static TypeCode getTypeCode(LogicalType logicalType, PhysicalType physicalType) {
-      return logicalType.Type switch {
-         // LogicalTypeEnum.Int => TypeCode.Int32,
-         // LogicalTypeEnum.Decimal => TypeCode.Double,
-         LogicalTypeEnum.Date or LogicalTypeEnum.Time or LogicalTypeEnum.Timestamp => TypeCode.DateTime,
-         LogicalTypeEnum.String => TypeCode.String,
-         LogicalTypeEnum.Int => FromInt(),
-         LogicalTypeEnum.Decimal => FromDecimal(),
-         _ => FromPhysicalType(),
-      };
-
-      TypeCode FromPhysicalType() =>
-         physicalType switch {
-            PhysicalType.Boolean => TypeCode.Boolean,
-            PhysicalType.Int32 => TypeCode.Int32,
-            PhysicalType.Int64 => TypeCode.Int64,
-            PhysicalType.Float => TypeCode.Single,
-            PhysicalType.Double => TypeCode.Double,
-            _ => throw new NotImplementedException($"TypeCode not implemented for {logicalType} (logical) {physicalType} (physical)"),
-         };
-
-      TypeCode FromInt() {
-         var ilt = (IntLogicalType)logicalType;
-         var bitWidth = ilt.BitWidth;
-         return ilt.IsSigned ? FromSignedInt() : FromUnsignedInt();
-
-         TypeCode FromSignedInt() =>
-            bitWidth switch {
-               8 => TypeCode.SByte,
-               16 => TypeCode.Int16,
-               32 => TypeCode.Int32,
-               64 => TypeCode.Int64,
-               _ => throw new NotSupportedException($"Bit width not supported for signed int ({bitWidth})"),
-            };
-
-         TypeCode FromUnsignedInt() =>
-            bitWidth switch {
-               8 => TypeCode.Byte,
-               16 => TypeCode.UInt16,
-               32 => TypeCode.UInt32,
-               64 => TypeCode.UInt64,
-               _ => throw new NotSupportedException($"Bit width not supported for unsigned int ({bitWidth})"),
-            };
-      }
-
-      TypeCode FromDecimal() {
-         var dlt = (DecimalLogicalType)logicalType;
-         var scale = dlt.Scale;
-         var precision = dlt.Precision;
-         return scale switch {
-            0 => throw new NotSupportedException($"Scale not supported for decimal ({scale})"),
-            _ => TypeCode.Double,
-         };
-      }
-   }
-}
@@ -1,234 +0,0 @@
-namespace Bts.DataStream.Parquet;
-
-using BCG;
-
-using bts;
-using bts.collections;
-
-using Bts.DataStreamCore;
-
-using ParquetSharp;
-
-using Spark.Core.Collections;
-
-using System;
-using System.Collections;
-using System.Collections.Generic;
-using System.Diagnostics.CodeAnalysis;
-
-public class ParquetGroupTable : IBlueRowCollection, IDisposable {
-   readonly RowGroupReader reader;
-   readonly IBlueTableSchema schema;
-   readonly bool leaveOpen;
-   readonly int fullCount;
-   readonly object syncRoot = new();
-   readonly int? start;
-   readonly int? stop;
-   readonly int startIndex;
-   readonly int stopIndex;
-   readonly int idColumnIndex;
-   readonly int idColumnIndexInFile;
-   readonly Func<int, int>? rowIdFactory;
-   DataCache? dataCacheBacking;
-
-   public ParquetGroupTable(
-      RowGroupReader reader,
-      IBlueTableSchema schema,
-      int? start = null,
-      int? stop = null,
-      int idColumnIndex = -1,
-      int idColumnIndexInFile = -1,
-      Func<int, int>? rowIdFactory = null,
-      bool leaveOpen = false
-   ) {
-      this.reader = reader;
-      this.schema = schema;
-      this.start = start;
-      this.stop = stop;
-      this.idColumnIndex = idColumnIndex;
-      this.idColumnIndexInFile = idColumnIndexInFile;
-      this.rowIdFactory = rowIdFactory;
-      this.leaveOpen = leaveOpen;
-      fullCount = checked((int)reader.MetaData.NumRows);
-      Log.assert(fullCount >= 0);
-      startIndex = SliceList.GetStartIndex(start, fullCount, true);
-      stopIndex = SliceList.GetStopIndex(stop, fullCount, true);
-      stopIndex = Math.Max(stopIndex, startIndex);
-      Log.assert(
-         startIndex >= 0
-         && startIndex <= fullCount
-         && stopIndex >= 0
-         && stopIndex <= fullCount
-         && stopIndex >= startIndex);
-   }
-
-   ~ParquetGroupTable() =>
-      Dispose();
-
-   public int Count => stopIndex - startIndex;
-
-   IBlueTableSchema IBlueRowCollection.schema => schema;
-
-   DataCache dataCache =>
-      EnsureInit();
-
-   public IBlueRow this[int id] => dataCache[id];
-
-   public bool containsID(int id) =>
-      dataCache.RowIdToIndex.ContainsKey(id);
-
-   public IBlueRow GetRow(int index) =>
-      dataCache.Rows[index];
-
-   public int IndexOf(IBlueRow item) {
-      int id = item.Id();
-      return dataCache.RowIdToIndex.TryGetValue(id, out var idx) && BlueRowEqualityComparer.Equals(dataCache.Rows[idx], item)
-         ? idx
-         : -1;
-   }
-
-   public bool TryGetValue(int id, [NotNullWhen(true)] out IBlueRow? row) {
-      if (dataCache.RowIdToIndex.TryGetValue(id, out var idx)) {
-         row = dataCache.Rows[idx];
-         return true;
-      }
-      row = null;
-      return false;
-   }
-
-   public IEnumerator<IBlueRow> GetEnumerator() {
-      for (int i = 0; i < dataCache.Rows.Length; i++) {
-         yield return dataCache.Rows[i];
-      }
-   }
-
-   public void Dispose() {
-      GC.SuppressFinalize(this);
-      if (!leaveOpen) {
-         reader.Dispose();
-      }
-   }
-
-   [MemberNotNull(nameof(dataCacheBacking))]
-   internal DataCache EnsureInit() {
-      if (dataCacheBacking.HasValue) {
-         return dataCacheBacking.Value;
-      }
-      lock (syncRoot) {
-         if (dataCacheBacking.HasValue) {
-            return dataCacheBacking.Value;
-         }
-         var rows = new IBlueRow[Count];
-         int numColumns = reader.MetaData.NumColumns;
-         var blueColumns = schema.columnInfo;
-         Log.assert((blueColumns.Length - numColumns) switch {
-            0 or 1 => true,
-            _ => false,
-         });
-         var columnValues = allocateColumnValues(
-            reader, blueColumns.Length, Count, idColumnIndex, idColumnIndexInFile, startIndex, stopIndex, rowIdFactory);
-         var rowIdToIndex = new Dictionary<int, int>();
-         for (int i = 0; i < rows.Length; i++) {
-            rows[i] = new BlueRowColumnWrapper(schema, i, columnValues);
-         }
-         dataCacheBacking = new(rows, rowIdToIndex, columnValues);
-      }
-      return dataCacheBacking.Value;
-   }
-
-   static IList<object?>[] allocateColumnValues(
-      RowGroupReader reader,
-      int numColumns,
-      int numRows,
-      int idColIdx,
-      int idColIdxInFile,
-      int startIndex,
-      int stopIndex,
-      Func<int, int>? idFactory
-   ) {
-      if (idColIdx < 0 || idColIdx >= numColumns) {
-         throw new ArgumentException("Id column index out of bounds", nameof(idColIdx));
-      }
-      var columnValues = new IList<object?>[numColumns];
-      if (idColIdxInFile == -1) {
-         // Generate ids (there is no id column in the file)
-         ArgumentNullException.ThrowIfNull(idFactory);
-         for (int i = 0; i < numColumns; i++) {
-            if (i < idColIdx) {
-               ReadColumn(i, i);
-            } else if (i == idColIdx) {
-               columnValues[idColIdx] = new StepList<int>(0, 1, numRows).WrapSelector(idx => (object?)idFactory(idx), null);
-            } else {
-               ReadColumn(i - 1, i);
-            }
-         }
-      } else if (idFactory is not null) {
-         // Generate ids (skip id column in the file)
-         for (int i = 0; i < numColumns; i++) {
-            if (i == idColIdx) {
-               columnValues[idColIdx] = new StepList<int>(0, 1, numRows).WrapSelector(idx => (object?)idFactory(idx), null);
-            } else {
-               if (i > idColIdx && i < idColIdxInFile) {
-                  ReadColumn(i - 1, i);
-               } else if (i > idColIdxInFile && i < idColIdx) {
-                  ReadColumn(i + 1, i);
-               } else {
-                  ReadColumn(i, i);
-               }
-            }
-         }
-      } else {
-         // Get ids from the file (output table id column index may differ from file)
-         // Case 1) File id col index > output
-         //    src cols: a, b, c, d, (id), e, f, ...
-         //    dst cols: a, b, (id), c, d, e, f, ...
-         //
-         // Case 2) File id col index == output
-         //    src cols: a, b, (id), c, d, e, f, ...
-         //    dst cols: a, b, (id), c, d, e, f, ...
-         //
-         // Case 3) File id col index < output
-         //    src cols: a, b, (id), c, d, e, f, ...
-         //    dst cols: a, b, c, d, (id), e, f, ...
-
-         for (int i = 0; i < numColumns; i++) {
-            if (i == idColIdx) {
-               ReadColumn(idColIdxInFile, idColIdx);
-            } else if (i > idColIdx && i < idColIdxInFile) {
-               ReadColumn(i - 1, i);
-            } else if (i > idColIdxInFile && i < idColIdx) {
-               ReadColumn(i, i - 1);
-            } else {
-               ReadColumn(i, i);
-            }
-         }
-      }
-      return columnValues;
-
-      void ReadColumn(int srcIdx, int dstIdx) {
-         using var colReader = reader.GetColumnReader(srcIdx);
-         columnValues[dstIdx] = colReader.Read(startIndex, stopIndex);
-      }
-   }
-
-   IEnumerator IEnumerable.GetEnumerator() =>
-      GetEnumerator();
-
-   internal readonly struct DataCache {
-      public readonly IBlueRow[] Rows;
-      public readonly Dictionary<int, int> RowIdToIndex;
-      public readonly IList<object?>[] ColumnValues;
-
-      public DataCache(IBlueRow[] rows, Dictionary<int, int> rowIdToIndex, IList<object?>[] columnValues) {
-         Rows = rows;
-         RowIdToIndex = rowIdToIndex;
-         ColumnValues = columnValues;
-      }
-
-      public IBlueRow this[int id] =>
-         Rows[RowIdToIndex[id]];
-
-      public IBlueRow GetRow(int index) =>
-         Rows[index];
-   }
-}
@@ -1,65 +0,0 @@
-namespace Bts.DataStream.Parquet;
-
-using BCG;
-
-using bts;
-using bts.collections;
-
-using Bts.DataStreamCore;
-using Bts.PubSub;
-
-using Spark.Core.Disposables;
-
-using System;
-using System.Threading;
-
-[DataStreamUserRespresentation(false)]
-public class ParquetReaderStream : ViewStream {
-   readonly string file;
-   readonly int? start;
-   readonly int? stop;
-   readonly IWorkScheduler workScheduler;
-   IBlueRowCollection? table;
-   IDisposable? workerSub;
-
-   public ParquetReaderStream(string file, IWorkScheduler workScheduler) {
-      this.file = file;
-      this.workScheduler = workScheduler;
-   }
-
-   public ParquetReaderStream(string file, int? start, int? stop, IWorkScheduler workScheduler) {
-      this.file = file;
-      this.start = start;
-      this.stop = stop;
-      this.workScheduler = workScheduler;
-   }
-
-   protected override void myOnFirstSubscriber() {
-      Start();
-      var worker = new AnonymousWorker(DoWork);
-      workScheduler.Add(worker);
-      workerSub = Disposable.Create(() => workScheduler.Remove(worker));
-
-      async void Start() {
-         try {
-            Volatile.Write(ref table, await ParquetTableReader.ReadAsync(file, start, stop));
-         } catch (Exception ex) {
-            Log.DataFlow.error($"[parquet_reader_stream_err.json]{{ \"err\": \"{ex}\" }}");
-         }
-      }
-
-      void DoWork() {
-         var table = Volatile.Read(ref this.table);
-         if (table is null) {
-            return;
-         }
-         Disposable.dispose_thread_safe(ref workerSub);
-         if (table.schema.columnInfo.Length > 1) {
-            myFireEvent(new DataPacket(table, table, new BlueTableImmutable(table.schema)));
-         }
-      }
-   }
-
-   protected override void myOnLastSubscriber() =>
-      Disposable.dispose_thread_safe(ref workerSub);
-}
@@ -1,26 +0,0 @@
-// namespace Bts.DataStream.Parquet;
-
-// using BCG;
-
-// using global::Parquet.Data;
-
-// public class ParquetRow : IBlueRow {
-//    readonly int id;
-//    readonly int rowIndex;
-//    readonly DataColumn[] data;
-
-//    public ParquetRow(int id, int rowIndex, DataColumn[] data, IBlueTableSchema schema) {
-//       this.id = id;
-//       this.rowIndex = rowIndex;
-//       this.data = data;
-//       Schema = schema;
-//    }
-
-//    public object? this[int index] =>
-//       index == 0
-//          ? id
-//          : data[index - 1].Data.GetValue(rowIndex);
-
-//    public IBlueTableSchema Schema { get; }
-//    IBlueTableSchema IBlueRow.schema => Schema;
-// }
@@ -1,145 +0,0 @@
-namespace Bts.DataStream.Parquet;
-
-using BCG;
-
-using bts;
-using bts.collections;
-
-using Bts.DataStreamCore;
-
-using ParquetSharp;
-
-using Spark.Core.Collections;
-
-using System;
-using System.Collections;
-using System.Collections.Generic;
-using System.Linq;
-using System.Threading.Tasks;
-
-public class ParquetTable : IBlueRowCollection, IDisposable {
-   readonly SortedList<int, ParquetGroupTable> groups = new();
-   readonly int fullCount;
-   readonly IBlueTableSchema schema;
-   readonly ParquetFileReader? reader;
-   readonly int startIndex;
-   readonly int stopIndex;
-
-   public ParquetTable(ParquetFileReader reader, string tableName = "parquet", int? start = null, int? stop = null, bool leaveOpen = false) {
-      schema = reader.FileMetaData.ToBlueSchema(tableName);
-      int idColIdx;
-      if (schema.TryResolveIndices("id", out int idColIdxInFile)) {
-         idColIdx = idColIdxInFile;
-      } else {
-         var origCols = schema.columnInfo;
-         var cols = new ColumnInfo[origCols.Length + 1];
-         idColIdx = 0;
-         cols[0] = new("id", 0, typeof(int));
-         for (int i = 0; i < origCols.Length; i++) {
-            var c = origCols[i];
-            cols[i + 1] = new(c.colName, i + 1, c.colType);
-         }
-         schema = new BlueTableSchema(cols, schema.tableName);
-      }
-      fullCount = checked((int)reader.FileMetaData.NumRows);
-      Log.assert(fullCount >= 0);
-      startIndex = SliceList.GetStartIndex(start, fullCount, true);
-      stopIndex = SliceList.GetStopIndex(stop, fullCount, true);
-      stopIndex = Math.Max(stopIndex, startIndex);
-      Log.assert(startIndex >= 0 && startIndex <= fullCount && stopIndex >= 0 && stopIndex <= fullCount && stopIndex >= startIndex);
-      int numRowGroups = reader.FileMetaData.NumRowGroups;
-      int rowGroupLastIdx = 0;
-      object syncRoot = new();
-      for (int i = 0; i < numRowGroups && rowGroupLastIdx < Count; i++) {
-         int? grpStart = start.HasValue ? startIndex - rowGroupLastIdx : null;
-         int? grpStop = stop.HasValue ? stopIndex - rowGroupLastIdx : null;
-         int offset = rowGroupLastIdx;
-         var grp = new ParquetGroupTable(reader.RowGroup(i), schema, grpStart, grpStop, idColIdx, idColIdxInFile, GetAutoId);
-         if (grp.Count == 0) {
-            continue;
-         }
-         rowGroupLastIdx += grp.Count;
-         groups.Add(rowGroupLastIdx, grp);
-
-         int GetAutoId(int index) =>
-            index + offset;
-      }
-      Parallel.For(0, groups.Count, i => groups.GetValueAtIndex(i).EnsureInit());
-      if (!leaveOpen) {
-         this.reader = reader;
-      }
-   }
-
-   ~ParquetTable() =>
-      Dispose();
-
-   public IBlueTableSchema Schema => schema;
-
-   public int Count => stopIndex - startIndex;
-
-   IBlueTableSchema IBlueRowCollection.schema => schema;
-
-   public IBlueRow this[int index] {
-      get {
-         foreach (var grp in groups.Values) {
-            if (grp.TryGetValue(index, out var r)) {
-               return r;
-            }
-         }
-         throw new KeyNotFoundException();
-      }
-   }
-
-   public static ParquetTable FromFile(string path, int? start = null, int? stop = null, string tableName = "parquet") =>
-      new(new(path), tableName, start, stop, false);
-
-   public IBlueRow GetRow(int index) {
-      if (index < 0 || index >= Count) {
-         throw new IndexOutOfRangeException();
-      }
-      int rowGroupIdx = groups.Keys.BinarySearch(index);
-      if (rowGroupIdx < 0) {
-         rowGroupIdx = ~rowGroupIdx;
-      }
-      Log.assert(rowGroupIdx <= groups.Count);
-      var grp = groups[rowGroupIdx];
-      int idxWithinGrp = index;
-      if (rowGroupIdx > 0) {
-         idxWithinGrp -= groups.Keys[rowGroupIdx - 1];
-      }
-      return grp[idxWithinGrp];
-   }
-
-   public bool containsID(int id) =>
-      groups.Values.Any(rgl => rgl.containsID(id));
-
-   public int IndexOf(IBlueRow item) {
-      int idx;
-      int grpFirstIdx = 0;
-      foreach (var (grpLastIdx, grp) in groups) {
-         idx = grp.IndexOf(item);
-         if (idx >= 0) {
-            return idx + grpFirstIdx;
-         }
-         grpFirstIdx = grpLastIdx;
-      }
-      return -1;
-   }
-
-   public IEnumerator<IBlueRow> GetEnumerator() {
-      foreach (var rgl in groups.Values) {
-         foreach (var r in rgl) {
-            yield return r;
-         }
-      }
-   }
-
-   public void Dispose() {
-      GC.SuppressFinalize(this);
-      groups.Values.DisposeAll();
-      reader?.Dispose();
-   }
-
-   IEnumerator IEnumerable.GetEnumerator() =>
-      GetEnumerator();
-}
@@ -1,32 +0,0 @@
-#nullable enable
-namespace Bts.DataStream.Parquet;
-
-using BCG;
-
-using Bts.DataStreamCore;
-
-using ParquetSharp;
-
-using System.IO;
-using System.Threading.Tasks;
-
-public class ParquetTableReader {
-   public static Task<IBlueRowCollection> ReadAsync(Stream stream, bool leaveOpen = false, int? start = null, int? stop = null) =>
-      Task.Run(() => Read(stream, leaveOpen, start, stop));
-
-   public static Task<IBlueRowCollection> ReadAsync(string path, int? start = null, int? stop = null) =>
-      Task.Run(() => Read(path, start, stop));
-
-   public static IBlueRowCollection Read(Stream stream, bool leaveOpen = false, int? start = null, int? stop = null) {
-      using var fileReader = new ParquetFileReader(stream, leaveOpen);
-      var rv = fileReader.ReadBlueTable(start, stop);
-      rv.schema.Validate();
-      return rv;
-   }
-
-   public static IBlueRowCollection Read(string path, int? start = null, int? stop = null) {
-      IBlueRowCollection rv = ParquetTable.FromFile(path, start, stop);
-      rv.schema.Validate();
-      return rv;
-   }
-}
@@ -1,38 +0,0 @@
-namespace Bts.DataStream.Parquet;
-
-using BCG;
-
-using Bts.DataStreamCore;
-
-using bts;
-
-using ParquetSharp;
-
-using System.IO;
-using System.Threading.Tasks;
-
-public static class ParquetTableWriter {
-   public static Task WriteAsync(IBlueRowCollection blueRows, Stream outStream, bool leaveOpen = false) =>
-      Task.Run(() => Write(blueRows, outStream, leaveOpen));
-
-   public static void Write(IBlueRowCollection blueRows, Stream outStream, bool leaveOpen = false) {
-      var (parquetColumns, parquetMeta) = blueRows.schema.ToParquetSchema();
-      using var schema = Column.CreateSchemaNode(parquetColumns);
-      using var writerProps = createWriterProperties();
-      using var writer = new ParquetFileWriter(outStream, schema, writerProps, keyValueMetadata: parquetMeta, leaveOpen: leaveOpen);
-      using var rowGroup = writer.AppendRowGroup();
-      for (int i = 0; i < blueRows.schema.columnInfo.Length; i++) {
-         var col = blueRows.schema.columnInfo[i];
-         Log.assert(col.colNo == i);
-         using var colWriter = ParquetAdapter.GetColumnWriter(col.type_code(), rowGroup.NextColumn());
-         colWriter.Write(blueRows.GetColumn(i));
-      }
-      writer.Close();
-   }
-
-   static WriterProperties createWriterProperties(Compression compression = Compression.Snappy) {
-      using var builder = new WriterPropertiesBuilder();
-      builder.Compression(compression);
-      return builder.Build();
-   }
-}
@@ -1,87 +0,0 @@
-namespace Bts.DataStream.Parquet;
-
-using BCG;
-
-using bts.collections;
-
-using Bts.DataStreamCore;
-using Bts.PubSub;
-
-using mbts.utils;
-
-using Spark.Core.Disposables;
-using Spark.Core.Threading;
-
-using System;
-using System.IO;
-
-[DataStreamUserRespresentation(false)]
-public class ParquetWriterStream : ViewStream {
-   readonly string file;
-   readonly ViewStream inputStream;
-   readonly IWorkScheduler workScheduler;
-   readonly AtomicGate gate = new();
-   IDataPacket? pendingOutput;
-   IDisposable? dispose;
-
-   public ParquetWriterStream(ViewStream inputStream, string file, IWorkScheduler workScheduler) {
-      this.inputStream = inputStream;
-      this.file = file;
-      this.workScheduler = workScheduler;
-   }
-
-   protected override void myOnFirstSubscriber() {
-      BlueTableImmutable.DataPacketBuilder? outputBuilder = null;
-      bool isFirstUpdate = false;
-      var worker = new AnonymousWorker(DoWork);
-      inputStream.onChange += OnInputStreamChange;
-      workScheduler.Add(worker);
-      dispose = Disposable.Create(Dispose);
-
-      void Dispose() {
-         workScheduler.Remove(worker);
-         inputStream.onChange -= OnInputStreamChange;
-      }
-
-      void OnInputStreamChange(object? dpObj) {
-         var dp = ExceptionAssert.Argument.Is<IDataPacket>(dpObj);
-         using (gate.BeginTransaction()) {
-            outputBuilder ??= new BlueTableImmutable(dp.Schema).ToDataPacketBuilder();
-            foreach (IBlueRow r in dp.addedRows) {
-               outputBuilder.AddOrUpdate(r);
-            }
-            foreach (IBlueRow r in dp.deletedRows) {
-               outputBuilder.Remove(r.Id());
-            }
-         }
-      }
-
-      async void DoWork() {
-         using (gate.BeginTransaction()) {
-            if (outputBuilder is null) {
-               return;
-            }
-            if (pendingOutput is not null) {
-               myFireEvent(pendingOutput);
-               pendingOutput = null;
-            }
-            var dp = outputBuilder.ToDataPacket();
-            if (isFirstUpdate) {
-               isFirstUpdate = false;
-            } else if (!dp.HasChanges()) {
-               return;
-            }
-            {
-               using var outStream = File.OpenWrite(file);
-               await ParquetTableWriter.WriteAsync(dp.newRows, outStream);
-            }
-            // Fire the update at the beginning of the next doWork call so it happens on the
-            // 'work' thread.
-            pendingOutput = dp;
-         }
-      }
-   }
-
-   protected override void myOnLastSubscriber() =>
-      Disposable.dispose_thread_safe(ref dispose);
-}
@@ -1,188 +0,0 @@
-namespace bts.BlueOrb {
-   using BCG;
-   using System;
-   using System.Collections;
-   using System.Collections.Generic;
-   using System.Linq;
-
-   #region mappings
-   internal class ProjectData {
-      // schema from the base stream
-      public readonly IBlueTableSchema in_schema;
-      // this is the projected schema
-      public readonly IBlueTableSchema out_schema;
-      // projected cols to the input cols, for cols who's names are changed
-      public readonly IDictionary<string, string> mapped_to_orig_col_map = new Dictionary<string, string>();
-      public ProjectData(IBlueTableSchema in_schema_, IBlueTableSchema out_schema_,
-                  IDictionary<string, string> mapped_to_orig_col_map_) {
-         in_schema = in_schema_;
-         out_schema = out_schema_;
-         mapped_to_orig_col_map = mapped_to_orig_col_map_;
-      }
-   }
-   #endregion
-
-
-#region Row with mappings build in
-   internal class NewProjectRow : IBlueRow {
-      #region IBlueRow members
-      private ProjectData _project_data;
-      private IBlueRow _row;
-      public NewProjectRow(ProjectData project_data, IBlueRow row) {
-         _project_data = project_data;
-         _row = row;
-      }
-      public IBlueTableSchema schema { get { return _project_data.out_schema; } }
-      #endregion
-
-      #region ICollection members
-      public int Count { get { return schema.columnNames.Length; } }
-
-      public bool IsSynchronized { get { return false; } }
-
-      public object SyncRoot { get { return this; } }
-
-      public void CopyTo(Array array, int index) {
-         int i = index;
-         foreach (object o in this as IEnumerable)
-            array.SetValue(o, i++);
-      }
-      #endregion
-
-      #region IDictionary members
-      public bool IsFixedSize { get { return true; } }
-
-      public bool IsReadOnly { get { return true; } }
-
-      public object this[object key] {
-         get {
-            if (key is string)
-               return this[(string)key];
-            return this[(int)key];
-         }
-         set { throw new NotSupportedException("The IBlueRow is readonly"); }
-      }
-
-      public object this[int index] {
-         get {
-            return this[schema.columnInfo[index].colName];
-         }
-      }
-
-      public object this[string col_name] {
-         get {
-            string mapped_to_col = null;
-            if (_project_data.mapped_to_orig_col_map.TryGetValue(col_name, out mapped_to_col)) {
-               return _row[mapped_to_col];
-            }
-            return _row[col_name];
-         }
-      }
-
-      public ICollection Keys { get { return schema.columnNames; } }
-
-      public ICollection Values {
-         get {
-            object[] retVal = new object[schema.columnNames.Length];
-            CopyTo(retVal, 0);
-            return retVal;
-         }
-      }
-
-      public void Add(object key, object value) { throw new NotSupportedException("The IBlueRow is readonly"); }
-
-      public void Clear() { throw new NotSupportedException("The IBlueRow is readonly"); }
-
-      public bool Contains(object key) { return schema[(string)key] != null; }
-
-      /// <summary>
-      /// Note, the default enumerator is the IDictionary's enumerator, so if user needs to
-      /// iterate through the values, typecast this into an IEnumerable
-      /// </summary>
-      /// <returns></returns>
-      public IDictionaryEnumerator GetEnumerator() { return new BlueRowDictionaryEnumerator(this); }
-
-      IEnumerator IEnumerable.GetEnumerator() { return new BlueRowEnumerator(this); }
-
-      public void Remove(object key) { throw new NotSupportedException("The IBlueRow is readonly"); }
-      #endregion
-   }
-   #endregion
-
-   [DataStreamUserRespresentation("ProjectStream")]
-   public class NewProjectStream : ViewStream, IStreamMapper {
-
-      private ViewStream _stream;
-      private string _project_expression;
-      private ProjectData _project_data = null;
-      //public ProjectData 
-      public NewProjectStream(ViewStream vs, string project_expression) {
-         _stream = vs;
-         _project_expression = project_expression;
-         _stream = new MapStream(vs, this);
-         mySetRealSource(_stream);
-      }
-
-      public IBlueTableSchema getOutputSchema(IBlueTableSchema input_schema) {
-         _project_data = _create_project_data(input_schema, _project_expression);
-         return _project_data.out_schema;
-      }
-
-      public IBlueRow map(IBlueRow row) {
-         return new NewProjectRow(_project_data, row);
-      }
-
-      // this is where most of the logic is, in terms of translating cols
-      private static ProjectData _create_project_data(IBlueTableSchema input_schema,
-                                                      string project_expression) {
-         project_expression = project_expression.Trim();
-         string[] project_toks = project_expression.Split(' ', ',');
-         HashSet<string> removed_cols = new HashSet<string>();
-         Dictionary<string, string> orig_to_mapped = new Dictionary<string, string>();      
-         // by default every col will show up, so if we have the projection as (a:a1)
-         // it will convert a to a1, 
-         // modifications (6/30/2025):
-         // only the columns represented here, either mapped
-         // or not, will show
-         // examples: if a schema has cols a b c
-         // project(parent_str, "a:aa b") -> aa b
-         foreach (string o_p_tok in project_toks) {
-            string p_tok = o_p_tok.Trim();
-            if (p_tok.Contains(":")) {
-               // explicit mapping
-               string[] map_toks = p_tok.Split(":");
-               string orig_col = map_toks[0];
-               string mapped_col = map_toks[1];
-               orig_to_mapped.Add(orig_col, mapped_col);  
-            } else {
-               // this is a col_name explicitely stated
-               if (input_schema.columnNames.Contains(p_tok)) {
-                  string orig_col = p_tok;
-                  string mapped_col = orig_col;
-                  orig_to_mapped.Add(orig_col, mapped_col);
-               }
-            }
-         }
-
-         // make the new schema
-         List<ColumnInfo> new_cols = new List<ColumnInfo>();
-         foreach (ColumnInfo ci in input_schema.columnInfo) {
-            if (ci.colName == "id") {
-               ColumnInfo nc = new ColumnInfo(ci.colName, new_cols.Count, ci.colType);
-               new_cols.Add(nc);
-               continue;           
-            } else if (orig_to_mapped.ContainsKey(ci.colName)) {
-               string mapped_name = orig_to_mapped[ci.colName];
-               ColumnInfo nc = new ColumnInfo(mapped_name, new_cols.Count, ci.colType);
-               new_cols.Add(nc);
-            }
-         }
-         IBlueTableSchema new_schema = new BlueTableSchema(input_schema.tableName, new_cols.ToArray());
-         var mapped_to_orig_map = orig_to_mapped.ToDictionary(kv => kv.Value, kv => kv.Key);
-         return new ProjectData(input_schema, new_schema, mapped_to_orig_map);
-      }
-
-   
-
-   }
-}
@@ -1,128 +0,0 @@
-namespace bts.BlueOrb {
-   using BCG;
-   using System.IO;
-   using System.Diagnostics;
-   using System;
-
-   [DataStreamUserRespresentation("PythonDataframeStream")]
-   public class PythonDataframeStream : ViewStream {
-      private ViewStream _parent_str;
-      private readonly string _in_csv_file_name;
-      private readonly string _out_csv_filename;
-      private readonly string _transform_function;
-      public PythonDataframeStream(ViewStream parent_str, string transform_function) {
-         var guid = System.Guid.NewGuid();
-         _in_csv_file_name = $"pdfs_in_{guid}.csv";
-         _out_csv_filename = $"pdfs_out_{guid}.csv";
-         _parent_str = new CSVSaverStream(parent_str, _in_csv_file_name);
-         _transform_function = transform_function;
-      }
-
-      protected override void myOnFirstSubscriber() {
-         _parent_str.onChange += _parent_str_onchange;
-      }
-
-      protected override void myOnLastSubscriber() {
-         _parent_str.onChange -= _parent_str_onchange;
-      }
-
-      private void _parent_str_onchange(object dp_obj) {
-         Log.assert(dp_obj != null);
-         // the file _in_csv_file_name is now populated.
-         // call py_func_wrapper_main and pass this file name as well
-         // as the output file name
-         // py_func_wrapper_main reads _in_csv_file_name into the df and calls 
-         // transform function, which returns the tranformed df. py_func_wrapper_main then
-         // writes it into the _out_csv_filename file.
-         // now this function should read _out_csv_filename into a dp
-         // and fire it
-         _compute_and_write();
-         //ViewStream rs = new SchemaGuesserStream(new CSVReaderStream(_out_csv_filename));
-         ViewStream rs = new CSVReaderStream(_out_csv_filename);
-         IBlueRowCollection nrows = null;
-         rs.onChange += (dp_obj) => {
-            nrows = (dp_obj as DataPacket).newRows;
-         };
-         var dp = BlueDataUtility.build_dp(nrows, myLastValue as DataPacket);
-         myFireEvent(dp);
-      }
-
-      private void _compute_and_write() {
-         File.WriteAllText("user_code.py", _transform_function);
-         // wrapper function
-         string py_func_wrapper_main = @"
-import sys
-from user_code import transform
-import pandas as pd
-
-if __name__ == '__main__':
-   input_file_name = sys.argv[1]
-   output_file_name = sys.argv[2]
-# 1. load input_file_name into df
-# 2. do some test calc
-# 3. write the output of 2 into output_file_name
-   df = pd.read_csv(input_file_name)   
-   df_out = transform(df)
-   df_out.to_csv(output_file_name, index=False)";
-
-
-         File.WriteAllText("runner.py", py_func_wrapper_main);
-
-         ProcessStartInfo psi = new ProcessStartInfo {
-            FileName = "python",
-            Arguments = $"runner.py {_in_csv_file_name} {_out_csv_filename}",
-            RedirectStandardOutput = false,
-            UseShellExecute = false
-         };
-         var proc = Process.Start(psi);
-         proc.WaitForExit();
-
-      }
-
-      #region Unused test code
-      private void _simple_test() {
-
-         // actual function that needs to be called ////////////////////////
-         string py_real_function = @"
-def test(x, out_path):
-   rv = x * 10
-   with open(out_path, 'w') as f:
-      f.write(str(rv))
-   return rv";
-         // the function is in a string object.
-         File.WriteAllText("user_code.py", py_real_function);
-
-
-         // runner app (top level main function) //////////////////////////////
-         string py_func_wrapper_main = @"
-import sys
-from user_code import test
-
-if __name__ == '__main__':
-   arg = int(sys.argv[1])
-   out_path = sys.argv[2]
-   rv = test(arg, out_path)
-   rv = rv + 12
-   with open(out_path, 'w') as f:to \
-      f.write(str(rv))";
-   
-         File.WriteAllText("runner.py", py_func_wrapper_main);
-
-         string outputPath = "py_out.txt";
-         ProcessStartInfo psi = new ProcessStartInfo {
-            FileName = "python",
-            Arguments = $"runner.py 4 {outputPath}",
-            RedirectStandardOutput = false,
-            UseShellExecute = false
-         };
-         var proc = Process.Start(psi);
-         proc.WaitForExit();
-
-         // Now read the output written by the function
-         string result = File.ReadAllText(outputPath);
-         Console.WriteLine(result); // Should print "6"
-
-      }
-      #endregion
-   }
-}
@@ -1,30 +0,0 @@
-using BCG;
-
-namespace bts.BlueOrb {
-   public class SchemaGuesserStream : ViewStream {
-
-      private IBlueRowCollection _rows = null;
-      private ViewStream _vs;
-      public SchemaGuesserStream(ViewStream vs) {
-         _vs = vs;
-      }
-
-      protected override void myOnFirstSubscriber() {
-         _vs.onChange += _on_data;
-      }
-      protected override void myOnLastSubscriber() {
-         _vs.onChange -= _on_data;
-      }
-
-      private void _on_data(object obj) {
-         DataPacket dp = obj as DataPacket;
-         if (_rows == null) {
-            _rows = BlueDataUtility.get_typed_table(dp.newRows);
-         }
-         var dp_out = BlueDataUtility.build_dp_inc(_rows, myLastValue as DataPacket);
-         myFireEvent(dp_out);
-
-      }
-
-   }
-}
@@ -1,67 +0,0 @@
-using Bts.ArithmeticExpressions;
-
-using System;
-using System.Collections.Generic;
-using System.Linq;
-using System.Text;
-using BCG;
-
-namespace bts.BlueOrb {
-
-   [DataStreamUserRespresentation("SingleRowMathStream")]
-   public class MathStream : ViewStream, IStreamMapper {
-      private IArithmeticProgram myExp;
-      private BooleanExpressionFilter _bool;
-      List<string> myExtraColumns = new List<string>();
-      BlueTableSchema mySchema;
-
-
-      public MathStream(ViewStream vs, string mathExpression)
-         : this(vs, mathExpression, string.Empty) {
-      }
-
-      public MathStream(ViewStream vs, string mathExpression, string boolean_expression) {
-         myExp = ArithmeticProgramParser.Parse(mathExpression) ??
-            throw new Exception($"Failed to parse ArithmeticExpression '{mathExpression}'");
-         myExtraColumns = myExp.OutputIdentifiers.ToList();
-         if (!string.IsNullOrEmpty(boolean_expression))
-            _bool = new BooleanExpressionFilter(boolean_expression);
-         mySetRealSource(new MapStream(vs, this));
-      }
-
-      #region IStreamMapper Members
-
-      public IBlueTableSchema getOutputSchema(IBlueTableSchema inputSchema) {
-         if (mySchema == null) {
-            Dictionary<string, ColumnInfo> colIdx = new Dictionary<string, ColumnInfo>();
-            foreach (ColumnInfo ci in inputSchema.columnInfo)
-               colIdx[ci.colName] = ci;
-
-            foreach (string ncn in myExtraColumns) {
-               ColumnInfo newC;
-               if (colIdx.ContainsKey(ncn))
-                  newC = new ColumnInfo(colIdx[ncn].colName, colIdx[ncn].colNo, typeof(double));
-               else
-                  newC = new ColumnInfo(ncn, colIdx.Count, typeof(double));
-               colIdx[ncn] = newC;
-            }
-            mySchema = new BlueTableSchema(colIdx.Values.ToArray(), "math(" + inputSchema.tableName + ")");
-         }
-         return mySchema;
-      }
-
-      #endregion
-
-      #region IDynamicStreamMapper Members
-
-      public IBlueRow map(IBlueRow row) {
-         MutableBlueRow result = new MutableBlueRow(mySchema, row);
-         if (_bool == null || _bool.filter(row)) {
-            myExp.Evaluate(result.AsDictionary());
-         }
-         return result;
-      }
-
-      #endregion
-   }
-}
@@ -1,17 +0,0 @@
-namespace bts.BlueOrb {
-   using BCG;
-
-   using System;
-   using System.Collections.Generic;
-   using System.Linq;
-   using System.Text;
-   using System.Threading.Tasks;
-
-   [DataStreamUserRespresentation("SortStream")]
-   public class SortStream: ViewStream {
-      public SortStream (ViewStream parent_stream, string sort_expr) {
-         ViewStream rs = new BCG.SortStream(parent_stream, sort_expr);
-         mySetRealSource(rs);
-      }
-   }
-}
@@ -1,16 +0,0 @@
-namespace bts.BlueOrb {
-   using BCG;
-
-   using System;
-   using System.Collections.Generic;
-   using System.Linq;
-   using System.Text;
-   using System.Threading.Tasks;
-   [DataStreamUserRespresentation("TimeBasedRollingFilterStream")]
-   public class TimeBasedRollingFilterStream : ViewStream {
-      public TimeBasedRollingFilterStream(ViewStream parent_str, string time_col, double n_seconds) {
-         ViewStream rs = new BCG.TimeBasedFilterStream(ViewStreamDatabase.instance, parent_str, time_col, TimeSpan.FromSeconds(n_seconds));
-         mySetRealSource(rs);
-      }
-   }
-}
@@ -1,81 +0,0 @@
-using BCG;
-
-using bts.utils;
-
-using Bts.DataStreamCore;
-
-using mbts.utils;
-
-using Spark.Core.Collections;
-using Spark.Core.Threading;
-
-using System;
-using System.Collections.Generic;
-using System.Collections.Immutable;
-using System.Linq;
-
-namespace bts.BlueOrb {
-   // Keeps adding changing rows to the stream, hence preserving the history
-   // and creating a time series, for example this can be used to make bbo updates
-   // into a time series of prices/qty
-   [DataStreamUserRespresentation("TimeSeriesStream")]
-   public class TimeSeriesStream : ViewStream {
-      readonly ViewStream _stream;
-      readonly string timeColumn;
-      readonly AtomicGate gate = new();
-      CircularBufferTable? _table;
-      private int _id = 0;
-      private int _size = 0;
-
-      public TimeSeriesStream(ViewStream stream)
-         : this(stream, Int32.MaxValue) { }
-
-      public TimeSeriesStream(ViewStream stream, int size, string timeColumn = "ts_event") {
-         _stream = stream;
-         _size = size;
-         this.timeColumn = timeColumn;
-      }
-
-      protected override void myOnFirstSubscriber() {
-         _stream.onChange += new ChangeDelegate(_onstream_change);
-      }
-      protected override void myOnLastSubscriber() {
-         _stream.onChange -= new ChangeDelegate(_onstream_change);
-      }
-
-
-      private void _onstream_change(object o) {
-         var dp = ExceptionAssert.Argument.Is<IDataPacket>(o);
-         DateTime now = DateTime.Now;
-         using var _ = gate.BeginTransaction();
-         if (_table == null) {
-            var colInfos = dp.newRows.schema.columnInfo;
-            if (colInfos.TryGetFirst(c => c.colName == timeColumn, out var currTimeCol, out var currTimeColIdx)) {
-               colInfos = colInfos.ToArray();
-               colInfos[currTimeColIdx] = new(timeColumn, currTimeCol.colNo, typeof(DateTime));
-            } else {
-               colInfos = ArrayHelper.add(colInfos, new(timeColumn, colInfos.Length, typeof(DateTime)));
-            }
-            _table = new CircularBufferTable(new BlueTableSchema(colInfos, dp.newRows.schema.tableName), _size);
-         }
-         var addedRowsBd = new BlueTableImmutable(_table.schema).ToDataPacketBuilder();
-         var deletedRowsBd = new BlueTableImmutable(_table.schema).ToDataPacketBuilder();
-         foreach (IBlueRow r in dp.addedRows) {
-            MutableBlueRow ar = new MutableBlueRow(_table.schema, r);
-            ar.setValue(timeColumn, now);
-            int id = _id++;
-            ar.setValue("id", id);
-            _table = _table.Add(ar, out var oldRow);
-            addedRowsBd.AddOrUpdate(ar);
-            if (oldRow is not null) {
-               deletedRowsBd.AddOrUpdate(oldRow);
-            }
-         }
-         var addedRows = addedRowsBd.ToImmutable();
-         var deletedRows = deletedRowsBd.ToImmutable();
-         var dp_out = new DataPacket(_table, addedRows, deletedRows);
-         myFireEvent(dp_out);
-      }
-   }
-}
-
@@ -1,197 +0,0 @@
-namespace Bts.DataStream;
-
-using bts;
-
-using Spark.Core.Collections;
-
-using System;
-using System.Collections.Generic;
-using System.Collections.Immutable;
-
-/// <summary>
-/// Maps a timeseries of keys to unique int32 ids.
-/// </summary>
-public class TimeseriesIndex<T> where T : notnull {
-   static readonly IComparer<KeyValuePair<DateTime, int>> idTimeseriesComp =
-      KeyValuePairComparer.KeyComparer<DateTime, int>();
-
-   readonly ImmutableDictionary<T, ImmutableList<KeyValuePair<DateTime, int>>> nameToId;
-   readonly ImmutableDictionary<int, TimeseriesKey<T>> idToName;
-   readonly int idCounter;
-
-   public TimeseriesIndex()
-      : this(ImmutableDictionary<T, ImmutableList<KeyValuePair<DateTime, int>>>.Empty, ImmutableDictionary<int, TimeseriesKey<T>>.Empty, 0) { }
-
-   TimeseriesIndex(ImmutableDictionary<T, ImmutableList<KeyValuePair<DateTime, int>>> nameToId, ImmutableDictionary<int, TimeseriesKey<T>> idToName, int idCounter) {
-      this.nameToId = nameToId;
-      this.idToName = idToName;
-      this.idCounter = idCounter;
-   }
-
-   public int Count => idToName.Count;
-
-   public ImmutableDictionary<T, ImmutableList<KeyValuePair<DateTime, int>>> NameToId => nameToId;
-   public ImmutableDictionary<int, TimeseriesKey<T>> IdToName => idToName;
-
-   public bool ContainsId(int id) =>
-      idToName.ContainsKey(id);
-
-   public int GetNearest(TimeseriesKey<T> key) {
-      if (!nameToId.TryGetValue(key.Name, out var ids) || ids.Count == 0) {
-         throw new KeyNotFoundException();
-      }
-      int idx = ids.BinarySearch(new(key.Timestamp, -1), idTimeseriesComp);
-      if (idx < 0) {
-         idx = ~idx;
-         if (idx == 0) {
-            throw new KeyNotFoundException();
-         }
-         idx--;
-      }
-      return ids[idx].Value;
-   }
-
-   public bool TryGetNearest(TimeseriesKey<T> key, out int result) {
-      if (!nameToId.TryGetValue(key.Name, out var ids) || ids.Count == 0) {
-         result = -1;
-         return false;
-      }
-      int idx = ids.BinarySearch(new(key.Timestamp, -1), idTimeseriesComp);
-      if (idx < 0) {
-         idx = ~idx;
-         if (idx == 0) {
-            result = -1;
-            return false;
-         }
-         idx--;
-      }
-      result = ids[idx].Value;
-      return true;
-   }
-
-   public TimeseriesIndex<T> GetNearestOrAdd(TimeseriesKey<T> key, out int result) {
-      int idx;
-      if (!nameToId.TryGetValue(key.Name, out var ids) || ids.Count == 0) {
-         ids = ImmutableList<KeyValuePair<DateTime, int>>.Empty;
-         idx = 0;
-      } else {
-         idx = ids.BinarySearch(new(key.Timestamp, -1), idTimeseriesComp);
-         if (idx < 0) {
-            idx = ~idx;
-         }
-      }
-      if (idx == 0) {
-         result = idCounter + 1;
-         ids = ids.Insert(0, new(key.Timestamp, result));
-         return new(
-            nameToId.SetItem(key.Name, ids),
-            idToName.SetItem(result, key),
-            result);
-      }
-      result = ids[idx].Value;
-      return this;
-   }
-
-   public TimeseriesIndex<T> GetOrAdd(TimeseriesKey<T> key, out int result) {
-      int idx;
-      if (!nameToId.TryGetValue(key.Name, out var ids) || ids.Count == 0) {
-         ids = ImmutableList<KeyValuePair<DateTime, int>>.Empty;
-         idx = 0;
-      } else {
-         idx = ids.BinarySearch(new(key.Timestamp, -1), idTimeseriesComp);
-         if (idx < 0) {
-            idx = ~idx;
-         } else {
-            result = ids[idx].Value;
-            return this;
-         }
-      }
-      result = idCounter + 1;
-      ids = ids.Insert(idx, new(key.Timestamp, result));
-      return new(
-         nameToId.SetItem(key.Name, ids),
-         idToName.SetItem(result, key),
-         result);
-   }
-
-   public IList<KeyValuePair<DateTime, int>> GetIds(T name) =>
-      nameToId[name];
-
-   public Builder ToBuilder() =>
-      new(this);
-
-   public class Builder {
-      readonly ImmutableDictionary<T, ImmutableList<KeyValuePair<DateTime, int>>>.Builder nameToId;
-      readonly ImmutableDictionary<int, TimeseriesKey<T>>.Builder idToName;
-      int idCounter;
-
-      public Builder(TimeseriesIndex<T> source)
-         : this(source.nameToId, source.idToName, source.idCounter) { }
-
-      public Builder(ImmutableDictionary<T, ImmutableList<KeyValuePair<DateTime, int>>> nameToId, ImmutableDictionary<int, TimeseriesKey<T>> idToName, int idCounter) {
-         this.nameToId = nameToId.ToBuilder();
-         this.idToName = idToName.ToBuilder();
-         this.idCounter = idCounter;
-      }
-
-      public int GetNearest(TimeseriesKey<T> key) =>
-         getOrAdd(key, false, false);
-
-      public int GetNearestOrAdd(TimeseriesKey<T> key) =>
-         getOrAdd(key, false, true);
-
-      public int GetOrAdd(TimeseriesKey<T> key) =>
-         getOrAdd(key, true, true);
-
-      public int GetLatest(T key) =>
-         nameToId.TryGetValue(key, out var ids)
-            ? ids[^1].Value
-            : throw new KeyNotFoundException();
-
-      public bool TryGetLatest(T key, out int result) {
-         if (nameToId.TryGetValue(key, out var ids)) {
-            result = ids[^1].Value;
-            return true;
-         }
-         result = 0;
-         return false;
-      }
-
-      public ImmutableList<KeyValuePair<DateTime, int>> GetIds(T name) =>
-         nameToId.TryGetValue(name, out var ids)
-            ? ids
-            : ImmutableList<KeyValuePair<DateTime, int>>.Empty;
-
-      public TimeseriesIndex<T> ToImmutable() =>
-         new(nameToId.ToImmutable(), idToName.ToImmutable(), idCounter);
-
-      int getOrAdd(TimeseriesKey<T> key, bool requireExactTimestamp, bool canAdd) {
-         var ids = nameToId.GetOrAdd(key.Name, _ => ImmutableList<KeyValuePair<DateTime, int>>.Empty);
-         int idx = ids.BinarySearch(new(key.Timestamp, 0), idTimeseriesComp);
-         int id;
-         if (idx < 0) {
-            Log.Gen.info($"Adding key to timeseries index ({key.Timestamp}, {key.Name})");
-            // We didn't get an exact match. If requireExactTimestamp is false and a record with an
-            // earlier Timestamp exists, use that. Otherwise, add a new record with this Timestamp.
-            idx = ~idx;
-            if (requireExactTimestamp || idx == 0) {
-               if (!canAdd) {
-                  throw new KeyNotFoundException();
-               }
-               id = ++idCounter;
-               ids = ids.Insert(idx, new(key.Timestamp, id));
-               nameToId[key.Name] = ids;
-               idToName.Add(id, key);
-            } else {
-               if (idx == ids.Count) {
-                  --idx;
-               }
-               id = ids[idx].Value;
-            }
-         } else {
-            id = ids[idx].Value;
-         }
-         return id;
-      }
-   }
-}
@@ -1,22 +0,0 @@
-namespace Bts.DataStream;
-
-using Spark.Core.Collections;
-
-using System;
-using System.Collections.Generic;
-
-public readonly record struct TimeseriesKey<T>(T Name, DateTime Timestamp) : IDualComparable<TimeseriesKey<T>> {
-   public int CompareTo(TimeseriesKey<T> other) {
-      var rv = Timestamp.CompareTo(other.Timestamp);
-      if (rv == 0) {
-         rv = Comparer<T>.Default.Compare(Name, other.Name);
-      }
-      return rv;
-   }
-
-   public static implicit operator (T Name, DateTime Timestamp)(TimeseriesKey<T> key) =>
-      (key.Name, key.Timestamp);
-
-   public static implicit operator TimeseriesKey<T>((T Name, DateTime Timestamp) tup) =>
-      new(tup.Name, tup.Timestamp);
-}
@@ -1,11 +0,0 @@
-namespace bts.BlueOrb {
-   using BCG;
-   [DataStreamUserRespresentation("TopStream")]
-   public class TopStream : ViewStream { 
-      public TopStream(ViewStream vs, int n_rows, string sort_col) {
-         ViewStream rs = new BCG.TopStream (vs, n_rows, sort_col);
-         mySetRealSource(rs);
-      }
-   }
-
-}
@@ -1,16 +0,0 @@
-namespace bts.BlueOrb {
-   using System;
-   using System.Collections.Generic;
-
-   using mbts;
-   using BCG;
-
-   [DataStreamUserRespresentation("UnionStream")]
-   public class UnionStream : ViewStream {
-      public UnionStream(ViewStream left_stream, ViewStream right_stream) {
-         ViewStream rs = new BCG.MutableAppendStream(left_stream, right_stream);
-         mySetRealSource(rs);
-      }
-   }
-
-}
@@ -1,7 +1,7 @@
 <Project Sdk="Microsoft.NET.Sdk">

  <PropertyGroup>
-    <TargetFramework>net8.0</TargetFramework>
+    <TargetFramework>net7.0</TargetFramework>
    <Nullable>enable</Nullable>
  </PropertyGroup>

@@ -17,13 +17,12 @@
  </ItemGroup>

  <ItemGroup>
-    <PackageReference Include="IronPython" Version="3.4.2" />
+    <PackageReference Include="HtmlAgilityPack" Version="1.11.57" />
    <PackageReference Include="Microsoft.Data.Analysis" Version="0.21.1" />
    <PackageReference Include="MongoDB.Bson" Version="2.23.1" />
    <PackageReference Include="MongoDB.Driver" Version="2.23.1" />
-    <PackageReference Include="ParquetSharp" Version="20.0.0" />
+    <PackageReference Include="Parquet.Net" Version="5.0.1" />
    <PackageReference Include="System.Data.SQLite" Version="1.0.118" />
-    <PackageReference Include="MathNet.Numerics" Version="5.0.0" />
  </ItemGroup>

 </Project>
@@ -2,277 +2,25 @@ namespace Bts.DataStream;

 using BCG;

-using bts;
-using bts.api.core;
-
-using Bts.DataStreamCore;
-using Bts.PubSub;
-
-using Microsoft.Extensions.Logging;
-
 using Spark.Core.Disposables;
-using Spark.Core.Threading;

 using System;
-using System.Linq;
 using System.Reactive.Linq;
-using System.Threading;
-using System.Threading.Tasks;

 public static class DataStreamExtensions {
-   public static ViewStream AsViewStream<T>(this IObservable<IDataPacket<T>> source, string? name = null, ILogger? logger = null) where T : IBlueRow =>
-      source as ViewStream ?? new ObservableViewStream(source.Select(dp => new DataPacket(dp.newRows, dp.addedRows, dp.deletedRows)), name, logger);
+   public static ViewStream AsViewStream(this IObservable<DataPacket> source) =>
+      source as ViewStream ?? new ObservableViewStream(source);

-   // public static IObservable<IDataPacket<T>> SelectDataPacket<T>(this IObservable<IDataPacket> source, Func<IBlueRow, T> selector) where T : IBlueRow =>
-   //    source.Select(dp => {
-   //       dp.get_new_added_deleted(out var newRows, out var addedRows, out var deletedRows);
-   //       return new DataPacket<T>(CastRows(newRows), CastRows(addedRows), CastRows(deletedRows));
+   public static IObservable<DataPacket> AsObservable(this ViewStream source) {
+      return source as IObservable<DataPacket> ?? Observable.Create<DataPacket>(Subscribe);

-   //       IBlueRowCollection<T> CastRows(ICollection rows) =>
-   //          new BlueTableSelectorWrapper<T>(
-   //             rows as IBlueRowCollection ?? new BlueRowCollection(rows, dp.Schema), selector);
-   //    });
+      IDisposable Subscribe(IObserver<DataPacket> observer) {
+         source.onChange += OnChange;
+         return Disposable.Create(() => source.onChange -= OnChange);

-   public static IObservable<TableUpdate<IBlueRow, int>> AsTables(this IObservable<IDataPacket> source) =>
-      source.AsTables<IBlueRow, int>(r => r, r => r.Id());
-
-   public static IObservable<TableUpdate<T, int>> AsTables<T>(this IObservable<IDataPacket> source, Func<IBlueRow, T> valueSelector) where T : IBlueRow =>
-      source.AsTables<T, int>(valueSelector, r => r.Id());
-
-   public static IObservable<TableUpdate<T, int>> AsTables<T>(
-      this IObservable<IDataPacket> source, Func<IBlueRow, T> valueSelector, Func<T, int> keySelector) =>
-      source.AsTables<T, int>(valueSelector, keySelector);
-
-   public static IObservable<TableUpdate<T, TKey>> AsTables<T, TKey>(
-      this IObservable<IDataPacket> source, Func<IBlueRow, T> valueSelector, Func<T, TKey> keySelector, Func<IBlueRow, TKey>? rowKeySelector = null) where TKey : IEquatable<TKey> {
-      rowKeySelector ??= r => keySelector(valueSelector(r));
-      return source.Scan(TableUpdate<T, TKey>.mk_empty(keySelector), (res, dp) => {
-         var bd = res.to_builder();
-         dp.get_new_added_deleted(out var newRows, out var addedRows, out var deletedRows);
-         foreach (IBlueRow r in addedRows) {
-            bd.add_or_update(valueSelector(r));
-         }
-         foreach (IBlueRow r in deletedRows) {
-            bd.remove(rowKeySelector(r));
-         }
-         return bd.to_table();
-      });
-   }
-
-   public static IObservable<IDataPacket> AsDataPackets(this IObservable<TableUpdate<IBlueRow, int>> source) =>
-      source.Scan(
-         (Table: (BlueTableImmutable?)null, Dp: (IDataPacket?)null),
-         (res, table) => {
-            var resTable = res.Table;
-            if (resTable is null) {
-               if (table.state.Count == 0) {
-                  return (null, null);
-               }
-               resTable = new(table.state.Values.First().schema);
-            }
-            var bd = resTable.ToDataPacketBuilder();
-            foreach (var r in table.removed) {
-               bd.Remove(r.Id());
-            }
-            foreach (var r in table.added) {
-               bd.AddOrUpdate(r);
-            }
-            var resDp = bd.ToDataPacket(out var newRows);
-            return (newRows, resDp);
-         })
-         .Where(t => t.Dp is not null && (t.Dp.addedRows.Count > 0 || t.Dp.deletedRows.Count > 0))
-         .Select(t => t.Dp!);
-
-   public static IObservable<IDataPacket> AsDataPackets<T>(this IObservable<TableUpdate<T, int>> source, IBlueTableSchema schema) where T : IBlueRow =>
-      source.Scan(
-         (Table: new BlueTableImmutable(schema), Dp: (IDataPacket)new DataPacket(schema)),
-         (res, table) => {
-            var bd = res.Table.ToDataPacketBuilder();
-            foreach (var r in table.removed) {
-               bd.Remove(r.Id());
-            }
-            foreach (var r in table.added) {
-               bd.AddOrUpdate(r);
-            }
-            var resDp = bd.ToDataPacket(out var newRows);
-            return (newRows, resDp);
-         })
-         .Where(t => t.Dp.addedRows.Count > 0 || t.Dp.deletedRows.Count > 0)
-         .Select(t => t.Dp);
-
-   public static IObservable<IDataPacket> AsDataPackets<T, TKey>(this IObservable<TableUpdate<T, TKey>> source, BlueTableSchemaAdapter<T> schema) where TKey : IEquatable<TKey> {
-      return Observable.Create<IDataPacket>(Subscribe);
-
-      IDisposable Subscribe(IObserver<IDataPacket> o) {
-         var adapter = new BlueTableImmutableAdapter<T>(schema);
-         IDataPacket initDataPacket = new DataPacket(schema);
-         return source.Scan((Adapter: adapter, DataPacket: initDataPacket), OnUpdate).Select(res => res.DataPacket).Subscribe(o);
-
-         (BlueTableImmutableAdapter<T> Adapter, IDataPacket DataPacket) OnUpdate(
-            (BlueTableImmutableAdapter<T> Adapter, IDataPacket DataPacket) state, TableUpdate<T, TKey> tableUpdate) {
-            var bd = state.Adapter.ToDataPacketBuilder();
-            bd.Update(tableUpdate);
-            var newRows = bd.ToImmutable(out var dp);
-            return (newRows, dp);
-         }
-      }
-   }
-
-   public static ViewStream AsViewStream<T>(this IObservable<TableUpdate<T, int>> source, IBlueTableSchema schema) where T : IBlueRow =>
-      source.AsDataPackets(schema).AsViewStream();
-
-   public static IObservable<IDataPacket> OnWorkScheduler(this IObservable<IDataPacket> source, IWorkScheduler scheduler) =>
-      Observable.Create<IDataPacket>(o => {
-         var inputGate = new AtomicGate();
-         var outputGate = new AtomicGate();
-         BlueTableImmutable? lastTable = null;
-         BlueTableImmutable.DataPacketBuilder? bd = null;
-         IDataPacket? pendingDpOut = null;
-         return Disposable.Create(source.Subscribe(OnNext), scheduler.Subscribe(DoWork));
-
-         void OnNext(IDataPacket value) {
-            using (inputGate.BeginTransaction()) {
-               lastTable ??= new(value.newRows.schema);
-               bd ??= lastTable.ToDataPacketBuilder();
-               if (pendingDpOut is null) {
-                  // The last outgoing datapacket was just fired. Don't merge this update with the
-                  // previous one (otherwise its adds/removes would be fired twice).
-                  bd.Reset();
-               }
-               foreach (IBlueRow r in value.deletedRows) {
-                  bd.Remove(r.Id());
-               }
-               foreach (IBlueRow r in value.addedRows) {
-                  bd.AddOrUpdate(r);
-               }
-               pendingDpOut = bd.ToDataPacket(out lastTable, false);
-            }
-         }
-
-         void DoWork() {
-            IDataPacket? dpOut = null;
-            if (outputGate.TryEnter()) {
-               using var exit = Disposable.Create(outputGate.Exit);
-               (dpOut, pendingDpOut) = (pendingDpOut, dpOut);
-               if (dpOut is not null) {
-                  o.OnNext(dpOut);
-               }
-            }
-         }
-      });
-
-   public static IObservable<IDataPacket> NonIncrementalAsyncTransform(this IObservable<IDataPacket> source, Func<IBlueRowCollection, Task<IBlueRowCollection?>> transformAsync, IWorkScheduler workScheduler) {
-      const int STATE_IDLE = 0;
-      const int STATE_BUSY = 1;
-      const int STATE_DISPOSED = 2;
-      return Observable.Create<IDataPacket>(Subscribe);
-      IDisposable Subscribe(IObserver<IDataPacket> o) {
-         var syncRoot = new object();
-         IBlueRowCollection? lastTableIn = null;
-         BlueTableImmutable? lastTableOut = null;
-         int nConcurrentDoWorks = 0;
-         int state = STATE_IDLE;
-         IDisposable? dispose = null;
-         dispose = Disposable.Create(source.Subscribe(OnNext), workScheduler.Subscribe(DoWork));
-         return Disposable.Create(Dispose);
-
-         void Dispose() {
-            lock (syncRoot) {
-               switch (state) {
-                  case STATE_DISPOSED:
-                     return;
-                  case STATE_BUSY:
-                     state = STATE_DISPOSED;
-                     return;
-               }
-               state = STATE_DISPOSED;
-            }
-            Disposable.dispose(ref dispose);
-         }
-
-         void OnNext(IDataPacket value) =>
-            Volatile.Write(ref lastTableIn, value.newRows);
-
-         async void DoWork() {
-            lock (syncRoot) {
-               nConcurrentDoWorks++;
-               switch (state) {
-                  case STATE_DISPOSED:
-                  case STATE_BUSY:
-                     return;
-                  case STATE_IDLE:
-                     break;
-                  default:
-                     Log.assert(false, $"Unexpected state ({StateName(state)}) when DoWork called");
-                     return;
-               }
-               state = STATE_BUSY;
-            }
-            var dp = await PublishTable();
-            bool isDisposed = false;
-            lock (syncRoot) {
-               switch (state) {
-                  case STATE_DISPOSED:
-                     isDisposed = true;
-                     break;
-                  case STATE_BUSY:
-                     break;
-                  default:
-                     Log.assert(false, $"Unexpected state ({StateName(state)}) after building output table (async)");
-                     return;
-               }
-            }
-            if (isDisposed) {
-               Disposable.dispose(ref dispose);
-               return;
-            }
-            if (dp is not null) {
-               o.OnNext(dp);
-            }
-            lock (syncRoot) {
-               nConcurrentDoWorks = 0;
-               switch (state) {
-                  case STATE_DISPOSED:
-                     isDisposed = true;
-                     break;
-                  case STATE_BUSY:
-                     state = STATE_IDLE;
-                     break;
-                  default:
-                     Log.assert(false, $"Unexpected state ({StateName(state)}) after firing output table");
-                     return;
-               }
-            }
-            if (isDisposed) {
-               Disposable.dispose(ref dispose);
-            }
-
-            async Task<DataPacket?> PublishTable() {
-               var tableIn = Interlocked.Exchange(ref lastTableIn, null);
-               if (tableIn is null) {
-                  return null;
-               }
-               var tableOut = await transformAsync(tableIn);
-               if (tableOut is null) {
-                  return null;
-               }
-               lastTableOut ??= new(tableOut.schema);
-               var bd = lastTableOut.ToDataPacketBuilder();
-               bd.Clear();
-               foreach (IBlueRow r in tableOut) {
-                  bd.AddOrUpdate(r, BlueRowEqualityComparer.Instance);
-               }
-               return bd.ToDataPacket(out lastTableOut);
-            }
-         }
-
-         static string StateName(int state) =>
-            state switch {
-               STATE_IDLE => "IDLE",
-               STATE_BUSY => "BUSY",
-               STATE_DISPOSED => "DISPOSED",
-               _ => $"STATE_{state}",
-            };
+         void OnChange(object? newValue) =>
+            observer.OnNext(newValue as DataPacket ?? throw new ArgumentException(
+               $"Expected DataPacket, got {newValue?.GetType()}", nameof(newValue)));
      }
   }
 }
@@ -1,26 +1,26 @@
-namespace bts.BlueOrb {
+namespace bts.SilverPine {
   using System;
   using BCG;
   using System.Collections.Generic;
   using Bts.DataStreamExpressions;
-   using Bts.DataStreamCore;

   // All this does is take in a dsl expression, parse it and fire the data that come out of running it.
   // So for example, if an expresison is passed 
   [DataStreamUserRespresentation("dsl")]
   public class DSLStream : ViewStream {
      private readonly string _dsl;
-      public DSLStream(string dsl, IDataStreamFactory streamFactory) {
+      public DSLStream(string dsl) {
         Log.Gen.info($"DSLStream.cons dsl: {dsl}");
         _dsl = dsl;
-         var rs = build_stream(streamFactory, dsl) ?? new LiteralStream("error", $"msg:invalid dsl");
+         ViewStream rs = build_stream(dsl);
+         rs = rs ?? new LiteralStream("error", $"msg:invalid dsl \"{dsl}\"");
         mySetRealSource(rs);
      }

-      static ViewStream? build_stream(IDataStreamFactory streamFactory, string stream_expression) {
+      static ViewStream? build_stream(string stream_expression) {
         try {
            var exp = DataStreamExpressionsInterpreter.Evaluate(stream_expression);
-            var rv = exp.getView(new Dictionary<string, StatementExpression>(0), streamFactory);
+            var rv = exp.getView(new Dictionary<string, StatementExpression>(0));
            return rv;

         } catch (Exception ex) {
@@ -29,4 +29,4 @@ namespace bts.BlueOrb {
         }
      }
   }
-}
+}
@@ -40,7 +40,7 @@ namespace BCG {
        protected override void myOnFirstSubscriber() {
            MasterWorker.getInstance(false).addWorker(this, _send_period);
            var server_rci = new RemoteConnectionInfo(_ip, _port);
-            bts.AsyncTCPConnector_AsyncWrapper.connect_to(server_rci, this);
+            bts.AsyncTCPConnector_APM.connect_to(server_rci, this);
        }

        protected override void myOnLastSubscriber() {
@@ -76,4 +76,4 @@ namespace BCG {


    }
-}
+}
@@ -0,0 +1,37 @@
+namespace Bts.DataStream;
+
+using BCG;
+
+using bts;
+
+using System;
+
+public class ObservableViewStream : ViewStream, IObservable<DataPacket>, IDisposable {
+   readonly IObservable<DataPacket> source;
+   IDisposable? subscription;
+
+   public ObservableViewStream(IObservable<DataPacket> source) =>
+      this.source = source;
+
+   public void Dispose() {
+      GC.SuppressFinalize(this);
+      if (subscription is not null) {
+         subscription.Dispose();
+         subscription = null;
+      }
+   }
+
+   protected override void myOnFirstSubscriber() {
+      Log.isnull(subscription);
+      subscription = source.Subscribe(myFireEvent);
+   }
+
+   protected override void myOnLastSubscriber() {
+      Log.notnull(subscription);
+      subscription.Dispose();
+      subscription = null;
+   }
+
+   IDisposable IObservable<DataPacket>.Subscribe(IObserver<DataPacket> observer) =>
+      source.Subscribe(observer);
+}
@@ -0,0 +1,90 @@
+using System;
+using System.Collections.Generic;
+
+using Bts.DataStreamExpressions;
+using BCG.Utilities.General;
+using bts;
+
+namespace BCG {
+   // listens to the requests posted on "cloud_streams" request table, and 
+   // if unfulfilled starts a ViewPubsliher
+   public class RemoteQueryManager : IWorker {
+      public class StreamPubUnit {
+         public string name;
+         public string stream_expression;
+         public ViewStream stream;
+         public bool persist;
+         public ViewPublisher view_publisher;
+
+         public StreamPubUnit(string name_, string stream_expression_, ViewStream stream_, bool persist_, ViewPublisher view_publisher_) {
+            name = name_;
+            stream_expression = stream_expression_;
+            stream = stream_;
+            persist = persist_;
+            view_publisher = view_publisher_;
+         }
+      }
+      static ViewStream? build_stream(string stream_expression)
+      {
+         try
+         {
+            var exp = DataStreamExpressionsInterpreter.Evaluate(stream_expression);
+            var rv = exp.getView(new Dictionary<string, StatementExpression>(0));
+            return rv;
+
+         } catch (Exception ex) {
+            bts.Log.assert(false, $"Could not build stream: {stream_expression} ex: {ex}");
+            return null;
+         }
+      }
+      private IDictionary<string, StreamPubUnit>
+         _stream_pub_map = new Dictionary<string, StreamPubUnit>();
+      private ViewStream _stream = new TableStream("cloud_streams");
+      private PubTableBroker _broker = new PubTableBroker();
+      private ManagedSubscriber _sub = ManagedSubscriber.instance;
+
+
+      public RemoteQueryManager() {
+         _stream.onChange += _on_str;
+         _sub.subscribe("RemoteRequest", (string tag, byte[] data) => {
+            RemoteRequest req = RemoteRequest.construct_from_bytes(data);
+            _spawn_remote_stream(req.published_table_name, req.stream_expression, false);
+         });
+      }
+      private void _on_str(object obj)
+      {
+         if (obj is not DataPacket dp) {
+            return;
+         }
+         foreach (IBlueRow r in dp.newRows)
+         {
+            string name = r.get<string>("name");
+            bool persist = r.get<bool>("persist");
+            string stream_exp = r.get<string>("stream_exp");
+            if (!_stream_pub_map.ContainsKey(name)) {
+               _spawn_remote_stream(name, stream_exp, persist);
+            }
+         }
+      }
+
+      private void _spawn_remote_stream(string name, string stream_expression, bool persist) {
+         bts.Log.Gen.info($"CloudStreamManager.assign name: {name} stream_expression: {stream_expression}");
+         ViewStream stream = build_stream(stream_expression);
+         _expose_to_ws(name, stream);
+         if (persist) {
+            stream = new PersistorStream(stream, name, "C:\\Temp\\", 1000000);   // 10MBs
+         }
+         ViewPublisher view_pub = new ViewPublisher(name, _broker, stream);
+         StreamPubUnit u = new StreamPubUnit(name, stream_expression, stream, persist, view_pub);
+         _stream_pub_map.Add(name, u);
+      }
+
+      private void _expose_to_ws(string name, ViewStream stream) {
+         // TODO: john
+      }
+
+      public void doWork() {
+
+      }
+   }
+}
@@ -1,168 +0,0 @@
-using bts;
-
-using Bts.DataStreamCore;
-using Bts.DataStreamExpressions;
-using Bts.PubSub;
-
-using Microsoft.Extensions.Hosting;
-
-using Spark.Core.Disposables;
-
-using System;
-using System.Collections.Generic;
-using System.IO;
-using System.Linq;
-using System.Reactive.Linq;
-using System.Threading;
-using System.Threading.Tasks;
-
-namespace BCG {
-   // listens to the requests posted on "cloud_streams" request table, and 
-   // if unfulfilled starts a ViewPubsliher
-   public class RemoteQueryManager : IHostedService {
-      public class StreamPubUnit {
-         public string name;
-         public string stream_expression;
-         public ViewStream stream;
-         public bool persist;
-         public ViewPublisher view_publisher;
-
-         public StreamPubUnit(string name_, string stream_expression_, ViewStream stream_, bool persist_, ViewPublisher view_publisher_) {
-            name = name_;
-            stream_expression = stream_expression_;
-            stream = stream_;
-            persist = persist_;
-            view_publisher = view_publisher_;
-         }
-      }
-      static ViewStream? build_stream(IDataStreamFactory streamFactory, string stream_expression)
-      {
-         try
-         {
-            var exp = DataStreamExpressionsInterpreter.Evaluate(stream_expression);
-            var rv = exp.getView(new Dictionary<string, StatementExpression>(0), streamFactory);
-            return rv;
-
-         } catch (Exception ex) {
-            bts.Log.assert(false, $"Could not build stream: {stream_expression} ex: {ex}");
-            return null;
-         }
-      }
-      private IDictionary<string, StreamPubUnit>
-         _stream_pub_map = new Dictionary<string, StreamPubUnit>();
-      readonly IPubTableBroker _broker;
-      readonly IManagedSubscriber _sub;
-      readonly IDataStreamFactory streamFactory;
-      readonly ITableStreamFactory tableStreamFactory;
-      readonly object syncRoot = new();
-      IDisposable? cloudStreamsSub;
-      IDisposable? remoteRequestsSub;
-      State state;
-
-      public RemoteQueryManager(IManagedSubscriber managedSubscriber, IPubTableBroker pubTableBroker, ITableStreamFactory tableStreamFactory, IEnumerable<IDataStreamFactory> streamFactories) {
-         _sub = managedSubscriber;
-         _broker = pubTableBroker;
-         this.tableStreamFactory = tableStreamFactory;
-         var streamFactoryList = streamFactories.ToList();
-         streamFactory = streamFactoryList.Count == 1
-            ? streamFactoryList[0]
-            : new CompositeDataStreamFactory(streamFactoryList);
-      }
-
-      enum State {
-         Idle = 0,
-         Running = 1,
-         Disposed = 2,
-         WaitingToDispose = 3,
-      }
-
-      public Task StartAsync(CancellationToken cancellationToken) {
-         lock (syncRoot) {
-            switch (state) {
-               case State.Idle:
-                  break;
-               case State.WaitingToDispose:
-               case State.Disposed:
-                  throw new ObjectDisposedException(nameof(RemoteQueryManager));
-               default:
-                  return Task.CompletedTask;
-            }
-            state = State.Running;
-         }
-         if (cloudStreamsSub is not null) {
-            throw new InvalidOperationException("cloudStreamsSub already created");
-         }
-         if (remoteRequestsSub is not null) {
-            throw new InvalidOperationException("remoteRequestsSub already created");
-         }
-         var cloudStream = tableStreamFactory.Create("cloud_streams")
-            ?? throw new Exception("Failed to create TableStream 'cloud_stream'");
-         cloudStreamsSub = cloudStream.AsObservable().Subscribe(_on_str);
-         remoteRequestsSub = _sub.Observe("RemoteRequest").Subscribe(data => {
-            RemoteRequest req = RemoteRequest.construct_from_bytes(data);
-            _spawn_remote_stream(req.published_table_name, req.stream_expression, false);
-         });
-         lock (syncRoot) {
-            if (state == State.WaitingToDispose) {
-               state = State.Disposed;
-            }
-         }
-         if (state == State.Disposed) {
-            Disposable.dispose(ref cloudStreamsSub);
-            Disposable.dispose(ref remoteRequestsSub);
-         }
-         return Task.CompletedTask;
-      }
-
-      public Task StopAsync(CancellationToken cancellationToken) {
-         lock (syncRoot) {
-            switch (state) {
-               case State.Running:
-                  break;
-               default:
-                  return Task.CompletedTask;
-            }
-            state = State.Disposed;
-         }
-         Disposable.dispose(ref cloudStreamsSub);
-         Disposable.dispose(ref remoteRequestsSub);
-         return Task.CompletedTask;
-      }
-
-      private void _on_str(IDataPacket dp)
-      {
-         foreach (IBlueRow r in dp.newRows)
-         {
-            string name = r.get<string>("name");
-            bool persist = r.get<bool>("persist");
-            string stream_exp = r.get<string>("stream_exp");
-            if (!_stream_pub_map.ContainsKey(name)) {
-               _spawn_remote_stream(name, stream_exp, persist);
-            }
-         }
-      }
-
-      private void _spawn_remote_stream(string name, string stream_expression, bool persist) {
-         bts.Log.Gen.info($"CloudStreamManager.assign name: {name} stream_expression: {stream_expression}");
-         var stream = build_stream(streamFactory, stream_expression);
-         if (stream is null) {
-            Log.DataFlow.error($"Failed to spawn remote stream. name: {name}; expression: {stream_expression}");
-            return;
-         }
-         _expose_to_ws(name, stream);
-         if (persist) {
-            string tempDir = Path.GetTempPath();
-        Directory.CreateDirectory(tempDir); // make sure it exists
-
-        stream = new PersistorStream(stream, name, tempDir, 10_000_000); // 10 MB
-         }
-         ViewPublisher view_pub = new ViewPublisher(name, _broker, stream);
-         StreamPubUnit u = new StreamPubUnit(name, stream_expression, stream, persist, view_pub);
-         _stream_pub_map.Add(name, u);
-      }
-
-      private void _expose_to_ws(string name, ViewStream stream) {
-         // TODO: john
-      }
-   }
-}
@@ -2,30 +2,17 @@ using BCG;

 using bts;

-using Bts.DataStreamCore;
-
-using Spark.Core.Disposables;
-
 using System;

 namespace BCG {
-   /// <summary>
-   /// This is a stream so we can run it from the DSQA (since dsqa can only load streams).
-   /// <para>
-   /// Note: It's hidden because this stream is now created by an IDataStreamFactory
-   /// (see RemoteQueryStreamFactory.cs for details)
-   /// </para>
-   /// </summary>
-   [DataStreamUserRespresentation(false)]
+   // this is a stream so we can run it from the DSQA (since dsqa can only load streams)
+   [DataStreamUserRespresentation("remoteQuery")]
   public class RemoteQueryStream : ViewStream {
      private readonly string _name, _stream_exp;
-      readonly ITableStreamFactory tableStreamFactory;
-      readonly IPublisherCore _publisher_core;
-      IDisposable? subscription;
+      private PublisherCore _publisher_core = new PublisherCore();
+      private ViewStream _source_str;

-      public RemoteQueryStream(ITableStreamFactory tableStreamFactory, IPublisherCore publisherCore, params string[] lines) {
-         this.tableStreamFactory = tableStreamFactory;
-         _publisher_core = publisherCore;
+      public RemoteQueryStream(params string[] lines) {
         _stream_exp = get_expression(lines);
         _name = Guid.NewGuid().ToString();
         _name = $"remotereq_{DateTimeOffset.UtcNow.ToUnixTimeMilliseconds().ToString()}";
@@ -34,17 +21,17 @@ namespace BCG {
      }

      protected override void myOnFirstSubscriber() {
-         Log.isnull(subscription);
         RemoteRequest rem_req = new RemoteRequest(_stream_exp, _name);
         byte[] rem_req_bytes = rem_req.serialize();
         _publisher_core.publish("System", "RemoteRequest", rem_req_bytes);
-         var sourceStream = tableStreamFactory.Create(_name);
-         sourceStream.onChange += _handle_data;
-         subscription = Disposable.Create(() => sourceStream.onChange -= _handle_data);
+         _source_str = new TableStream(_name);
+         _source_str.onChange += new ChangeDelegate(_handle_data);
      }

-      protected override void myOnLastSubscriber() =>
-         Disposable.dispose(ref subscription);
+      protected override void myOnLastSubscriber() {
+         //maybe there should be a way to turn remote requests off ?
+         _source_str.onChange -= new ChangeDelegate(_handle_data);
+      }

      static string get_expression(params string[] lines) {
         for (int i = 0; i < lines.Length; i++) {
@@ -1,51 +1,33 @@
 using System;
-using System.Collections.Generic;

-namespace BCG {   
+namespace BCG {
   public class AddColumnsStream : ViewStream {
-      public static Tuple<string, Type, object> parse_single_tok(string exp) {
+      public static Tuple<string, Type, object> parse_exp(string exp) {
         string[] toks = exp.Split(':');
-         string col_name = toks[0].Trim();
-         string col_value = toks[1].Trim();
+         string col_name = toks[0];
+         string col_value = toks[1];
         Tuple<Type, object> tup = BlueDataUtility.guess_type(col_value);
         return Tuple.Create(col_name, tup.Item1, tup.Item2);
      }

-      public static List<Tuple<string, Type, object>> parse_exp(string exp) {
-         string[] toks = exp.Split(",");
-         List<Tuple<string, Type, object>> rv = new List<Tuple<string, Type, object>>();
-         foreach (string s in toks) {
-            var tup = parse_single_tok(s.Trim());
-            rv.Add(tup);
-         }
-         return rv;
-      }
-
      private Tuple<string, Type, object> _new_col;
-      private List<Tuple<string, Type, object>> _new_col_map;
      private IBlueTableSchema _out_schema;

      public AddColumnsStream(ViewStream vs, string col_def) {
-         _new_col_map = parse_exp(col_def);
+         _new_col = parse_exp(col_def);
         ViewStream rs = new MapStream(vs, (IBlueTableSchema s) => _get_schema(s), (IBlueRow r) => _map(r));
         mySetRealSource(rs);
      }

      private IBlueTableSchema _get_schema(IBlueTableSchema input_schema) {
-         if (_out_schema == null) {
-            _out_schema = input_schema;
-            foreach (var m in _new_col_map) {
-               _out_schema = BlueDataUtility.add_column(_out_schema, m.Item1, m.Item2);
-            }
-         }
+         if (_out_schema == null)
+            _out_schema = BlueDataUtility.add_column(input_schema, _new_col.Item1, _new_col.Item2);
         return _out_schema;
      }

      private IBlueRow _map(IBlueRow r) {
         MutableBlueRow rv = new MutableBlueRow(_get_schema(r.schema), r);
-         foreach (var m in _new_col_map) {
-            rv.setValue(m.Item1, m.Item3);
-         }
+         rv.setValue(_new_col.Item1, _new_col.Item3);
         return rv;
      }
   }
@@ -1,44 +1,52 @@
 using System;

-namespace bts.BlueOrb {
-   using BCG;
-   [DataStreamUserRespresentation("oldCount")]
-   public class CountStream : ViewStream {
+namespace BCG
+{
+   [DataStreamUserRespresentation ("count")]
+	public class CountStream : ViewStream
+	{
      private DataPacket myLastFiredDataPacket;
-      private IBlueTableSchema mySchema = new BlueTableSchema("count",
-         new ColumnInfo("id", 0, typeof(int)),
-         new ColumnInfo("count", 1, typeof(int)));
+      private IBlueTableSchema mySchema = new BlueTableSchema ("count", 
+         new ColumnInfo ("id", 0, typeof(int)),
+         new ColumnInfo ("count", 1, typeof(int)));
      private int myId = 0;
      private ViewStream myVS;

-      public CountStream(ViewStream vs) {
+      public CountStream (ViewStream vs)
+      {
         myVS = vs;
      }

-      protected override void myOnFirstSubscriber() {
+      public CountStream (string tableName)
+         : this (new TableStream (tableName))
+      {}
+
+      protected override void myOnFirstSubscriber () {
         myVS.onChange += new ChangeDelegate(myHandleChanges);
      }
-      protected override void myOnLastSubscriber() {
+      protected override void myOnLastSubscriber () {
         myVS.onChange -= new ChangeDelegate(myHandleChanges);
      }


-      private void myHandleChanges(object newValue) {
-         myFireEvent(
+      private void myHandleChanges(object newValue)
+      {
+         myFireEvent (
            myLastFiredDataPacket =
               myLastFiredDataPacket == null ?
-                  new DataPacket(myGetRows(newValue, ++myId), myGetRows(newValue, myId),
-                  new BlueTable(mySchema))
+                  new DataPacket (myGetRows (newValue, ++myId), myGetRows (newValue, myId), 
+                  new BlueTable (mySchema))
                  :
-                  new DataPacket(myGetRows(newValue, ++myId), myGetRows(newValue, myId),
+                  new DataPacket (myGetRows (newValue, ++myId), myGetRows (newValue, myId), 
                  myLastFiredDataPacket.newRows)
            );
      }

-      private IBlueRowCollection myGetRows(object dp, int id) {
-         BlueTable retVal = new BlueTable(mySchema);
+      private IBlueRowCollection myGetRows (object dp, int id)
+      {
+         BlueTable retVal = new BlueTable (mySchema);
         retVal.addRow
-            (new BlueRow(mySchema, id, (dp as DataPacket).newRows.Count));
+            (new BlueRow (mySchema, id, (dp as DataPacket).newRows.Count));
         return retVal;
      }
Author	SHA1	Message	Date
Taha Afzal	66ad51280f	Reversehedging fixed	2025-04-08 22:13:33 +00:00
arham	029edd8a86	Format: ran clang format on file	2025-04-07 15:39:57 +00:00
arham	dfdcbab2a0	Add: config changes to make server controller work and set up hk zen server	2025-04-07 15:33:26 +00:00
Taha Afzal	6d7c0702f0	Fixing build:	2025-04-05 23:38:05 +00:00
Taha Afzal	86eaddea93	Fixing a bug, where the spreader was isidentifying the quoting leg	2025-04-05 15:38:30 +00:00
				`@@ -1 +0,0 @@`
				`PYTHONPATH=/workspaces/spark/build/sparkd-debug-clang/src/lib_python_interop;/workspaces/spark/src/lib_python_interop/python_interop_plugins;/workspaces/spark/tools/py`