cleanup: Remove unnecessary py_module.cpp files

We used to have separate modules for md and execution. I forgot to remove these after unifying them to a single module.
diag: Reduce book log noise in md plugins
2025-06-16 14:31:28 +00:00 · 2025-06-16 14:28:09 +00:00 · 2025-06-16 14:17:45 +00:00 · 2025-06-13 21:26:36 +00:00 · 2025-06-13 20:50:30 +00:00 · 2025-06-13 20:50:30 +00:00
1970 changed files with 82349 additions and 60419 deletions
@@ -9,7 +9,7 @@
      ]
    },
    "fable": {
-      "version": "4.0.6",
+      "version": "4.24.0",
      "commands": [
        "fable"
      ]
@@ -27,4 +27,4 @@
      ]
    }
  }
-}
+}
@@ -65,7 +65,7 @@ RUN \
   && apt-get install -y cmake

 # Dotnet
-RUN apt-get update && apt-get install -y dotnet7
+RUN apt-get update && apt-get install -y dotnet8

 # Node
 ARG node_major=18
@@ -129,5 +129,3 @@ RUN \
   -G sudo \
   --password "$(openssl passwd -1 "$docker_pass")" \
   "$docker_user"
-
-RUN usermod -aG docker "$docker_user"
@@ -1,15 +1,10 @@
 {
  "dockerFile": "Dockerfile",
-  // Creates local dir to mount to HOME in docker 
-  "initializeCommand": "mkdir -p ${localEnv:HOME}/.bts/devcontainer-mounts/${devcontainerId}",
  "hostRequirements": {
-    "memory": "4gb",
-    "storage": "6gb"
+    "memory": "8gb",
+    "storage": "10gb"
  },
-  "postCreateCommand": "/home/bts/spark/docker/sparkd/dev/devcontainer_post_create.sh",
  "containerUser": "bts",
-  "workspaceMount": "source=${localWorkspaceFolder},target=/home/bts/spark,type=bind",
-  "workspaceFolder": "/home/bts/spark",
  "updateRemoteUserUID": true,
  "shutdownAction": "none",
  // Note: If running multiple containers on the same host, each must use a
@@ -17,26 +12,19 @@
  // example, if 8000-8003 is taken, go with 9000-9003, then 10000-10003, etc.
  "runArgs": [
    "--shm-size",
-    "2gb",
-    "--network",
-    "host"
+    "8gb"
  ],
  "mounts": [
-    "source=${localEnv:HOME}/.bts/devcontainer-mounts/${devcontainerId},target=/home/bts,type=bind",
    "source=/var/run/docker.sock,target=/var/run/docker.sock,type=bind"
  ],
  "customizations": {
    "vscode": {
-      "settings": {
-        "clangd.arguments": [
-          "--compile-commands-dir=${workspaceFolder}/build/clang/debug"
-        ]
-      },
      "extensions": [
        "eamodio.gitlens",
        "llvm-vs-code-extensions.vscode-clangd",
        "ms-vscode.cmake-tools",
-        "ms-vscode.cpptools"
+        "ms-vscode.cpptools",
+        "xaver.clang-format"
      ]
    }
  }
@@ -207,23 +207,17 @@ dotnet_naming_symbols.parameters.applicable_accessibilities = public,internal,pr
 dotnet_naming_symbols.private_symbols.applicable_kinds = property,method,field,event,delegate
 dotnet_naming_symbols.private_symbols.applicable_accessibilities = protected,private

-# local symbols
-dotnet_naming_symbols.local_symbols.applicable_kinds = local
-
-# parameter symbols
-dotnet_naming_symbols.parameter_symbols.applicable_kinds = parameter
-
 # Naming styles

-dotnet_naming_style.pascal_case.required_prefix =
-dotnet_naming_style.pascal_case.required_suffix =
-dotnet_naming_style.pascal_case.word_separator =
-dotnet_naming_style.pascal_case.capitalization = pascal_case
+dotnet_naming_style.snake_case.required_prefix =
+dotnet_naming_style.snake_case.required_suffix =
+dotnet_naming_style.snake_case.word_separator = _
+dotnet_naming_style.snake_case.capitalization = all_lower

-dotnet_naming_style.camel_case.required_prefix =
-dotnet_naming_style.camel_case.required_suffix =
-dotnet_naming_style.camel_case.word_separator =
-dotnet_naming_style.camel_case.capitalization = camel_case
+dotnet_naming_style.private_snake_case.required_prefix = _
+dotnet_naming_style.private_snake_case.required_suffix =
+dotnet_naming_style.private_snake_case.word_separator = _
+dotnet_naming_style.private_snake_case.capitalization = all_lower

 dotnet_naming_style.user_types.required_prefix =
 dotnet_naming_style.user_types.required_suffix =
@@ -235,10 +229,6 @@ dotnet_naming_style.interface.required_suffix =
 dotnet_naming_style.interface.word_separator =
 dotnet_naming_style.interface.capitalization = pascal_case

-dotnet_naming_style.local.capitalization = camel_case
-
-dotnet_naming_style.parameter.capitalization = camel_case
-
 # rules
 dotnet_naming_rule.user_types.symbols = user_types
 dotnet_naming_rule.user_types.style = user_types
@@ -248,18 +238,14 @@ dotnet_naming_rule.interfaces.symbols = interface
 dotnet_naming_rule.interfaces.style = interface_style
 dotnet_naming_rule.interfaces.severity = suggestion

-dotnet_naming_rule.public_members.symbols = public_symbols
-dotnet_naming_rule.public_members.style = pascal_case
-dotnet_naming_rule.public_members.severity = suggestion
+dotnet_naming_rule.public_members_snake_case.symbols = public_symbols
+dotnet_naming_rule.public_members_snake_case.style = snake_case
+dotnet_naming_rule.public_members_snake_case.severity = suggestion

-dotnet_naming_rule.private_members.symbols = private_symbols
-dotnet_naming_rule.private_members.style = camel_case
-dotnet_naming_rule.private_members.severity = suggestion
+dotnet_naming_rule.private_members_snake_case.symbols = private_symbols
+dotnet_naming_rule.private_members_snake_case.style = private_snake_case
+dotnet_naming_rule.private_members_snake_case.severity = suggestion

-dotnet_naming_rule.variables_are_camel_case.severity = suggestion
-dotnet_naming_rule.variables_are_camel_case.symbols = local_symbols
-dotnet_naming_rule.variables_are_camel_case.style = local
-
-dotnet_naming_rule.parameters.symbols = parameter_symbols
-dotnet_naming_rule.parameters.style = parameter
+dotnet_naming_rule.parameters.symbols = parameters
+dotnet_naming_rule.parameters.style = snake_case
 dotnet_naming_rule.parameters.severity = suggestion
@@ -0,0 +1,49 @@
+# This file defines the global default configuration for docker deployments.
+# It is tracked in source control. For temporary modifications, please create
+# a separate file called ".env.local" (it's in .gitignore, so won't be tracked).
+# If a variable is defined in both files, the value in .env.local will be used.
+#
+# About the deployment procedure:
+#   We use Docker Compose to build images and launch containers for various services
+#   that make up our software suite:
+#     - sparkd
+#     - TODO: pubsub
+#     - TODO: md-publisher
+#     - TODO: webapp (DSQA on the web)
+#     - TODO: api-server (websocket based)
+#   From inside a devcontainer, you can launch all services in separate containers
+#   on your machine by running:
+#
+#     tools/deploy.sh
+#
+#   This script builds release versions for each service, packaged (generally) in
+#   self-contained directories. Next, it runs a few "docker compose" commands to
+#   build docker images loaded with the self-contained binaries from the previous
+#   step, and start containers with those images to run each service. To view the
+#   commands that deploy.sh runs internally (without actually running them), run:
+#
+#     tools/deploy.sh --dry-run
+#
+#   For additional information/help, run:
+#
+#     tools/deploy.sh --help
+#
+
+### Sparkd ###
+# The internal image for sparkd to use. For more info, view:
+#   src/server/sparkd/Dockerfile
+SPARKD__IMAGE=sparkd
+# The config dir for this service to use
+SPARKD__CONFIG_DIR=src/app_sparkd/configurations/dev
+# Mount a specific volume or dir to "~/sparkdata/data" inside the docker.
+# Defaults to a volume named "sparkd_<config_port>" (e.g., sparkd_8000).
+# SPARKD__DATA_DIR=/home/bts/sparkdata/data
+SPARKD__SOURCE_DIR=install/sparkd-release-gcc
+SPARKD__RUNTIME_CONF_DIR=/home/ubuntu/8000/sparkdata/conf/
+SPARKD__CONFIG_PORT=8000
+SPARKD__QUICKBUS_PORT=8001
+SPARKD__SIM_FIX_PORT=8002
+SPARKD__SHM_SIZE=4gb
+SPARKD__PYTHON_REQUIREMENTS=requirements.txt
+
+# TODO: add config for other services (pubsub, md-publisher, webapp, api-server, etc.)
@@ -1,4 +1,3 @@
 *.cmd text=auto eol=crlf
 *.cs text=auto eol=crlf
 *.sh text=auto eol=lf
-*.py text=auto eol=lf
@@ -59,3 +59,5 @@ trace_data/
 .devcontainer/
 *.Generated.cs
 *.lst
+.env.local
+.secrets
@@ -1,73 +1,19 @@
 {
  "version": "0.2.0",
  "configurations": [
+    {
+      "name": "Python Debugger: Current File with Arguments",
+      "type": "debugpy",
+      "request": "launch",
+      "program": "${file}",
+      "console": "integratedTerminal",
+      "args": "${command:pickArgs}"
+    },
    {
      "name": ".NET Core Attach",
      "type": "coreclr",
      "request": "attach"
    },
-    {
-      "name": "Api Server",
-      "type": "coreclr",
-      "request": "launch",
-      "program": "",
-      "windows": {
-        "preLaunchTask": "Build Api Server (Debug)",
-        "program": "spark_api_server.exe",
-        "args": [
-          "run",
-          "--sparkd",
-          "http://127.0.0.1:8000/config"
-        ],
-        "cwd": "build/Debug/bin/net7.0",
-      },
-      "stopAtEntry": false,
-      "console": "integratedTerminal"
-    },
-    {
-      "name": "Api Server (Skip Build)",
-      "type": "coreclr",
-      "request": "launch",
-      "program": "",
-      "windows": {
-        "program": "spark_api_server.exe",
-        "args": [
-          "run",
-          "--sparkd",
-          "http://127.0.0.1:8000/config"
-        ],
-        "cwd": "build/Debug/bin/net7.0",
-      },
-      "stopAtEntry": false,
-      "console": "integratedTerminal"
-    },
-    {
-      "name": "Bts.DataStreamWebApp.Server",
-      "type": "coreclr",
-      "request": "launch",
-      "preLaunchTask": "Build DataStream WebApp (Debug)",
-      "program": "${workspaceFolder}/build/Debug/bin/net7.0/Bts.DataStreamWebApp.Server.dll",
-      "cwd": "${workspaceFolder}/build/Debug/bin/net7.0",
-      "env": {
-        "ASPNETCORE_HTTP_PORT": "8080",
-        "ASPNETCORE_HTTPS_PORT": "8443",
-        "ASPNETCORE_ENVIRONMENT": "${input:dataStreamWebAppServerEnvironment}",
-        "ASPNETCORE_HOSTINGSTARTUPASSEMBLIES": "Microsoft.AspNetCore.SpaProxy",
-      }
-    },
-    {
-      "name": "Bts.DataStreamWebApp.Server (Skip Build)",
-      "type": "coreclr",
-      "request": "launch",
-      "program": "${workspaceFolder}/build/Debug/bin/net7.0/Bts.DataStreamWebApp.Server.dll",
-      "cwd": "${workspaceFolder}/build/Debug/bin/net7.0",
-      "env": {
-        "ASPNETCORE_HTTP_PORT": "8080",
-        "ASPNETCORE_HTTPS_PORT": "8443",
-        "ASPNETCORE_ENVIRONMENT": "${input:dataStreamWebAppServerEnvironment}",
-        "ASPNETCORE_HOSTINGSTARTUPASSEMBLIES": "Microsoft.AspNetCore.SpaProxy",
-      }
-    },
    {
      "name": "DataStreamPublisher",
      "type": "coreclr",
@@ -80,36 +26,63 @@
      "windows": {
        "program": "DataStreamPublisher.exe",
      },
-      "env": {
-        "DOTNET_ENVIRONMENT": "${input:dataStreamPublisherEnvironment}",
-      },
-      "cwd": "${workspaceFolder}/build/Debug/bin/net7.0",
+      "args": [
+        "--config",
+        "${workspaceFolder}/src/DataStreamSuite/DataStreamPublisher/Configurations/${input:dataStreamPublisherConfig}.json"
+      ],
+      "cwd": "build/Debug/bin/net8.0",
      "stopAtEntry": false,
      "console": "integratedTerminal"
    },
    {
-      "name": "DataStreamPublisher (Skip Build)",
-      "type": "coreclr",
+      "name": "(gdb) Launch",
+      "type": "cppdbg",
      "request": "launch",
-      "program": "",
-      "linux": {
-        "program": "DataStreamPublisher",
-      },
-      "windows": {
-        "program": "DataStreamPublisher.exe",
-      },
-      "env": {
-        "DOTNET_ENVIRONMENT": "${input:dataStreamPublisherEnvironment}",
-      },
-      "cwd": "${workspaceFolder}/build/Debug/bin/net7.0",
+      "program": "/home/bts/spark/build/sparkd-debug-clang/server/sparkd",
+      "args": [
+        "run"
+      ],
      "stopAtEntry": false,
-      "console": "integratedTerminal"
+      "cwd": "/home/bts/spark/build/sparkd-debug-clang/server",
+      "environment": [],
+      "externalConsole": false,
+      "MIMode": "gdb",
+    },
+    {
+      "name": "spreader tests2",
+      "type": "cppdbg",
+      "request": "launch",
+      "program": "/home/bts/spark/build/sparkd-debug-clang/server/sparkd",
+      "args": [
+        "test",
+        "[spreader2]"
+      ],
+      "stopAtEntry": false,
+      "cwd": "/home/bts/spark/build/sparkd-debug-clang/server",
+      "environment": [],
+      "externalConsole": false,
+      "MIMode": "gdb",
+    },
+    {
+      "name": "spreader tests",
+      "type": "cppdbg",
+      "request": "launch",
+      "program": "/home/bts/spark/build/sparkd-debug-clang/server/sparkd",
+      "args": [
+        "test",
+        "[spreader]"
+      ],
+      "stopAtEntry": false,
+      "cwd": "/home/bts/spark/build/sparkd-debug-clang/server",
+      "environment": [],
+      "externalConsole": false,
+      "MIMode": "gdb",
    },
    {
      "name": "DataStreamQueryAnalyzer",
      "type": "coreclr",
      "request": "launch",
-      "preLaunchTask": "Build DataStreamQueryAnalyzer (Debug)",
+      //"preLaunchTask": "Build DataStreamQueryAnalyzer (Debug)",
      "program": "",
      "windows": {
        "program": "DataStreamQueryAnalyzer.exe",
@@ -117,7 +90,7 @@
          "--config",
          "${workspaceFolder}/src/DataStreamSuite/DataStreamQueryAnalyzer/Configurations/DataStreamQueryAnalyzer.json"
        ],
-        "cwd": "build/Debug/bin/net7.0-windows",
+        "cwd": "build/Debug/bin/net8.0-windows",
        "env": {
          "PYTHON_EXE": "${workspaceFolder}/.venv/Scripts/python.exe"
        },
@@ -137,28 +110,11 @@
      "windows": {
        "program": "PubSubServer.exe",
      },
-      "cwd": "build/Debug/bin/net7.0",
-      "env": {
-        "DOTNET_ENVIRONMENT": "${input:pubSubServerEnvironment}",
-      },
-      "stopAtEntry": false,
-      "console": "integratedTerminal"
-    },
-    {
-      "name": "PubSubServer (Skip Build)",
-      "type": "coreclr",
-      "request": "launch",
-      "program": "",
-      "linux": {
-        "program": "PubSubServer",
-      },
-      "windows": {
-        "program": "PubSubServer.exe",
-      },
-      "cwd": "build/Debug/bin/net7.0",
-      "env": {
-        "DOTNET_ENVIRONMENT": "${input:pubSubServerEnvironment}",
-      },
+      "args": [
+        "--config",
+        "${workspaceFolder}/src/DataStreamSuite/PubSubServer/Configurations/${input:pubSubServerConfig}.json"
+      ],
+      "cwd": "build/Debug/bin/net8.0",
      "stopAtEntry": false,
      "console": "integratedTerminal"
    },
@@ -170,11 +126,19 @@
      "windows": {
        "preLaunchTask": "Build Spark (Debug)",
        "program": "Spark.exe",
-        "args": [
-          "--sparkd",
-          "http://127.0.0.1:8000/config"
-        ],
-        "cwd": "build/Debug/bin/net7.0-windows",
+        "cwd": "build/Debug/bin/net8.0-windows",
+      },
+      "stopAtEntry": false,
+      "console": "integratedTerminal"
+    },
+    {
+      "name": "Spark (Skip Build)",
+      "type": "coreclr",
+      "request": "launch",
+      "program": "",
+      "windows": {
+        "program": "Spark.exe",
+        "cwd": "build/Debug/bin/net8.0-windows",
      },
      "stopAtEntry": false,
      "console": "integratedTerminal"
@@ -191,7 +155,40 @@
          "--sparkd",
          "${input:sparkConfigServer}"
        ],
-        "cwd": "build/Debug/bin/net7.0-windows",
+        "cwd": "build/Debug/bin/net8.0-windows",
+      },
+      "stopAtEntry": false,
+      "console": "integratedTerminal"
+    },
+    {
+      "name": "Spark Test",
+      "type": "coreclr",
+      "request": "launch",
+      "program": "",
+      "windows": {
+        "preLaunchTask": "Build Spark (Debug)",
+        "program": "Spark.Console.exe",
+        "cwd": "build/Debug/bin/net8.0",
+        "args": [
+          "test",
+          "${input:sparkTest}"
+        ]
+      },
+      "stopAtEntry": false,
+      "console": "integratedTerminal"
+    },
+    {
+      "name": "Spark Test (Skip Build)",
+      "type": "coreclr",
+      "request": "launch",
+      "program": "",
+      "windows": {
+        "program": "Spark.Console.exe",
+        "cwd": "build/Debug/bin/net8.0",
+        "args": [
+          "test",
+          "${input:sparkTest}"
+        ]
      },
      "stopAtEntry": false,
      "console": "integratedTerminal"
@@ -200,7 +197,6 @@
      "name": "Sparkd",
      "type": "cppdbg",
      "request": "launch",
-      "preLaunchTask": "Build Sparkd (Debug | Clang)",
      "program": "",
      "linux": {
        "program": "${workspaceFolder}/build/sparkd-debug-clang/src/app_sparkd/sparkd",
@@ -234,7 +230,7 @@
        "program": "${workspaceFolder}/build/sparkd-debug-clang/src/app_sparkd/sparkd",
      },
      "args": [
-        "test-py-md-provider"
+        "run"
      ],
      "cwd": "${workspaceFolder}/build/sparkd-debug-clang/src/app_sparkd",
      "stopAtEntry": false,
@@ -256,41 +252,26 @@
  ],
  "inputs": [
    {
-      "id": "dataStreamPublisherEnvironment",
+      "id": "dataStreamPublisherConfig",
      "type": "pickString",
      "description": "Publisher Configuration",
      "options": [
-        "Development",
-        "MarketData",
-        "Production",
-        "ProductionAws",
-        "Staging",
+        "MarketDataPublisher",
+        "SimpleEchoClient",
+        "SimpleTestDataPublisher",
+        "TestDataPublisher",
      ],
      "default": "MarketDataPublisher",
    },
    {
-      "id": "pubSubServerEnvironment",
+      "id": "pubSubServerConfig",
      "type": "pickString",
      "description": "PubSub Configuration",
      "options": [
-        "Development",
-        "EchoServer",
-        "Production",
-        "ProductionAws",
-        "Staging",
+        "PubSubServer",
+        "SimpleEchoClient",
      ],
-      "default": "Development",
-    },
-    {
-      "id": "dataStreamWebAppServerEnvironment",
-      "type": "pickString",
-      "description": "DataStream WebApp Environment",
-      "options": [
-        "Development",
-        "Production",
-        "ProductionAws",
-        "Staging",
-      ]
+      "default": "PubSubServer",
    },
    {
      "id": "sparkConfigServer",
@@ -299,14 +280,14 @@
      "default": "http://localhost:8000/config",
    },
    {
-      "id": "isSparkApiEnabled",
-      "type": "pickString",
-      "description": "Enable Spark Api (connect to sparkd)",
-      "options": [
-        "True",
-        "False",
-      ],
-      "default": "True",
-    }
+      "id": "sparkTest",
+      "type": "promptString",
+      "description": "Test to run",
+    },
+    {
+      "id": "sparkdTest",
+      "type": "promptString",
+      "description": "Test to run",
+    },
  ]
 }
@@ -121,117 +121,6 @@
        "color": "terminal.ansiCyan"
      },
    },
-    {
-      "label": "Build DataStreamQueryAnalyzer",
-      "group": "build",
-      "type": "process",
-      "command": "",
-      "windows": {
-        "command": "tools/build_data_stream_query_analyzer.cmd",
-      },
-      "args": [
-        "--configuration",
-        "${input:buildConfiguration}",
-      ],
-      "icon": {
-        "id": "terminal",
-        "color": "terminal.ansiCyan"
-      },
-    },
-    {
-      "label": "Build DataStreamQueryAnalyzer (Debug)",
-      "group": "build",
-      "type": "process",
-      "command": "",
-      "windows": {
-        "command": "tools/build_data_stream_query_analyzer.cmd",
-      },
-      "args": [
-        "--configuration",
-        "Debug",
-      ],
-      "icon": {
-        "id": "terminal",
-        "color": "terminal.ansiCyan"
-      },
-    },
-    {
-      "label": "Build DataStreamQueryAnalyzer (Release)",
-      "group": "build",
-      "type": "process",
-      "command": "",
-      "windows": {
-        "command": "tools/build_data_stream_query_analyzer.cmd",
-      },
-      "args": [
-        "--configuration",
-        "Release",
-      ],
-      "icon": {
-        "id": "terminal",
-        "color": "terminal.ansiCyan"
-      },
-    },
-    {
-      "label": "Build DataStream WebApp",
-      "group": "build",
-      "type": "process",
-      "command": "",
-      "linux": {
-        "command": "tools/build_data_stream_web_app.sh",
-      },
-      "windows": {
-        "command": "tools/build_data_stream_web_app.cmd",
-      },
-      "args": [
-        "--configuration",
-        "${input:buildConfiguration}",
-      ],
-      "icon": {
-        "id": "terminal",
-        "color": "terminal.ansiCyan"
-      },
-    },
-    {
-      "label": "Build DataStream WebApp (Debug)",
-      "group": "build",
-      "type": "process",
-      "command": "",
-      "linux": {
-        "command": "tools/build_data_stream_web_app.sh",
-      },
-      "windows": {
-        "command": "tools/build_data_stream_web_app.cmd",
-      },
-      "args": [
-        "--configuration",
-        "Debug",
-      ],
-      "icon": {
-        "id": "terminal",
-        "color": "terminal.ansiCyan"
-      },
-    },
-    {
-      "label": "Build DataStream WebApp (Release)",
-      "group": "build",
-      "type": "process",
-      "command": "",
-      "linux": {
-        "command": "tools/build_data_stream_web_app.sh",
-      },
-      "windows": {
-        "command": "tools/build_data_stream_web_app.cmd",
-      },
-      "args": [
-        "--configuration",
-        "Release",
-      ],
-      "icon": {
-        "id": "terminal",
-        "color": "terminal.ansiCyan"
-      },
-    },
    {
      "label": "Build Spark",
      "group": "build",
@@ -414,22 +303,6 @@
        "color": "terminal.ansiCyan"
      },
    },
-    {
-      "label": "Publish DataStreamWebApp",
-      "group": "build",
-      "type": "process",
-      "command": "",
-      "linux": {
-        "command": "tools/publish_data_stream_web_app.sh",
-      },
-      "windows": {
-        "command": "tools/publish_data_stream_web_app.cmd",
-      },
-      "icon": {
-        "id": "terminal",
-        "color": "terminal.ansiCyan"
-      },
-    },
    {
      "label": "Edit DataStream Config",
      "type": "process",
@@ -480,7 +353,7 @@
    },
    {
      "label": "Run DataStreamQueryAnalyzer",
-      "dependsOn": "Build DataStreamQueryAnalyzer",
+      "dependsOn": "Build DataStream",
      "type": "process",
      "command": "",
      "windows": {
@@ -500,7 +373,7 @@
    },
    {
      "label": "Run DataStreamQueryAnalyzer (Debug)",
-      "dependsOn": "Build DataStreamQueryAnalyzer (Debug)",
+      "dependsOn": "Build DataStream (Debug)",
      "type": "process",
      "command": "",
      "windows": {
@@ -520,7 +393,7 @@
    },
    {
      "label": "Run DataStreamQueryAnalyzer (Release)",
-      "dependsOn": "Build DataStreamQueryAnalyzer (Release)",
+      "dependsOn": "Build DataStream (Release)",
      "type": "process",
      "command": "",
      "windows": {
@@ -607,89 +480,6 @@
        "color": "terminal.ansiGreen",
      },
    },
-    {
-      "label": "Run DataStream WebApp (Debug)",
-      "dependsOn": "Build DataStream WebApp (Debug)",
-      "command": "",
-      "linux": {
-        "command": "Bts.DataStreamWebApp.Server",
-      },
-      "windows": {
-        "command": "Bts.DataStreamWebApp.Server.exe",
-      },
-      "args": [
-        "--config",
-        "${workspaceFolder}/src/DataStreamSuite/Bts.DataStreamWebApp.Server/Configurations/DevServer.json",
-      ],
-      "options": {
-        "cwd": "${workspaceFolder}/build/Debug/bin/net7.0",
-        "env": {
-          "ASPNETCORE_HTTP_PORT": "8080",
-          "ASPNETCORE_HTTPS_PORT": "8443",
-          "ASPNETCORE_ENVIRONMENT": "Development",
-          "ASPNETCORE_HOSTINGSTARTUPASSEMBLIES": "Microsoft.AspNetCore.SpaProxy",
-        }
-      },
-      "icon": {
-        "id": "run",
-        "color": "terminal.ansiGreen",
-      },
-    },
-    {
-      "label": "Run DataStream WebApp (Release)",
-      "dependsOn": "Build DataStream WebApp (Release)",
-      "command": "",
-      "linux": {
-        "command": "Bts.DataStreamWebApp.Server",
-      },
-      "windows": {
-        "command": "Bts.DataStreamWebApp.Server.exe",
-      },
-      "args": [
-        "--config",
-        "${workspaceFolder}/src/DataStreamSuite/Bts.DataStreamWebApp.Server/Configurations/DevServer.json",
-      ],
-      "options": {
-        "cwd": "${workspaceFolder}/build/Release/bin/net7.0",
-        "env": {
-          "ASPNETCORE_HTTP_PORT": "8080",
-          "ASPNETCORE_HTTPS_PORT": "8443",
-          "ASPNETCORE_ENVIRONMENT": "Production",
-          "ASPNETCORE_HOSTINGSTARTUPASSEMBLIES": "Microsoft.AspNetCore.SpaProxy",
-        }
-      },
-      "icon": {
-        "id": "run",
-        "color": "terminal.ansiGreen",
-      },
-    },
-    {
-      "label": "Run DataStream WebApp (Release | Skip Build)",
-      "command": "",
-      "linux": {
-        "command": "Bts.DataStreamWebApp.Server",
-      },
-      "windows": {
-        "command": "Bts.DataStreamWebApp.Server.exe",
-      },
-      "args": [
-        "--config",
-        "${workspaceFolder}/src/DataStreamSuite/Bts.DataStreamWebApp.Server/Configurations/DevServer.json",
-      ],
-      "options": {
-        "cwd": "${workspaceFolder}/build/Release/bin/net7.0",
-        "env": {
-          "ASPNETCORE_HTTP_PORT": "8080",
-          "ASPNETCORE_HTTPS_PORT": "8443",
-          "ASPNETCORE_ENVIRONMENT": "Production",
-          "ASPNETCORE_HOSTINGSTARTUPASSEMBLIES": "Microsoft.AspNetCore.SpaProxy",
-        }
-      },
-      "icon": {
-        "id": "run",
-        "color": "terminal.ansiGreen",
-      },
-    },
    {
      "label": "Run PubSubServer",
      "dependsOn": "Build DataStream",
@@ -851,6 +641,21 @@
        "color": "terminal.ansiGreen",
      }
    },
+    {
+      "label": "Run Spark (Skip Build)",
+      "type": "process",
+      "command": "",
+      "windows": {
+        "command": "Spark.exe",
+        "options": {
+          "cwd": "${workspaceFolder}/build/${input:buildConfiguration}/bin/net7.0-windows"
+        },
+      },
+      "icon": {
+        "id": "run",
+        "color": "terminal.ansiGreen",
+      }
+    },
    {
      "label": "Run Spark Config Query",
      "dependsOn": "Build Spark",
@@ -894,6 +699,24 @@
        "color": "terminal.ansiGreen",
      },
    },
+    {
+      "label": "Run Sparkd (Skip Build)",
+      "type": "process",
+      "command": "",
+      "linux": {
+        "command": "sparkd",
+      },
+      "args": [
+        "run"
+      ],
+      "options": {
+        "cwd": "${workspaceFolder}/build/${input:sparkdCmakePreset}/server"
+      },
+      "icon": {
+        "id": "run",
+        "color": "terminal.ansiGreen",
+      },
+    },
    {
      "label": "Run Sparkd Config Query",
      "dependsOn": "Build Sparkd",
@@ -915,6 +738,65 @@
        "color": "terminal.ansiGreen",
      },
    },
+    {
+      "label": "Test Spark (Debug)",
+      "dependsOn": "Build Spark (Debug)",
+      "type": "process",
+      "icon": {
+        "id": "coverage",
+        "color": "terminal.ansiYellow",
+      },
+      "command": "",
+      "windows": {
+        "command": "Spark.Console.exe",
+        "args": [
+          "test",
+          "${input:sparkTest}"
+        ],
+        "options": {
+          "cwd": "${workspaceFolder}/build/Debug/bin/net7.0",
+        }
+      },
+    },
+    {
+      "label": "Test Sparkd",
+      "dependsOn": "Build Sparkd",
+      "type": "process",
+      "command": "",
+      "linux": {
+        "command": "sparkd",
+      },
+      "args": [
+        "test",
+        "${input:sparkdTest}",
+      ],
+      "options": {
+        "cwd": "${workspaceFolder}/build/${input:sparkdCmakePreset}/server"
+      },
+      "icon": {
+        "id": "coverage",
+        "color": "terminal.ansiYellow",
+      },
+    },
+    {
+      "label": "Test Sparkd (Skip Build)",
+      "type": "process",
+      "command": "",
+      "linux": {
+        "command": "sparkd",
+      },
+      "args": [
+        "test",
+        "${input:sparkdTest}",
+      ],
+      "options": {
+        "cwd": "${workspaceFolder}/build/${input:sparkdCmakePreset}/server"
+      },
+      "icon": {
+        "id": "coverage",
+        "color": "terminal.ansiYellow",
+      },
+    },
    {
      "label": "View DataStreamQueryAnalyzer Logs (Last Run)",
      "type": "process",
@@ -1033,6 +915,16 @@
      "id": "sparkdConfigQuery",
      "type": "promptString",
      "description": "Config Type(s)"
-    }
+    },
+    {
+      "id": "sparkTest",
+      "type": "promptString",
+      "description": "Test to run",
+    },
+    {
+      "id": "sparkdTest",
+      "type": "promptString",
+      "description": "Test to run",
+    },
  ]
 }
@@ -1,7 +1,10 @@
 cmake_minimum_required(VERSION 3.12)
 project(spark)
+include(GNUInstallDirs)
+include(${CMAKE_SOURCE_DIR}/cmake/GitVersion.cmake)
 set(CMAKE_CXX_STANDARD 20)

+get_version_from_git()
 if(WIN32)
  set(platform "windows")
 else()
@@ -18,7 +21,7 @@ set(
 # SHARED_MEMORY_PREFIX serves as a kind of namespace so multiple applications
 # can run simultaneously without fighting over the same shm files (e.g., sparkd,
 # api server, etc.).
-set(SHARED_MEMORY_PREFIX "\${BTS_SHARED_MEMORY_PREFIX}" CACHE STRING "Prefix to use when creating shared memory files")
+set(SHARED_MEMORY_PREFIX "bts" CACHE STRING "Prefix to use when creating shared memory files")
 set(EXTERNALS_BIN "${EXTERNALS_ROOT}/bin")
 set(EXTERNALS_INCLUDE "${EXTERNALS_ROOT}/include")
 set(EXTERNALS_LIB "${EXTERNALS_ROOT}/lib/$<CONFIG>")
@@ -37,8 +40,9 @@ include_directories(BEFORE SYSTEM ${EXTERNALS_INCLUDE} ${EXTERNALS_ROOT}/include
 link_directories(${EXTERNALS_LIB})

 add_compile_definitions(SHARED_MEMORY_PREFIX="${SHARED_MEMORY_PREFIX}")
-if(DEFINED APP_DATA_DIR)
-  add_compile_definitions(APP_DATA_DIR="${APP_DATA_DIR}")
+add_compile_definitions(APP_DATA_DIR="${APP_DATA_DIR}")
+if(DEFINED APP_DATA_DIR_OVERRIDE)
+  add_compile_definitions(APP_DATA_DIR_OVERRIDE="${APP_DATA_DIR_OVERRIDE}")
 endif()
 add_compile_definitions(EXPORT_MD_CONSUMER="${EXPORT_MD_CONSUMER}")
 if(WIN32)
@@ -82,9 +86,15 @@ else()
    add_compile_definitions(BINANCE_US_PRICE_STATS_PATH="/api/v3/ticker/24hr")

    # ** Binance USDS Futures **
-    add_compile_definitions(CCAPI_BINANCE_USDS_FUTURES_URL_REST_BASE="https://dapi.binance.com")
-    add_compile_definitions(BINANCE_USDS_FUTURES_INSTRUMENTS_PATH="/dapi/v1/exchangeInfo")
-    add_compile_definitions(BINANCE_USDS_FUTURES_PRICE_STATS_PATH="/dapi/v1/ticker/24hr")
+    add_compile_definitions(CCAPI_BINANCE_USDS_FUTURES_URL_REST_BASE="https://fapi.binance.com")
+    add_compile_definitions(BINANCE_USDS_FUTURES_INSTRUMENTS_PATH="/fapi/v1/exchangeInfo")
+    add_compile_definitions(BINANCE_USDS_FUTURES_PRICE_STATS_PATH="/fapi/v1/ticker/24hr")
+    add_compile_definitions(BINANCE_USDS_FUTURES_USER_DATA_PATH="/api/v3/userDataStream")
+
+    # ** Binance Coin Futures **
+    add_compile_definitions(CCAPI_BINANCE_COIN_FUTURES_URL_REST_BASE="https://dapi.binance.com")
+    add_compile_definitions(BINANCE_COIN_FUTURES_INSTRUMENTS_PATH="/dapi/v1/exchangeInfo")
+    add_compile_definitions(BINANCE_COIN_FUTURES_PRICE_STATS_PATH="/dapi/v1/ticker/24hr")

    # ** Deribit **
    add_compile_definitions(DERIBIT_INSTRUMENT_PATH="/api/v2/public/get_instrument")
@@ -103,15 +113,15 @@ else()
  endif()

  # subdirs
+  add_subdirectory(src/lib_spark)
  if (BUILD_SPARKD)
-    add_subdirectory(src/lib_system)
-    add_subdirectory(src/lib_domain)
-    add_subdirectory(src/lib_python_md)
-    add_subdirectory(src/lib_logging)
+    add_subdirectory(src/lib_python_interop)
+    add_subdirectory(src/lib_onixs_fix)
+    add_subdirectory(src/lib_binance_execution)
+    add_subdirectory(src/lib_okx_execution)
    add_subdirectory(src/app_sparkd)
    add_subdirectory(src/data_files)
  endif()
-  add_subdirectory(src/lib_spark)
  if (BUILD_ADMIN_WEB_APP)
    add_subdirectory(src/Spark.Admin.WebApp)
  endif()
@@ -20,20 +20,24 @@
        "SPARK_CMAKE_CXX_FLAGS": "/MP /WX /wd4250 /wd4251 /wd4244 /wd4267 /wd4355 /wd4275 /wd4345 /bigobj /EHsc /DCURL_STATICLIB /D_WINSOCK_DEPRECATED_NO_WARNINGS /D__TBB_NO_IMPLICIT_LINKAGE /DBOOST_ALL_NO_LIB /std:c++20 /D_SCL_SECURE_NO_WARNINGS /D_CRT_SECURE_NO_WARNINGS /D_WIN32_WINNT=0x0600"
      },
      "cacheVariables": {
+        "APP_DATA_DIR": "${dollar}{APPDATA}/bts",
+        "APP_DATA_DIR_OVERRIDE": "${dollar}{SPARK_DATA_DIR}",
        "CMAKE_CXX_FLAGS": "$env{SPARK_CMAKE_CXX_FLAGS}",
        "CMAKE_CXX_FLAGS_DEBUG": "$env{SPARK_CMAKE_CXX_FLAGS} /MTd",
        "CMAKE_CXX_FLAGS_RELEASE": "$env{SPARK_CMAKE_CXX_FLAGS} /MT",
        "CMAKE_CXX_STANDARD": "20",
+        "CMAKE_INSTALL_PREFIX": "${sourceDir}/install/${presetName}",
        "CMAKE_SHARED_LINKER_FLAGS_DEBUG": "/DEBUG",
        "CMAKE_SHARED_LINKER_FLAGS_RELEASE": "/OPT:REF /OPT:ICF /LTCG",
        "CMAKE_TOOLCHAIN_FILE": "${sourceDir}/vcpkg/scripts/buildsystems/vcpkg.cmake",
        "EXTERNALS_ROOT": "${sourceDir}/externals/windows",
-        "CONFIG_DEFAULTS_PATH": "config/profiles/default/*.conf",
-        "DOTNET_BINARY_DIR": "${sourceDir}/build/$<CONFIG>/bin/net7.0;${sourceDir}/build/$<CONFIG>/bin/net7.0-windows",
+        "CONFIG_DEFAULTS_PATH": "spark/config/profiles/default/*.conf",
+        "DOTNET_BINARY_DIR": "${sourceDir}/build/$<CONFIG>/bin/net8.0;${sourceDir}/build/$<CONFIG>/bin/net8.0-windows",
        "EXPORT_MD_CONSUMER": {
          "type": "BOOL",
          "value": "ON"
-        }
+        },
+        "SHARED_MEMORY_PREFIX": "spark"
      }
    },

@@ -49,10 +53,22 @@
      "binaryDir": "${sourceDir}/build/${presetName}",
      "cacheVariables": {
        "platform": "",
+        "APP_DATA_DIR": "${dollar}{HOME}/sparkdata",
+        "APP_DATA_DIR_OVERRIDE": "${dollar}{SPARKD_DATA_DIR}",
+        "CMAKE_BUILD_WITH_INSTALL_RPATH": {
+          "type": "BOOL",
+          "value": "OFF"
+        },
        "CMAKE_CXX_STANDARD": "20",
+        "CMAKE_INSTALL_RPATH": ".",
        "CMAKE_INSTALL_PREFIX": "${sourceDir}/install/${presetName}",
        "CMAKE_PREFIX_PATH": "${sourceDir}/cmake/;${sourceDir}/.venv/lib/python3.10/site-packages/pybind11/share/cmake/pybind11",
        "CMAKE_EXE_LINKER_FLAGS": "-Wno-undef",
+        "CMAKE_TOOLCHAIN_FILE": "${sourceDir}/vcpkg/scripts/buildsystems/vcpkg.cmake",
+        "DATABENTO_USE_EXTERNAL_JSON": {
+          "type": "BOOL",
+          "value": "ON"
+        },
        "EXTERNALS_ROOT": "${sourceDir}/externals/linux",
        "BUILD_SDK": {
          "type": "BOOL",
@@ -61,7 +77,8 @@
        "BUILD_ADMIN_WEB_APP": {
          "type": "BOOL",
          "value": "ON"
-        }
+        },
+        "SHARED_MEMORY_PREFIX": "sparkd"
      }
    },

@@ -76,7 +93,7 @@
          "value": "ON"
        },
        "CCAPI_LOG_LEVEL": "Info",
-        "CONFIG_DEFAULTS_PATH": "config/profiles/md_dev/*.conf;config/profiles/default/*.conf"
+        "CONFIG_DEFAULTS_PATH": "spark/config/profiles/md_dev/*.conf;spark/config/profiles/default/*.conf"
      }
    },

@@ -87,7 +104,7 @@
      "cacheVariables": {
        "CMAKE_BUILD_TYPE": "Release",
        "CCAPI_LOG_LEVEL": "Warn",
-        "CONFIG_DEFAULTS_PATH": "config/profiles/md_prod/*.conf;config/profiles/default/*.conf"
+        "CONFIG_DEFAULTS_PATH": "spark/config/profiles/md_prod/*.conf;spark/config/profiles/default/*.conf"
      }
    },

@@ -142,7 +159,11 @@
    {
      "name": "spark-api-server-windows",
      "inherits": "spark",
-      "displayName": "Spark Api Server (Windows)"
+      "displayName": "Spark Api Server (Windows)",
+      "cacheVariables": {
+        "APP_DATA_DIR_OVERRIDE": "${dollar}{SPARK_API_SERVER_DATA_DIR}",
+        "SHARED_MEMORY_PREFIX": "spark_api_server"
+      }
    },

    {
@@ -151,6 +172,7 @@
      "displayName": "Spark Api Server (Linux)",
      "generator": "Ninja Multi-Config",
      "cacheVariables": {
+        "APP_DATA_DIR_OVERRIDE": "${dollar}{SPARK_API_SERVER_DATA_DIR}",
        "BUILD_ADMIN_WEB_APP": {
          "type": "BOOL",
          "value": "OFF"
@@ -163,12 +185,13 @@
        "CMAKE_CXX_COMPILER": "clang++",
        "CMAKE_CXX_FLAGS_DEBUG": "-g -fPIC -Wall -pedantic -Wextra -Werror -Wno-missing-braces -Wno-ignored-qualifiers --std=c++20 -fdiagnostics-color -DDEBUG",
        "CMAKE_CXX_FLAGS_RELEASE": "-fPIC -Wall -pedantic -Wextra -Werror -Wno-missing-braces -Wno-ignored-qualifiers --std=c++20 -fdiagnostics-color -fno-omit-frame-pointer -DNDEBUG -O3",
+        "CMAKE_INSTALL_PREFIX": "${sourceDir}/install/${presetName}",
        "CMAKE_EXPORT_COMPILE_COMMANDS": {
          "type": "BOOL",
          "value": "ON"
        },
-        "CONFIG_DEFAULTS_PATH": "config/profiles/default/*.conf",
-        "DOTNET_BINARY_DIR": "${sourceDir}/build/Spark.Api.Server/$<CONFIG>/bin/net7.0",
+        "CONFIG_DEFAULTS_PATH": "spark/config/profiles/default/*.conf",
+        "DOTNET_BINARY_DIR": "${sourceDir}/build/$<CONFIG>/bin/net8.0",
        "EXPORT_MD_CONSUMER": {
          "type": "BOOL",
          "value": "ON"
@@ -206,8 +229,10 @@
      "description": "Data Stream utils ported over from Edge",
      "binaryDir": "${sourceDir}/build/${presetName}",
      "cacheVariables": {
-        "CONFIG_DEFAULTS_PATH": "config/profiles/default/*.conf;config/profiles/data_stream/*.conf",
-        "DOTNET_BINARY_DIR": "${sourceDir}/build/$<CONFIG>/bin/net7.0;${sourceDir}/build/$<CONFIG>/bin/net7.0-windows"
+        "APP_DATA_DIR": "${dollar}{APPDATA}/bts/DataStream",
+        "APP_DATA_DIR_OVERRIDE": "${dollar}{DATA_STREAM_DATA_DIR}",
+        "CONFIG_DEFAULTS_PATH": "spark/config/profiles/default/*.conf;spark/config/profiles/data_stream/*.conf",
+        "SHARED_MEMORY_PREFIX": "data_stream"
      }
    },

@@ -223,6 +248,8 @@
      "inherits": "sparkd",
      "generator": "Ninja Multi-Config",
      "cacheVariables": {
+        "APP_DATA_DIR": "${dollar}{HOME}/.local/share/DataStream",
+        "APP_DATA_DIR_OVERRIDE": "${dollar}{DATA_STREAM_DATA_DIR}",
        "BUILD_ADMIN_WEB_APP": {
          "type": "BOOL",
          "value": "OFF"
@@ -239,13 +266,13 @@
        "CMAKE_CXX_COMPILER": "clang++",
        "CMAKE_CXX_FLAGS_DEBUG": "-g -fPIC -Wall -pedantic -Wextra -Werror -Wno-missing-braces -Wno-ignored-qualifiers --std=c++20 -fdiagnostics-color -DDEBUG",
        "CMAKE_CXX_FLAGS_RELEASE": "-fPIC -Wall -pedantic -Wextra -Werror -Wno-missing-braces -Wno-ignored-qualifiers --std=c++20 -fdiagnostics-color -fno-omit-frame-pointer -DNDEBUG -O3",
-        "CONFIG_DEFAULTS_PATH": "config/profiles/default/*.conf;config/profiles/data_stream/*.conf",
-        "CONFIG_PROFILES": "default;data_stream",
-        "DOTNET_BINARY_DIR": "${sourceDir}/build/$<CONFIG>/bin/net7.0;${sourceDir}/build/$<CONFIG>/bin/net8.0",
+        "CONFIG_DEFAULTS_PATH": "spark/config/profiles/default/*.conf;spark/config/profiles/data_stream/*.conf",
+        "DOTNET_BINARY_DIR": "${sourceDir}/build/$<CONFIG>/bin/net7.0",
        "EXPORT_MD_CONSUMER": {
          "type": "BOOL",
          "value": "ON"
-        }
+        },
+        "SHARED_MEMORY_PREFIX": "data_stream"
      }
    }
  ],
@@ -294,14 +321,6 @@
      "inherits": "sparkd"
    },

-    {
-      "name": "sparkd-debug-clang-j6",
-      "configurePreset": "sparkd-debug-clang",
-      "displayName": "Sparkd (Debug | Clang | jobs=6)",
-      "inherits": "sparkd",
-      "jobs": 6
-    },
-
    {
      "name": "sparkd-debug-gcc",
      "configurePreset": "sparkd-debug-gcc",
@@ -14,6 +14,34 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "spark_test", "src\client\sp
 EndProject
 Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Spark.Console", "src\client\Spark.Console\Spark.Console.csproj", "{89D666E6-B500-498F-B026-4D167F68ADFC}"
 EndProject
+Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "src", "src", "{267F81EF-A8AB-49FD-9F56-D7EC3C1F7C7C}"
+EndProject
+Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "DataStreamSuite", "DataStreamSuite", "{C09ED521-D2DF-40BC-A76F-EB3CB870B2F6}"
+EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "DataStreamCore", "src\DataStreamSuite\DataStreamCore\DataStreamCore.csproj", "{EF26A475-6101-4C9C-A717-D60BB92A2302}"
+EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "SharedStreams", "src\DataStreamSuite\SharedStreams\SharedStreams.csproj", "{911B08A3-515A-441D-84C8-8A43740B5FC1}"
+EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "PubSub", "src\DataStreamSuite\PubSub\PubSub.csproj", "{14140310-2233-430A-821B-F3058A25234B}"
+EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "BaseUtilities", "src\DataStreamSuite\BaseUtilities\BaseUtilities.csproj", "{43146EA8-6FCC-46B8-88B0-42F9BB5E6E0A}"
+EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "ExpressionCore", "src\DataStreamSuite\ExpressionCore\ExpressionCore.csproj", "{6F22DB96-6EAB-4034-99D9-287021C36F01}"
+EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "ArithmeticExpressions", "src\DataStreamSuite\ArithmeticExpressions\ArithmeticExpressions.csproj", "{A5C33956-A41F-49B3-A41D-1082CBA59D02}"
+EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "DataStreamExpressions", "src\DataStreamSuite\DataStreamExpressions\DataStreamExpressions.csproj", "{2FE745C3-1EDF-4421-8CD7-A63EE0C1AE43}"
+EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Bts.ColumnSortExpressions", "src\DataStreamSuite\Bts.ColumnSortExpressions\Bts.ColumnSortExpressions.csproj", "{FDA791E0-2B91-4AD5-B91D-CF920A4C4E38}"
+EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "ColumnMapExpressions", "src\DataStreamSuite\ColumnMapExpressions\ColumnMapExpressions.csproj", "{B945A183-2174-4ACB-B289-8A5C546656FE}"
+EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "BooleanExpressions", "src\DataStreamSuite\BooleanExpressions\BooleanExpressions.csproj", "{9F6D7DE8-185B-4C1D-B809-3314D092DF28}"
+EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Bts.ColumnExpressions", "src\DataStreamSuite\Bts.ColumnExpressions\Bts.ColumnExpressions.csproj", "{6BD3F6D8-B6FC-44E8-B8FE-06352F638B9B}"
+EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "DataStream", "src\DataStreamSuite\DataStream\DataStream.csproj", "{39AEA075-6457-46C0-8821-1158BE6EFA57}"
+EndProject
 Global
 	GlobalSection(SolutionConfigurationPlatforms) = preSolution
 		Debug|x64 = Debug|x64
@@ -70,6 +98,102 @@ Global
 		{89D666E6-B500-498F-B026-4D167F68ADFC}.Release|x64.Build.0 = Release|x64
 		{89D666E6-B500-498F-B026-4D167F68ADFC}.RelWithDebInfo|x64.ActiveCfg = Debug|x64
 		{89D666E6-B500-498F-B026-4D167F68ADFC}.RelWithDebInfo|x64.Build.0 = Debug|x64
+		{EF26A475-6101-4C9C-A717-D60BB92A2302}.Debug|x64.ActiveCfg = Debug|Any CPU
+		{EF26A475-6101-4C9C-A717-D60BB92A2302}.Debug|x64.Build.0 = Debug|Any CPU
+		{EF26A475-6101-4C9C-A717-D60BB92A2302}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
+		{EF26A475-6101-4C9C-A717-D60BB92A2302}.MinSizeRel|x64.Build.0 = Debug|Any CPU
+		{EF26A475-6101-4C9C-A717-D60BB92A2302}.Release|x64.ActiveCfg = Release|Any CPU
+		{EF26A475-6101-4C9C-A717-D60BB92A2302}.Release|x64.Build.0 = Release|Any CPU
+		{EF26A475-6101-4C9C-A717-D60BB92A2302}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
+		{EF26A475-6101-4C9C-A717-D60BB92A2302}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
+		{911B08A3-515A-441D-84C8-8A43740B5FC1}.Debug|x64.ActiveCfg = Debug|Any CPU
+		{911B08A3-515A-441D-84C8-8A43740B5FC1}.Debug|x64.Build.0 = Debug|Any CPU
+		{911B08A3-515A-441D-84C8-8A43740B5FC1}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
+		{911B08A3-515A-441D-84C8-8A43740B5FC1}.MinSizeRel|x64.Build.0 = Debug|Any CPU
+		{911B08A3-515A-441D-84C8-8A43740B5FC1}.Release|x64.ActiveCfg = Release|Any CPU
+		{911B08A3-515A-441D-84C8-8A43740B5FC1}.Release|x64.Build.0 = Release|Any CPU
+		{911B08A3-515A-441D-84C8-8A43740B5FC1}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
+		{911B08A3-515A-441D-84C8-8A43740B5FC1}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
+		{14140310-2233-430A-821B-F3058A25234B}.Debug|x64.ActiveCfg = Debug|Any CPU
+		{14140310-2233-430A-821B-F3058A25234B}.Debug|x64.Build.0 = Debug|Any CPU
+		{14140310-2233-430A-821B-F3058A25234B}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
+		{14140310-2233-430A-821B-F3058A25234B}.MinSizeRel|x64.Build.0 = Debug|Any CPU
+		{14140310-2233-430A-821B-F3058A25234B}.Release|x64.ActiveCfg = Release|Any CPU
+		{14140310-2233-430A-821B-F3058A25234B}.Release|x64.Build.0 = Release|Any CPU
+		{14140310-2233-430A-821B-F3058A25234B}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
+		{14140310-2233-430A-821B-F3058A25234B}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
+		{43146EA8-6FCC-46B8-88B0-42F9BB5E6E0A}.Debug|x64.ActiveCfg = Debug|Any CPU
+		{43146EA8-6FCC-46B8-88B0-42F9BB5E6E0A}.Debug|x64.Build.0 = Debug|Any CPU
+		{43146EA8-6FCC-46B8-88B0-42F9BB5E6E0A}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
+		{43146EA8-6FCC-46B8-88B0-42F9BB5E6E0A}.MinSizeRel|x64.Build.0 = Debug|Any CPU
+		{43146EA8-6FCC-46B8-88B0-42F9BB5E6E0A}.Release|x64.ActiveCfg = Release|Any CPU
+		{43146EA8-6FCC-46B8-88B0-42F9BB5E6E0A}.Release|x64.Build.0 = Release|Any CPU
+		{43146EA8-6FCC-46B8-88B0-42F9BB5E6E0A}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
+		{43146EA8-6FCC-46B8-88B0-42F9BB5E6E0A}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
+		{6F22DB96-6EAB-4034-99D9-287021C36F01}.Debug|x64.ActiveCfg = Debug|Any CPU
+		{6F22DB96-6EAB-4034-99D9-287021C36F01}.Debug|x64.Build.0 = Debug|Any CPU
+		{6F22DB96-6EAB-4034-99D9-287021C36F01}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
+		{6F22DB96-6EAB-4034-99D9-287021C36F01}.MinSizeRel|x64.Build.0 = Debug|Any CPU
+		{6F22DB96-6EAB-4034-99D9-287021C36F01}.Release|x64.ActiveCfg = Release|Any CPU
+		{6F22DB96-6EAB-4034-99D9-287021C36F01}.Release|x64.Build.0 = Release|Any CPU
+		{6F22DB96-6EAB-4034-99D9-287021C36F01}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
+		{6F22DB96-6EAB-4034-99D9-287021C36F01}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
+		{A5C33956-A41F-49B3-A41D-1082CBA59D02}.Debug|x64.ActiveCfg = Debug|Any CPU
+		{A5C33956-A41F-49B3-A41D-1082CBA59D02}.Debug|x64.Build.0 = Debug|Any CPU
+		{A5C33956-A41F-49B3-A41D-1082CBA59D02}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
+		{A5C33956-A41F-49B3-A41D-1082CBA59D02}.MinSizeRel|x64.Build.0 = Debug|Any CPU
+		{A5C33956-A41F-49B3-A41D-1082CBA59D02}.Release|x64.ActiveCfg = Release|Any CPU
+		{A5C33956-A41F-49B3-A41D-1082CBA59D02}.Release|x64.Build.0 = Release|Any CPU
+		{A5C33956-A41F-49B3-A41D-1082CBA59D02}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
+		{A5C33956-A41F-49B3-A41D-1082CBA59D02}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
+		{2FE745C3-1EDF-4421-8CD7-A63EE0C1AE43}.Debug|x64.ActiveCfg = Debug|Any CPU
+		{2FE745C3-1EDF-4421-8CD7-A63EE0C1AE43}.Debug|x64.Build.0 = Debug|Any CPU
+		{2FE745C3-1EDF-4421-8CD7-A63EE0C1AE43}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
+		{2FE745C3-1EDF-4421-8CD7-A63EE0C1AE43}.MinSizeRel|x64.Build.0 = Debug|Any CPU
+		{2FE745C3-1EDF-4421-8CD7-A63EE0C1AE43}.Release|x64.ActiveCfg = Release|Any CPU
+		{2FE745C3-1EDF-4421-8CD7-A63EE0C1AE43}.Release|x64.Build.0 = Release|Any CPU
+		{2FE745C3-1EDF-4421-8CD7-A63EE0C1AE43}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
+		{2FE745C3-1EDF-4421-8CD7-A63EE0C1AE43}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
+		{FDA791E0-2B91-4AD5-B91D-CF920A4C4E38}.Debug|x64.ActiveCfg = Debug|Any CPU
+		{FDA791E0-2B91-4AD5-B91D-CF920A4C4E38}.Debug|x64.Build.0 = Debug|Any CPU
+		{FDA791E0-2B91-4AD5-B91D-CF920A4C4E38}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
+		{FDA791E0-2B91-4AD5-B91D-CF920A4C4E38}.MinSizeRel|x64.Build.0 = Debug|Any CPU
+		{FDA791E0-2B91-4AD5-B91D-CF920A4C4E38}.Release|x64.ActiveCfg = Release|Any CPU
+		{FDA791E0-2B91-4AD5-B91D-CF920A4C4E38}.Release|x64.Build.0 = Release|Any CPU
+		{FDA791E0-2B91-4AD5-B91D-CF920A4C4E38}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
+		{FDA791E0-2B91-4AD5-B91D-CF920A4C4E38}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
+		{B945A183-2174-4ACB-B289-8A5C546656FE}.Debug|x64.ActiveCfg = Debug|Any CPU
+		{B945A183-2174-4ACB-B289-8A5C546656FE}.Debug|x64.Build.0 = Debug|Any CPU
+		{B945A183-2174-4ACB-B289-8A5C546656FE}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
+		{B945A183-2174-4ACB-B289-8A5C546656FE}.MinSizeRel|x64.Build.0 = Debug|Any CPU
+		{B945A183-2174-4ACB-B289-8A5C546656FE}.Release|x64.ActiveCfg = Release|Any CPU
+		{B945A183-2174-4ACB-B289-8A5C546656FE}.Release|x64.Build.0 = Release|Any CPU
+		{B945A183-2174-4ACB-B289-8A5C546656FE}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
+		{B945A183-2174-4ACB-B289-8A5C546656FE}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
+		{39AEA075-6457-46C0-8821-1158BE6EFA57}.Debug|x64.ActiveCfg = Debug|Any CPU
+		{39AEA075-6457-46C0-8821-1158BE6EFA57}.Debug|x64.Build.0 = Debug|Any CPU
+		{39AEA075-6457-46C0-8821-1158BE6EFA57}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
+		{39AEA075-6457-46C0-8821-1158BE6EFA57}.MinSizeRel|x64.Build.0 = Debug|Any CPU
+		{39AEA075-6457-46C0-8821-1158BE6EFA57}.Release|x64.ActiveCfg = Release|Any CPU
+		{39AEA075-6457-46C0-8821-1158BE6EFA57}.Release|x64.Build.0 = Release|Any CPU
+		{39AEA075-6457-46C0-8821-1158BE6EFA57}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
+		{39AEA075-6457-46C0-8821-1158BE6EFA57}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
+		{9F6D7DE8-185B-4C1D-B809-3314D092DF28}.Debug|x64.ActiveCfg = Debug|Any CPU
+		{9F6D7DE8-185B-4C1D-B809-3314D092DF28}.Debug|x64.Build.0 = Debug|Any CPU
+		{9F6D7DE8-185B-4C1D-B809-3314D092DF28}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
+		{9F6D7DE8-185B-4C1D-B809-3314D092DF28}.MinSizeRel|x64.Build.0 = Debug|Any CPU
+		{9F6D7DE8-185B-4C1D-B809-3314D092DF28}.Release|x64.ActiveCfg = Release|Any CPU
+		{9F6D7DE8-185B-4C1D-B809-3314D092DF28}.Release|x64.Build.0 = Release|Any CPU
+		{9F6D7DE8-185B-4C1D-B809-3314D092DF28}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
+		{9F6D7DE8-185B-4C1D-B809-3314D092DF28}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
+		{6BD3F6D8-B6FC-44E8-B8FE-06352F638B9B}.Debug|x64.ActiveCfg = Debug|Any CPU
+		{6BD3F6D8-B6FC-44E8-B8FE-06352F638B9B}.Debug|x64.Build.0 = Debug|Any CPU
+		{6BD3F6D8-B6FC-44E8-B8FE-06352F638B9B}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
+		{6BD3F6D8-B6FC-44E8-B8FE-06352F638B9B}.MinSizeRel|x64.Build.0 = Debug|Any CPU
+		{6BD3F6D8-B6FC-44E8-B8FE-06352F638B9B}.Release|x64.ActiveCfg = Release|Any CPU
+		{6BD3F6D8-B6FC-44E8-B8FE-06352F638B9B}.Release|x64.Build.0 = Release|Any CPU
+		{6BD3F6D8-B6FC-44E8-B8FE-06352F638B9B}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
+		{6BD3F6D8-B6FC-44E8-B8FE-06352F638B9B}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
 	EndGlobalSection
 	GlobalSection(SolutionProperties) = preSolution
 		HideSolutionNode = FALSE
@@ -80,4 +204,19 @@ Global
 	GlobalSection(Performance) = preSolution
 		HasPerformanceSessions = true
 	EndGlobalSection
+	GlobalSection(NestedProjects) = preSolution
+		{C09ED521-D2DF-40BC-A76F-EB3CB870B2F6} = {267F81EF-A8AB-49FD-9F56-D7EC3C1F7C7C}
+		{EF26A475-6101-4C9C-A717-D60BB92A2302} = {C09ED521-D2DF-40BC-A76F-EB3CB870B2F6}
+		{911B08A3-515A-441D-84C8-8A43740B5FC1} = {C09ED521-D2DF-40BC-A76F-EB3CB870B2F6}
+		{14140310-2233-430A-821B-F3058A25234B} = {C09ED521-D2DF-40BC-A76F-EB3CB870B2F6}
+		{43146EA8-6FCC-46B8-88B0-42F9BB5E6E0A} = {C09ED521-D2DF-40BC-A76F-EB3CB870B2F6}
+		{6F22DB96-6EAB-4034-99D9-287021C36F01} = {C09ED521-D2DF-40BC-A76F-EB3CB870B2F6}
+		{A5C33956-A41F-49B3-A41D-1082CBA59D02} = {C09ED521-D2DF-40BC-A76F-EB3CB870B2F6}
+		{2FE745C3-1EDF-4421-8CD7-A63EE0C1AE43} = {C09ED521-D2DF-40BC-A76F-EB3CB870B2F6}
+		{FDA791E0-2B91-4AD5-B91D-CF920A4C4E38} = {C09ED521-D2DF-40BC-A76F-EB3CB870B2F6}
+		{B945A183-2174-4ACB-B289-8A5C546656FE} = {C09ED521-D2DF-40BC-A76F-EB3CB870B2F6}
+		{39AEA075-6457-46C0-8821-1158BE6EFA57} = {C09ED521-D2DF-40BC-A76F-EB3CB870B2F6}
+		{9F6D7DE8-185B-4C1D-B809-3314D092DF28} = {C09ED521-D2DF-40BC-A76F-EB3CB870B2F6}
+		{6BD3F6D8-B6FC-44E8-B8FE-06352F638B9B} = {C09ED521-D2DF-40BC-A76F-EB3CB870B2F6}
+	EndGlobalSection
 EndGlobal
@@ -1,3 +1,5 @@
+include_guard(GLOBAL)
+
 macro(export_to_parent)
  foreach(var IN ITEMS ${ARGN})
    set(${var} ${${var}} PARENT_SCOPE)
@@ -0,0 +1,48 @@
+include_guard(GLOBAL)
+
+function(get_version_from_git)
+  find_package(Git QUIET)
+  if(NOT Git_FOUND)
+    message(WARNING "Git not found")
+    return()
+  endif()
+
+  execute_process(
+    COMMAND ${GIT_EXECUTABLE} describe --tags --always
+    WORKING_DIRECTORY ${CMAKE_SOURCE_DIR}
+    OUTPUT_VARIABLE GIT_TAG
+    OUTPUT_STRIP_TRAILING_WHITESPACE
+    RESULT_VARIABLE GIT_RESULT
+  )
+
+  if(NOT GIT_RESULT EQUAL 0)
+    message(WARNING "Failed to get git tag")
+    return()
+  endif()
+
+  execute_process(
+    COMMAND ${GIT_EXECUTABLE} rev-parse --short=7 HEAD
+    WORKING_DIRECTORY ${CMAKE_SOURCE_DIR}
+    OUTPUT_VARIABLE GIT_COMMIT_SHORT_HASH
+    OUTPUT_STRIP_TRAILING_WHITESPACE
+  )
+  set(PROJECT_VERSION_BUILD_METADATA ${GIT_COMMIT_SHORT_HASH})
+  set(PROJECT_VERSION_BUILD_METADATA ${GIT_COMMIT_SHORT_HASH} PARENT_SCOPE)
+
+  string(REGEX REPLACE "^v" "" CLEAN_TAG "${GIT_TAG}")
+  if(CLEAN_TAG MATCHES "^([0-9]+)\\.([0-9]+)\\.([0-9]+)(-.*)?$")
+    set(PROJECT_VERSION_MAJOR ${CMAKE_MATCH_1})
+    set(PROJECT_VERSION_MAJOR ${CMAKE_MATCH_1} PARENT_SCOPE)
+    set(PROJECT_VERSION_MINOR ${CMAKE_MATCH_2})
+    set(PROJECT_VERSION_MINOR ${CMAKE_MATCH_2} PARENT_SCOPE)
+    set(PROJECT_VERSION_PATCH ${CMAKE_MATCH_3})
+    set(PROJECT_VERSION_PATCH ${CMAKE_MATCH_3} PARENT_SCOPE)
+
+    set(PROJECT_VERSION "${CMAKE_MATCH_1}.${CMAKE_MATCH_2}.${CMAKE_MATCH_3}") 
+    set(PROJECT_VERSION "${CMAKE_MATCH_1}.${CMAKE_MATCH_2}.${CMAKE_MATCH_3}" PARENT_SCOPE)
+    set(PROJECT_VERSION_FULL "${CMAKE_MATCH_1}.${CMAKE_MATCH_2}.${CMAKE_MATCH_3}+${GIT_COMMIT_SHORT_HASH}")
+    set(PROJECT_VERSION_FULL "${CMAKE_MATCH_1}.${CMAKE_MATCH_2}.${CMAKE_MATCH_3}+${GIT_COMMIT_SHORT_HASH}" PARENT_SCOPE)
+  else()
+    message(WARNING "Tag '${CLEAN_TAG}' does not match semver format")
+  endif()
+endfunction()
@@ -1,66 +1,142 @@
 include_guard(DIRECTORY)
 include(${CMAKE_SOURCE_DIR}/cmake/ExportToParent.cmake)
-include(${CMAKE_SOURCE_DIR}/cmake/TernaryOperator.cmake)
+include(${CMAKE_SOURCE_DIR}/cmake/InstallBasic.cmake)
+include(${CMAKE_SOURCE_DIR}/cmake/ProtobufGenerateCpp.cmake)
+include(${CMAKE_SOURCE_DIR}/cmake/SetFirstDefined.cmake)
+include(${CMAKE_SOURCE_DIR}/cmake/safe_dep_glob.cmake)

 function(init_library)
  set(_options SHARED STATIC)
-  set(_one_value_args SOURCE_DIR)
+  set(_one_value_args
+    NAME
+    SOURCE_DIR
+    INCLUDE_DIR
+  )
  set(_multi_value_args)
  cmake_parse_arguments(_ARGS "${_options}" "${_one_value_args}" "${_multi_value_args}" ${ARGN})
-
  if (_ARGS_SHARED AND _ARGS_STATIC)
    message(FATAL_ERROR "Invalid args passed to init_library: cannot specify STATIC and SHARED")
  endif()
+  set_first_defined(_name _ARGS_NAME PROJECT_NAME)
+  set_first_defined(_source_dir _ARGS_SOURCE_DIR _name)
+  set_first_defined(_include_dir _ARGS_INCLUDE_DIR _source_dir)

-  set_ternary(_source_dir _ARGS_SOURCE_DIR _ARGS_SOURCE_DIR ${PROJECT_NAME})
-  # Glob cpp sources
+  # Cpp sources
  file(
-    GLOB_RECURSE ${PROJECT_NAME}_SOURCES
+    GLOB_RECURSE ${_name}_SOURCES
    RELATIVE ${CMAKE_CURRENT_SOURCE_DIR}
    ${_source_dir}/*.cpp
  )
  # Note: cpp files inside 'tests' dir(s) are considered tests.
  # Remove tests from main sources, and add them to test sources
-  foreach(item ${${PROJECT_NAME}_SOURCES})
-    if(${item} MATCHES "tests/*.cpp")
-      list(REMOVE_ITEM ${PROJECT_NAME}_SOURCES ${item})
-      list(APPEND ${PROJECT_NAME}_TEST_SOURCES ${item})
+  foreach(item ${${_name}_SOURCES})
+    if(${item} MATCHES "tests/.*\.cpp$")
+      list(APPEND ${_name}_TEST_SOURCES ${item})
    endif()
  endforeach()
+  foreach(item ${${_name}_TEST_SOURCES})
+    list(REMOVE_ITEM ${_name}_SOURCES ${item})
+  endforeach()

-  export_to_parent(
-    ${PROJECT_NAME}_SOURCES
-    ${PROJECT_NAME}_TEST_SOURCES
+  # Proto sources 
+  file(
+    GLOB_RECURSE ${_name}_PROTO_SOURCES
+    RELATIVE ${CMAKE_CURRENT_SOURCE_DIR}
+    ${_source_dir}/protobufs/*.proto
+  )
+  update_deps_file("${${_name}_PROTO_SOURCES};")
+  protobuf_generate_cpp(
+    CXX_SOURCES ${_name}_CXX_PROTO_SOURCES
+    CXX_HEADERS ${_name}_CXX_PROTO_HEADERS
+    CXX_OUT_DIR ${CMAKE_CURRENT_BINARY_DIR}
+    PROTO_INCLUDE_DIRS
+      ${EXTERNALS_INCLUDE}
+      ${CMAKE_CURRENT_SOURCE_DIR}
+    PROTO_SOURCES
+      ${${_name}_PROTO_SOURCES}
  )

-  # Add main library
-  if (_ARGS_STATIC)
-    add_library(${PROJECT_NAME} STATIC ${${PROJECT_NAME}_SOURCES})
-  elseif(_ARGS_SHARED)
-    add_library(${PROJECT_NAME} SHARED ${${PROJECT_NAME}_SOURCES})
-  else()
-    add_library(${PROJECT_NAME} ${${PROJECT_NAME}_SOURCES})
-  endif()
+  export_to_parent(
+    ${_name}_SOURCES
+    ${_name}_TEST_SOURCES
+    ${_name}_PROTO_SOURCES
+    ${_name}_CXX_PROTO_SOURCES
+    ${_name}_CXX_PROTO_HEADERS
+  )

+  # Add main lib
+  if (_ARGS_STATIC)
+    add_library(${_name} STATIC ${${_name}_SOURCES} ${${_name}_CXX_PROTO_SOURCES})
+  elseif(_ARGS_SHARED)
+    add_library(${_name} SHARED ${${_name}_SOURCES} ${${_name}_CXX_PROTO_SOURCES})
+  else()
+    add_library(${_name} ${${_name}_SOURCES} ${${_name}_CXX_PROTO_SOURCES})
+  endif()
  target_include_directories(
-    ${PROJECT_NAME}
+    ${_name}
+    SYSTEM
+    BEFORE
+    PUBLIC
+      $<BUILD_INTERFACE:${EXTERNALS_ROOT}/include>
+      $<BUILD_INTERFACE:${EXTERNALS_ROOT}/include/zlib>
+  )
+  target_include_directories(
+    ${_name}
    PUBLIC
      $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}>
      $<INSTALL_INTERFACE:${CMAKE_INSTALL_INCLUDEDIR}>
  )
+  # Include protos (if any)
+  if (${_name}_PROTO_SOURCES)
+    target_include_directories(
+      ${_name}
+      PUBLIC
+        $<BUILD_INTERFACE:${CMAKE_CURRENT_BINARY_DIR}>
+    )
+  endif()
  target_link_directories(
-    ${PROJECT_NAME}
+    ${_name}
    PUBLIC
      $<BUILD_INTERFACE:${CMAKE_CURRENT_BINARY_DIR}>
+      $<BUILD_INTERFACE:${EXTERNALS_ROOT}/lib/$<CONFIG>>
      $<INSTALL_INTERFACE:${CMAKE_INSTALL_LIBDIR}>
  )
+  # Install main lib
+  foreach(_hdr ${${_name}_CXX_PROTO_HEADERS})
+    file(RELATIVE_PATH _relative_path ${CMAKE_CURRENT_BINARY_DIR} ${_hdr})
+    install(FILES ${_hdr} DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}/${_relative_path})
+  endforeach()
+  install_basic(
+    NAME ${_name}
+    INCLUDE_DIR ${_include_dir}
+  )

  # Add tests executable (if there are any)
-  if (${PROJECT_NAME}_TEST_SOURCES)
-    add_executable(${PROJECT_NAME}-tests ${${PROJECT_NAME}_TEST_SOURCES})
-    target_link_libraries(${PROJECT_NAME}-tests PRIVATE ${PROJECT_NAME})
+  if (${_name}_TEST_SOURCES AND NOT WIN32)
+    add_executable(${_name}-tests ${${_name}_TEST_SOURCES})
+
+    get_target_property(_lib_type ${_name} TYPE)
+    if (_lib_type STREQUAL "SHARED_LIBRARY")
+      # Set RPATH so the tests exe knows where to find the shared lib.
+      set_target_properties(
+        ${_name}-tests
+        PROPERTIES
+          BUILD_RPATH "$ORIGIN"
+          INSTALL_RPATH "$ORIGIN/../${CMAKE_INSTALL_LIBDIR}"
+      )
+    endif()
+
+    # Link main lib
+    target_link_directories(
+      ${_name}-tests
+      PRIVATE
+        $<BUILD_INTERFACE:${CMAKE_CURRENT_BINARY_DIR}>
+        $<INSTALL_INTERFACE:${CMAKE_INSTALL_LIBDIR}>
+    )
+    target_link_libraries(${_name}-tests PRIVATE ${_name})
+
    # Catch2
    find_package(Catch2 CONFIG REQUIRED COMPONENTS Catch2WithMain)
-    target_link_libraries(${PROJECT_NAME}-tests PRIVATE Catch2::Catch2WithMain)
+    target_link_libraries(${_name}-tests PRIVATE Catch2::Catch2WithMain)
  endif()
 endfunction()
@@ -0,0 +1,71 @@
+include_guard(DIRECTORY)
+include(GNUInstallDirs)
+include(${CMAKE_SOURCE_DIR}/cmake/ExportToParent.cmake)
+include(${CMAKE_SOURCE_DIR}/cmake/SetFirstDefined.cmake)
+
+function(init_python_module)
+  set(_options)
+  set(_one_value_args
+    NAME
+  )
+  set(_multi_value_args
+    SOURCES
+  )
+  cmake_parse_arguments(_ARGS "${_options}" "${_one_value_args}" "${_multi_value_args}" ${ARGN})
+  set_first_defined(_name _ARGS_NAME PROJECT_NAME)
+  set_first_defined(_sources _ARGS_SOURCES ${_name}_SOURCES)
+
+  set(module_proj ${_name}_module)
+  set(${module_proj}_SOURCES ${_sources})
+  pybind11_add_module(${module_proj} SHARED ${_sources})
+  target_link_libraries(
+    ${module_proj}
+    PUBLIC
+      pybind11::embed
+      spark
+      ${_name}
+  )
+  target_include_directories(
+    ${module_proj}
+    PRIVATE
+      $<BUILD_INTERFACE:${pybind11_INCLUDE_DIRS}>
+  )
+  target_include_directories(
+    ${module_proj}
+    PUBLIC
+      ${Python3_INCLUDE_DIRS}
+      $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}>
+      $<INSTALL_INTERFACE:${CMAKE_INSTALL_INCLUDEDIR}>
+  )
+  add_custom_command(
+    OUTPUT ${CMAKE_CURRENT_BINARY_DIR}/stubs/${_name}.pyi
+    DEPENDS ${_name} ${module_proj}
+    COMMAND PYTHONPATH=${CMAKE_CURRENT_BINARY_DIR} pybind11-stubgen --numpy-array-use-type-var --ignore-invalid-expressions '.*' ${_name}_module
+    COMMENT "Generating py stubs"
+  )
+  add_custom_target(
+    "${_name}_py_stubs"
+    ALL
+    DEPENDS
+      ${CMAKE_CURRENT_BINARY_DIR}/stubs/${_name}.pyi
+  )
+  install(
+    TARGETS ${module_proj}
+    EXPORT ${module_proj}Targets
+    RUNTIME
+      DESTINATION ${CMAKE_INSTALL_BINDIR}
+    LIBRARY
+      DESTINATION ${CMAKE_INSTALL_LIBDIR}
+    ARCHIVE
+      DESTINATION ${CMAKE_INSTALL_LIBDIR}
+  )
+  foreach(dir ${pybind11_INCLUDE_DIRS})
+    install(DIRECTORY ${dir} DESTINATION ${CMAKE_INSTALL_INCLUDEDIR})
+  endforeach()
+  install(
+    EXPORT ${module_proj}Targets
+    FILE ${module_proj}Targets.cmake
+    NAMESPACE ${_name}::
+    DESTINATION ${CMAKE_INSTALL_LIBDIR}/cmake/${_name}
+  )
+endfunction()
@@ -0,0 +1,65 @@
+include_guard(DIRECTORY)
+include(GNUInstallDirs)
+
+function(init_python_plugin)
+  set(_options)
+  set(_one_value_args NAME PLUGIN_NAME)
+  set(_multi_value_args SOURCES)
+  cmake_parse_arguments(_ARGS "${_options}" "${_one_value_args}" "${_multi_value_args}" ${ARGN})
+  set_first_defined(_name _ARGS_NAME PROJECT_NAME)
+  set(_plugin_name ${_ARGS_PLUGIN_NAME})
+  if (NOT DEFINED _ARGS_PLUGIN_NAME)
+    set(_plugin_name ${_name}_plugins)
+  endif()
+  set(${_plugin_name}_SOURCES ${_ARGS_SOURCES})
+  if (NOT DEFINED _ARGS_SOURCES)
+    file(
+      GLOB_RECURSE ${_plugin_name}_SOURCES
+      RELATIVE ${CMAKE_CURRENT_SOURCE_DIR}
+      ${_plugin_name}/*.py)
+  endif()
+  set(src_outs)
+  foreach(_src ${${_plugin_name}_SOURCES})
+    set(_src_in ${CMAKE_CURRENT_SOURCE_DIR}/${_src})
+    set(_src_out ${CMAKE_CURRENT_BINARY_DIR}/${_src})
+    add_custom_command(
+      OUTPUT ${_src_out}
+      COMMAND ${CMAKE_COMMAND} -E copy ${_src_in} ${_src_out}
+      DEPENDS ${_src_in}
+      COMMENT "Copying ${_plugin_name} plugin ${_src} to ${_src_out}"
+    )
+    list(APPEND src_outs ${_src_out})
+  endforeach()
+  
+  add_custom_target(copy_${_plugin_name} ALL DEPENDS ${src_outs})
+  
+  add_library(${_plugin_name} INTERFACE)
+  add_library(${_name}::${_plugin_name} ALIAS ${_plugin_name})
+  
+  set_target_properties(
+    ${_plugin_name}
+    PROPERTIES
+      INTERFACE_PYTHON_SCRIPTS_DIR
+        $<BUILD_INTERFACE:${CMAKE_CURRENT_BINARY_DIR}/${_plugin_name}>$<INSTALL_INTERFACE:${CMAKE_INSTALL_BINDIR}/${_plugin_name}>
+  )
+  
+  set_target_properties(
+    ${_plugin_name}
+    PROPERTIES
+    INTERFACE_PYTHON_SCRIPTS $<BUILD_INTERFACE:${CMAKE_CURRENT_BINARY_DIR}/${${_plugin_name}_SOURCES}>$<INSTALL_INTERFACE:${CMAKE_INSTALL_BINDIR}/${${_plugin_name}_SOURCES}>
+  )
+  
+  install(
+    TARGETS ${_plugin_name}
+    EXPORT ${_plugin_name}Targets
+    DESTINATION ${CMAKE_INSTALL_BINDIR}
+  )
+  
+  install(
+    EXPORT ${_plugin_name}Targets
+    FILE ${_plugin_name}Targets.cmake
+    NAMESPACE ${_name}::
+    DESTINATION ${CMAKE_INSTALL_LIBDIR}/cmake/${_name}
+  )
+  install(DIRECTORY ${_plugin_name} DESTINATION ${CMAKE_INSTALL_BINDIR})
+endfunction()
@@ -1,55 +0,0 @@
-function(init_shared_library)
-  include(${CMAKE_SOURCE_DIR}/cmake/ExportToParent.cmake)
-  set(source_dir)
-  if (${ARGC} EQUAL 0)
-    set(source_dir ${PROJECT_NAME})
-  elseif(${ARGC} GREATER 1)
-    message(FATAL_ERROR "init_shared_library received invalid arguments. Usage: init_shared_library(<OPTIONAL:source_dir>)")
-  else()
-    set(source_dir ${ARGV})
-  endif()
-
-  # Glob cpp sources
-  file(
-    GLOB_RECURSE ${PROJECT_NAME}_SOURCES
-    RELATIVE ${CMAKE_CURRENT_SOURCE_DIR}
-    ${source_dir}/*.cpp
-  )
-  # Note: cpp files inside 'tests' dir(s) are considered tests.
-  # Remove tests from main sources, and add them to test sources
-  foreach(item ${${PROJECT_NAME}_SOURCES})
-    if(${item} MATCHES "tests/*.cpp")
-      list(REMOVE_ITEM ${PROJECT_NAME}_SOURCES ${item})
-      list(APPEND ${PROJECT_NAME}_TEST_SOURCES ${item})
-    endif()
-  endforeach()
-
-  export_to_parent(
-    ${PROJECT_NAME}_SOURCES
-    ${PROJECT_NAME}_TEST_SOURCES
-  )
-
-  # Add main library
-  add_library(${PROJECT_NAME} SHARED ${${PROJECT_NAME}_SOURCES})
-  target_include_directories(
-    ${PROJECT_NAME}
-    PUBLIC
-      $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}>
-      $<INSTALL_INTERFACE:${CMAKE_INSTALL_INCLUDEDIR}>
-  )
-  target_link_directories(
-    ${PROJECT_NAME}
-    PUBLIC
-      $<BUILD_INTERFACE:${CMAKE_CURRENT_BINARY_DIR}>
-      $<INSTALL_INTERFACE:${CMAKE_INSTALL_LIBDIR}>
-  )
-
-  # Add tests executable (if there are any)
-  if (${PROJECT_NAME}_TEST_SOURCES)
-    add_executable(${PROJECT_NAME}-tests ${${PROJECT_NAME}_TEST_SOURCES})
-    target_link_libraries(${PROJECT_NAME}-tests PRIVATE ${PROJECT_NAME})
-    # Catch2
-    find_package(Catch2 CONFIG REQUIRED COMPONENTS Catch2WithMain)
-    target_link_libraries(${PROJECT_NAME}-tests PRIVATE Catch2::Catch2WithMain)
-  endif()
-endfunction(init_shared_library)
@@ -1,19 +1,28 @@
-include(${CMAKE_SOURCE_DIR}/cmake/PackBasic.cmake)
+include_guard(DIRECTORY)
 include(GNUInstallDirs)
+include(${CMAKE_SOURCE_DIR}/cmake/PackBasic.cmake)
+include(${CMAKE_SOURCE_DIR}/cmake/SetFirstDefined.cmake)

 function(install_basic)
-  file(
-    GLOB_RECURSE ${PROJECT_NAME}_HEADERS
-    RELATIVE ${CMAKE_CURRENT_SOURCE_DIR}
-    *.h)
+  set(_options)
+  set(_one_value_args NAME INCLUDE_DIR)
+  set(_multi_value_args)
+  cmake_parse_arguments(_ARGS "${_options}" "${_one_value_args}" "${_multi_value_args}" ${ARGN})
+  set_first_defined(_name _ARGS_NAME PROJECT_NAME)
+  set_first_defined(_include_dir _ARGS_INCLUDE_DIR _name)

-  target_sources(${PROJECT_NAME}
+  file(
+    GLOB_RECURSE ${_name}_HEADERS
+    RELATIVE ${CMAKE_CURRENT_SOURCE_DIR}
+    ${_include_dir}/*.h)
+
+  target_sources(${_name}
    PUBLIC
-      FILE_SET HEADERS FILES ${${PROJECT_NAME}_HEADERS})
+      FILE_SET HEADERS FILES ${${_name}_HEADERS})

  install(
-    TARGETS ${PROJECT_NAME}
-    EXPORT ${PROJECT_NAME}Targets
+    TARGETS ${_name}
+    EXPORT ${_name}Targets
    RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR}
    LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR}
    ARCHIVE DESTINATION ${CMAKE_INSTALL_LIBDIR}
@@ -21,10 +30,10 @@ function(install_basic)
  )

  install(
-    EXPORT ${PROJECT_NAME}Targets
-    FILE ${PROJECT_NAME}Targets.cmake
-    NAMESPACE ${PROJECT_NAME}::
-    DESTINATION ${CMAKE_INSTALL_LIBDIR}/cmake/${PROJECT_NAME})
+    EXPORT ${_name}Targets
+    FILE ${_name}Targets.cmake
+    NAMESPACE ${_name}::
+    DESTINATION ${CMAKE_INSTALL_LIBDIR}/cmake/${_name})

  pack_basic()
 endfunction()
@@ -1,6 +1,14 @@
+include_guard(DIRECTORY)
 include(CMakePackageConfigHelpers)
+include(${CMAKE_SOURCE_DIR}/cmake/SetFirstDefined.cmake)

 function(pack_basic)
+  set(_options)
+  set(_one_value_args NAME)
+  set(_multi_value_args)
+  cmake_parse_arguments(_ARGS "${_options}" "${_one_value_args}" "${_multi_value_args}" ${ARGN})
+  set_first_defined(_name _ARGS_NAME PROJECT_NAME)
+
  find_package(Git QUIET)

  if(GIT_FOUND)
@@ -17,20 +25,20 @@ function(pack_basic)
  endif()

  write_basic_package_version_file(
-    "${CMAKE_CURRENT_BINARY_DIR}/${PROJECT_NAME}ConfigVersion.cmake"
+    "${CMAKE_CURRENT_BINARY_DIR}/${_name}ConfigVersion.cmake"
    VERSION ${PROJECT_VERSION}
    COMPATIBILITY AnyNewerVersion
  )

  configure_package_config_file(
    "${CMAKE_CURRENT_SOURCE_DIR}/Config.cmake.in"
-    "${CMAKE_CURRENT_BINARY_DIR}/${PROJECT_NAME}Config.cmake"
-    INSTALL_DESTINATION ${CMAKE_INSTALL_LIBDIR}/cmake/${PROJECT_NAME}
+    "${CMAKE_CURRENT_BINARY_DIR}/${_name}Config.cmake"
+    INSTALL_DESTINATION ${CMAKE_INSTALL_LIBDIR}/cmake/${_name}
  )

  install(
    FILES
-      "${CMAKE_CURRENT_BINARY_DIR}/${PROJECT_NAME}Config.cmake"
-      "${CMAKE_CURRENT_BINARY_DIR}/${PROJECT_NAME}ConfigVersion.cmake"
-    DESTINATION ${CMAKE_INSTALL_LIBDIR}/cmake/${PROJECT_NAME})
+      "${CMAKE_CURRENT_BINARY_DIR}/${_name}Config.cmake"
+      "${CMAKE_CURRENT_BINARY_DIR}/${_name}ConfigVersion.cmake"
+    DESTINATION ${CMAKE_INSTALL_LIBDIR}/cmake/${_name})
 endfunction()
@@ -0,0 +1,164 @@
+set(Protobuf_PROTOC_EXECUTABLE "${EXTERNALS_BIN}/protoc")
+set(PROTOBUF_INCLUDE_DIRS "${EXTERNALS_INCLUDE}")
+set(PROTOBUF_STATIC_LIB "libprotobuf")
+set(_protobuf_src_dir ${CMAKE_SOURCE_DIR}/src/lib_spark)
+set(_protobuf_bin_dir ${CMAKE_BINARY_DIR}/src/lib_spark)
+
+function(Protobuf_Generate_Cpp)
+  set(_options)
+  set(_one_value_args CXX_SOURCES CXX_HEADERS CXX_OUT_DIR)
+  set(_multi_value_args PROTO_INCLUDE_DIRS PROTO_SOURCES)
+  cmake_parse_arguments(_ARGS "${_options}" "${_one_value_args}" "${_multi_value_args}" ${ARGN})
+  set(${_ARGS_CXX_SOURCES})
+  set(${_ARGS_CXX_HEADERS})
+
+  set(_proto_include_cli_args)
+  foreach(_dir ${_ARGS_PROTO_INCLUDE_DIRS})
+    list(APPEND _proto_include_cli_args -I ${_dir})
+  endforeach()
+
+  foreach(FIL ${_ARGS_PROTO_SOURCES})
+    get_filename_component(ABS_FIL ${FIL} ABSOLUTE)
+    get_filename_component(FIL_WE ${FIL} NAME_WE)
+    get_filename_component(FIL_DIR ${FIL} DIRECTORY)
+    if(FIL_DIR)
+      set(FIL_WE "${FIL_DIR}/${FIL_WE}")
+    endif()
+
+    list(APPEND ${_ARGS_CXX_SOURCES} "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.pb.cc")
+    list(APPEND ${_ARGS_CXX_HEADERS} "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.pb.h")
+
+    add_custom_command(
+      OUTPUT "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.pb.cc"
+             "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.pb.h"
+      COMMAND
+        ${Protobuf_PROTOC_EXECUTABLE} ARGS --cpp_out=${_ARGS_CXX_OUT_DIR} ${_proto_include_cli_args} ${ABS_FIL}
+      DEPENDS ${ABS_FIL}
+      COMMENT "Running C++ protocol buffer compiler on ${FIL}"
+      VERBATIM)
+  endforeach()
+
+  set_source_files_properties(${${_ARGS_CXX_SOURCES}} ${${_ARGS_CXX_HEADERS}} PROPERTIES GENERATED TRUE)
+
+  # remove some warnings from protobuf generated files
+  if(WIN32)
+    set_source_files_properties(${${_ARGS_CXX_SOURCES}} PROPERTIES COMPILE_FLAGS
+                                                      "/wd4309 /wd4838")
+  else()
+    set_source_files_properties(
+      ${${_ARGS_CXX_SOURCES}}
+      PROPERTIES
+        COMPILE_FLAGS
+        "-Wno-unused-parameter -Wno-effc++ -Wno-pedantic -Wno-sign-compare -Wno-uninitialized"
+    )
+  endif()
+
+  set(${_ARGS_CXX_SOURCES}
+      ${${_ARGS_CXX_SOURCES}}
+      PARENT_SCOPE)
+  set(${_ARGS_CXX_HEADERS}
+      ${${_ARGS_CXX_HEADERS}}
+      PARENT_SCOPE)
+endfunction()
+
+function(Service_Compiler_Generate_Cpp)
+  set(_options)
+  set(_one_value_args CXX_SOURCES CXX_HEADERS CXX_OUT_DIR)
+  set(_multi_value_args PROTO_INCLUDE_DIRS PROTO_SOURCES)
+  cmake_parse_arguments(_ARGS "${_options}" "${_one_value_args}" "${_multi_value_args}" ${ARGN})
+  set(${_ARGS_CXX_SOURCES})
+  set(${_ARGS_CXX_HEADERS})
+
+  set(_proto_include_cli_args)
+  foreach(_dir ${_ARGS_PROTO_INCLUDE_DIRS})
+    list(APPEND _proto_include_cli_args -I ${_dir})
+  endforeach()
+
+  foreach(FIL ${_ARGS_PROTO_SOURCES})
+    get_filename_component(ABS_FIL ${FIL} ABSOLUTE)
+    get_filename_component(FIL_WE ${FIL} NAME_WE)
+    get_filename_component(FIL_DIR ${FIL} DIRECTORY)
+    if(FIL_DIR)
+      set(FIL_WE "${FIL_DIR}/${FIL_WE}")
+    endif()
+
+    list(APPEND ${_ARGS_CXX_SOURCES} "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.pb.cc")
+    list(APPEND ${_ARGS_CXX_HEADERS} "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.pb.h")
+
+    list(APPEND ${_ARGS_CXX_SOURCES} "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.client.pb.cc")
+    list(APPEND ${_ARGS_CXX_HEADERS} "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.client.pb.h")
+
+    list(APPEND ${_ARGS_CXX_SOURCES} "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.server.pb.cc")
+    list(APPEND ${_ARGS_CXX_HEADERS} "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.server.pb.h")
+
+    set(TEMPLATES
+        ${CMAKE_SOURCE_DIR}/src/lib_spark/spark/api/service_compiler/templates/cpp/client.pb.cc
+        ${CMAKE_SOURCE_DIR}/src/lib_spark/spark/api/service_compiler/templates/cpp/client.pb.h
+        ${CMAKE_SOURCE_DIR}/src/lib_spark/spark/api/service_compiler/templates/cpp/server.pb.cc
+        ${CMAKE_SOURCE_DIR}/src/lib_spark/spark/api/service_compiler/templates/cpp/server.pb.h
+    )
+
+    set(_plugin_py ${CMAKE_SOURCE_DIR}/src/lib_spark/spark/api/service_compiler/plugin.py)
+    if(WIN32)
+      set(_plugin_cmd ${CMAKE_SOURCE_DIR}/src/lib_spark/spark/api/service_compiler/plugin.cmd)
+      add_custom_command(
+        OUTPUT "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.pb.cc"
+               "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.pb.h"
+               "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.client.pb.cc"
+               "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.client.pb.h"
+               "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.server.pb.cc"
+               "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.server.pb.h"
+        COMMAND SET PYTHONPATH=${SPARK_PYTHON}
+        COMMAND ${CMAKE_SOURCE_DIR}/.venv/Scripts/activate.bat
+        COMMAND
+          ${Protobuf_PROTOC_EXECUTABLE} ARGS --cpp_out=${_ARGS_CXX_OUT_DIR}
+          --plugin=protoc-gen-service=${_plugin_cmd}
+          --service_out=lang=cpp:${_ARGS_CXX_OUT_DIR} -I
+          ${PROTOBUF_INCLUDE_DIRS} -I ${_protobuf_src_dir} ${ABS_FIL}
+        DEPENDS ${ABS_FIL} ${TEMPLATES} ${_plugin_cmd} ${_plugin_py}
+        COMMENT "Running C++ service compiler on ${FIL}"
+        VERBATIM)
+    else()
+      add_custom_command(
+        OUTPUT "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.pb.cc"
+               "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.pb.h"
+               "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.client.pb.cc"
+               "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.client.pb.h"
+               "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.server.pb.cc"
+               "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.server.pb.h"
+        COMMAND ${CMAKE_SOURCE_DIR}/tools/venv_exec.sh
+        ARGS
+          ${Protobuf_PROTOC_EXECUTABLE}
+          --cpp_out=${_ARGS_CXX_OUT_DIR}
+          --plugin=protoc-gen-service=${_plugin_py}
+          --service_out=lang=cpp,make_file_name=${ABS_FIL}:${_ARGS_CXX_OUT_DIR}
+          ${_proto_include_cli_args} ${ABS_FIL}
+        DEPENDS ${ABS_FIL} ${TEMPLATES} ${_plugin_py}
+        COMMENT "Running C++ service compiler on ${FIL}"
+        VERBATIM)
+    endif()
+
+  endforeach()
+
+  # remove some warnings from protobuf generated files
+  if(WIN32)
+    set_source_files_properties(${${_ARGS_CXX_SOURCES}} PROPERTIES COMPILE_FLAGS
+                                                      "/wd4309 /wd4838")
+  else()
+    set_source_files_properties(
+      ${${_ARGS_CXX_SOURCES}}
+      PROPERTIES
+        COMPILE_FLAGS
+        "-Wno-unused-parameter -Wno-effc++ -Wno-pedantic -Wno-sign-compare -Wno-uninitialized"
+    )
+  endif()
+
+  set(${_ARGS_CXX_SOURCES}
+      ${${_ARGS_CXX_SOURCES}}
+      PARENT_SCOPE)
+  set(${_ARGS_CXX_HEADERS}
+      ${${_ARGS_CXX_HEADERS}}
+      PARENT_SCOPE)
+
+endfunction()
+
@@ -2,12 +2,24 @@ include_guard(GLOBAL)

 function(protobuf_generate_cpp)
  set(_options)
-  set(_one_value_args CXX_SOURCES CXX_HEADERS CXX_OUT_DIR PROTOC)
-  set(_multi_value_args PROTO_INCLUDE_DIRS PROTO_SOURCES)
+  set(_one_value_args
+    CXX_SOURCES
+    CXX_HEADERS
+    CXX_OUT_DIR
+    PROTOC
+  )
+  set(_multi_value_args
+    PROTO_INCLUDE_DIRS
+    PROTO_SOURCES
+  )
  cmake_parse_arguments(_ARGS "${_options}" "${_one_value_args}" "${_multi_value_args}" ${ARGN})

  if (NOT DEFINED _ARGS_PROTOC)
-    set(_ARGS_PROTOC "${EXTERNALS_BIN}/protoc")
+    if (WIN32)
+      set(_ARGS_PROTOC "${EXTERNALS_BIN}/protoc.exe")
+    else()
+      set(_ARGS_PROTOC "${EXTERNALS_BIN}/protoc")
+    endif()
  endif()
  set(${_ARGS_CXX_SOURCES})
  set(${_ARGS_CXX_HEADERS})
@@ -26,6 +38,9 @@ function(protobuf_generate_cpp)
    endif()
    list(APPEND ${_ARGS_CXX_SOURCES} "${_ARGS_CXX_OUT_DIR}/${_fil_we}.pb.cc")
    list(APPEND ${_ARGS_CXX_HEADERS} "${_ARGS_CXX_OUT_DIR}/${_fil_we}.pb.h")
+    message(STATUS "add_custom_command for ${_fil} -> ${_ARGS_CXX_OUT_DIR}/${_fil_we}.pb.h")
+    # TODO: Pass proto sources by relative path (so this fn can be called from
+    # arbitrary subdirs).
    add_custom_command(
      OUTPUT
        "${_ARGS_CXX_OUT_DIR}/${_fil_we}.pb.cc"
@@ -65,3 +80,4 @@ function(protobuf_generate_cpp)
  set(${_ARGS_CXX_SOURCES} ${${_ARGS_CXX_SOURCES}} PARENT_SCOPE)
  set(${_ARGS_CXX_HEADERS} ${${_ARGS_CXX_HEADERS}} PARENT_SCOPE)
 endfunction()
+
@@ -0,0 +1,61 @@
+include_guard(GLOBAL)
+
+function(protobuf_generate_cpp_config)
+  set(_options)
+  set(_one_value_args
+    PROTO_SOURCES_HEADER
+    PROTO_DEFAULTS_HEADER
+  )
+  set(_multi_value_args
+    PROTO_SOURCES
+    PROTO_DEFAULTS
+  )
+  cmake_parse_arguments(_ARGS "${_options}" "${_one_value_args}" "${_multi_value_args}" ${ARGN})
+
+  # Configure the 'config_proto_files' header. It provides an array of strings populated with the
+  # files in PROTO_SOURCES. At runtime, we use this array to generate a special 'config' proto
+  # message with fields for each message defined in said files. A global instance of this 'config'
+  # proto is then constructed, and populated with [compile-time] defaults, and [runtime] overrides
+  # (i.e., in spark.conf). Calls to `bts::config::get<T>()` retrieve the value from the field
+  # associated with type `T` in this global instance.
+  message(STATUS "PROTO_SOURCES: ${_ARGS_PROTO_SOURCES}")
+  message(STATUS "PROTO_SOURCES_HEADER: ${_ARGS_PROTO_SOURCES_HEADER}")
+  message(STATUS "PROTO_DEFAULTS_HEADER: ${_ARGS_PROTO_DEFAULTS_HEADER}")
+  if (_ARGS_PROTO_SOURCES_HEADER)
+    set(PROTO_SOURCES)
+    set(PROTO_SOURCES_CXX_ARRAY_ITEMS "")
+    foreach(_proto_path ${_ARGS_PROTO_SOURCES})
+      list(APPEND PROTO_SOURCES ${_proto_path})
+      string(APPEND PROTO_SOURCES_CXX_ARRAY_ITEMS "\"${_proto_path}\",\n")
+    endforeach()
+    list(LENGTH PROTO_SOURCES PROTO_SOURCES_LENGTH)
+    message(STATUS "CONFIGURING PROTO_SOURCES_HEADER): ${_ARGS_PROTO_SOURCES_HEADER}")
+    configure_file(${CMAKE_SOURCE_DIR}/cmake/config_proto_files.h.in ${_ARGS_PROTO_SOURCES_HEADER})
+  endif()
+
+  # Generate static defaults. Ideally, this would be a single string, but we use an array because
+  # the compiler complains if the string is longer than 65k.
+  set(_cfg_defaults_str "inline const char * config_static_defaults[] = { ")
+
+  foreach(_cfg_file_path ${_ARGS_PROTO_DEFAULTS})
+    file(READ ${_cfg_file_path} _cfg_file_content)
+    string(REPLACE "\"" "\\\"" _cfg_file_content_escaped ${_cfg_file_content})
+    string(REPLACE "\n" ";" _cfg_lines ${_cfg_file_content_escaped})
+
+    foreach(_line ${_cfg_lines})
+      string(APPEND _cfg_defaults_str "\n  \"${_line}\",")
+    endforeach(_line)
+  endforeach(_cfg_file_path)
+
+  string(APPEND _cfg_defaults_str "};")
+  set(_old_cfg_defaults_str "")
+
+  if(EXISTS ${_ARGS_PROTO_DEFAULTS_HEADER})
+    file(READ ${_ARGS_PROTO_DEFAULTS_HEADER} _old_cfg_defaults_str)
+  endif()
+
+  if(NOT _cfg_defaults_str STREQUAL _old_cfg_defaults_str)
+    file(WRITE "${_ARGS_PROTO_DEFAULTS_HEADER}" "${_cfg_defaults_str}")
+  endif()
+endfunction()
+
@@ -0,0 +1,10 @@
+include_guard(GLOBAL)
+
+macro(set_first_defined RESULT)
+  foreach(_arg ${ARGN})
+    if (DEFINED ${_arg})
+      set(${RESULT} ${${_arg}})
+      break()
+    endif()
+  endforeach()
+endmacro()
@@ -20,16 +20,14 @@ endfunction()

 function(sym_link_file)
  set(_options)
-  set(_one_value_args TARGET_NAME SOURCE DEST INSTALL_DEST)
+  set(_one_value_args SOURCE DEST INSTALL_DEST)
  set(_multi_value_args)
  cmake_parse_arguments(_ARGS "${_options}" "${_one_value_args}" "${_multi_value_args}" ${ARGN})
-  add_custom_command(
-    OUTPUT ${_ARGS_DEST}
-    COMMAND ln ARGS -s ${_ARGS_SOURCE} ${_ARGS_DEST}
-  )
-  add_custom_target(
-    ${_ARGS_TARGET_NAME} ALL
-    DEPENDS ${_ARGS_DEST}
+  file(
+    CREATE_LINK
+      ${_ARGS_SOURCE}
+      ${_ARGS_DEST}
+    SYMBOLIC
  )
  if (DEFINED _ARGS_INSTALL_DEST)
    install(
@@ -0,0 +1,39 @@
+include_guard(DIRECTORY)
+include(${CMAKE_SOURCE_DIR}/cmake/SymLink.cmake)
+
+function(sym_link_python_module)
+  set(_options)
+  set(_one_value_args LIB_NAME MODULE_NAME PLUGIN_NAME)
+  set(_multi_value_args)
+  cmake_parse_arguments(_ARGS "${_options}" "${_one_value_args}"
+                        "${_multi_value_args}" ${ARGN})
+  if(NOT DEFINED _ARGS_LIB_NAME)
+    message(
+      FATAL_ERROR "LIB_NAME must be specified for sym_link_python_module()")
+  endif()
+  set(_lib_name ${_ARGS_LIB_NAME})
+  set(_module_name ${_ARGS_MODULE_NAME})
+  if(NOT DEFINED _ARGS_MODULE_NAME)
+    set(_module_name ${_lib_name}_module)
+  endif()
+  set(_plugin_name ${_ARGS_PLUGIN_NAME})
+  set(_deps ${_lib_name} ${_module_name})
+  if(DEFINED _ARGS_PLUGIN_NAME)
+    list(APPEND _deps ${_plugin_name} copy_${_plugin_name})
+  endif()
+  add_dependencies(${PROJECT_NAME} ${_deps})
+  sym_link_local_lib(${_lib_name})
+  set(_module_suffix cpython-310-x86_64-linux-gnu.so)
+  sym_link_file(
+    SOURCE
+    ${CMAKE_BINARY_DIR}/src/lib_${_lib_name}/${_module_name}.${_module_suffix}
+    DEST
+    ${CMAKE_CURRENT_BINARY_DIR}/${_module_name}.${_module_suffix}
+    INSTALL_DEST
+    ${CMAKE_INSTALL_BINDIR}/)
+  if(DEFINED _ARGS_PLUGIN_NAME)
+    get_target_property(plugins ${_plugin_name} INTERFACE_PYTHON_SCRIPTS_DIR)
+    sym_link_dir(TARGET_NAME ${PROJECT_NAME}_link_${_plugin_name} SOURCE
+                 ${plugins} DEST ${CMAKE_CURRENT_BINARY_DIR}/${_plugin_name})
+  endif()
+endfunction()
@@ -0,0 +1,11 @@
+#pragma once
+
+#include <array>
+
+namespace bts::@PROJECT_NAME@::protobufs {
+
+inline std::array<const char *, @PROTO_SOURCES_LENGTH@> config_proto_files = {
+@PROTO_SOURCES_CXX_ARRAY_ITEMS@
+};
+
+} // namespace bts::@PROJECT_NAME@::protobufs
@@ -0,0 +1,11 @@
+#ifndef BTS_SPARK_VERSION_H
+#define BTS_SPARK_VERSION_H
+
+#define BTS_SPARK_VERSION_MAJOR @PROJECT_VERSION_MAJOR@
+#define BTS_SPARK_VERSION_MINOR @PROJECT_VERSION_MINOR@
+#define BTS_SPARK_VERSION_PATCH @PROJECT_VERSION_PATCH@
+#define BTS_SPARK_VERSION_BUILD_METADATA @PROJECT_VERSION_BUILD_METADATA@
+#define BTS_SPARK_VERSION "@PROJECT_VERSION@"
+#define BTS_SPARK_VERSION_FULL "@PROJECT_VERSION_FULL@"
+
+#endif
@@ -1,24 +0,0 @@
-cmake_minimum_required(VERSION 3.5)
-
-# Explicitly set policy to convert relative paths to absolute (although this is a default policy,
-# setting it explicitly prevents a compile-warning)
-cmake_policy(SET CMP0076 NEW)
-
-# add protos to the library to facilitate source_group
-target_sources(libspark PUBLIC ${Config_Protos} ${NonConfig_Protos} ${Service_Protos})
-
-function(assign_source_group)
-  foreach(_source IN ITEMS ${ARGN})
-    if(IS_ABSOLUTE "${_source}")
-      file(RELATIVE_PATH _source_rel "${CMAKE_CURRENT_SOURCE_DIR}" "${_source}")
-    else()
-      set(_source_rel "${_source}")
-    endif()
-    get_filename_component(_source_path "${_source_rel}" PATH)
-    string(REPLACE "/" "\\" _source_path_msvc "${_source_path}")
-    source_group("${_source_path_msvc}" FILES "${_source}")
-    set(SOURCE_FILES "${_source}")
-  endforeach()
-endfunction(assign_source_group)
-
-assign_source_group(${SparkLib_Sources} ${Config_Protos} ${NonConfig_Protos} ${Service_Protos})
@@ -1,51 +0,0 @@
-services:
-  sparkd:
-    build:
-      dockerfile: Dockerfile
-      context: ./sparkd
-    command: [ "./sparkd", "run" ]
-    ports: [ "8000:8000", "8001:8001", "8002:8002" ]
-
-  pubsub:
-    build:
-      dockerfile: pubsub.Dockerfile
-      context: ./DataStream
-    command: [ "./PubSubServer" ]
-    environment:
-      - DOTNET_ENVIRONMENT=Production
-    ports:
-      - 22019:22019
-
-  md-publisher:
-    build:
-      dockerfile: md-publisher.Dockerfile
-      context: ./DataStream
-    depends_on:
-      - pubsub
-      - sparkd
-    command:
-      [
-        "dockerize",
-        "-wait",
-        "tcp://pubsub:22019",
-        "-wait",
-        "tcp://sparkd:8000",
-        "-timeout",
-        "10s",
-        "./DataStreamPublisher"
-      ]
-    environment:
-      - DOTNET_ENVIRONMENT=MarketData
-
-  webapp:
-    ports:
-      - 8080:8080
-      - 8443:8443
-    build:
-      dockerfile: webapp.Dockerfile
-      context: ./DataStream
-    command: [ "./Bts.DataStreamWebApp.Server" ]
-    environment:
-      - ASPNETCORE_HTTP_PORT=8080
-      - ASPNETCORE_HTTPS_PORT=8443
-      - ASPNETCORE_ENVIRONMENT=Staging
@@ -1,62 +1,92 @@
 services:
  sparkd:
+    volumes:
+      - ${SPARKD__DATA_DIR:-sparkd_data}:/home/bts/sparkdata/data
    build:
-      dockerfile: src/server/sparkd/Dockerfile
-    command: [ "./sparkd", "run" ]
-    ports: [ "8000:8000", "8001:8001", "8002:8002" ]
-
-  pubsub:
-    environment:
-      - DOTNET_ENVIRONMENT=Staging
-    build:
-      dockerfile: src/DataStreamSuite/PubSubServer/Dockerfile
+      dockerfile: src/app_sparkd/Dockerfile
+      args:
+        - IMAGE=${SPARKD__IMAGE}
+        - SOURCE_DIR=${SPARKD__SOURCE_DIR}
+        - CONFIG_DIR=${SPARKD__CONFIG_DIR}
+        - PYTHON_REQUIREMENTS=${SPARKD__PYTHON_REQUIREMENTS}
+    shm_size: "${SPARKD__SHM_SIZE:-512mb}"
+    command: >
+      sh -c "
+        echo bts | sudo -S chown -R bts:bts /home/bts/sparkdata/data
+        if [ -e "/home/bts/sparkdata/data/logging.sparkd/log" ]; then
+          mv "/home/bts/sparkdata/data/logging.sparkd/log" "/home/bts/sparkdata/data/logging.sparkd/log_$(date '+%Y%m%d-%H%M')"
+        fi
+        if [ -e "/home/bts/sparkdata/conf/.env.local" ]; then
+          echo 'Sourcing /home/bts/sparkdata/conf/.env.local'
+          set -a
+          . /home/bts/sparkdata/conf/.env.local
+          set +a
+        fi
+        ./sparkd run"
+    env_file:
+      - .env
+      - ${SPARKD__CONFIG_DIR}/.env
    ports:
-      - 22019:22019
-    command: [ "./PubSubServer" ]
+      - "${SPARKD__CONFIG_PORT}:${SPARKD__CONFIG_PORT}"
+      - "${SPARKD__QUICKBUS_PORT}:${SPARKD__QUICKBUS_PORT}"
+      - "${SPARKD__SIM_FIX_PORT}:${SPARKD__SIM_FIX_PORT}"
+    deploy:
+      resources:
+        limits:
+          memory: "${SPARKD__MEMORY:-4gb}"

-  md-publisher:
+  sparkd-symbols:
+    volumes:
+      - ${SPARKD__DATA_DIR:-sparkd_data}:/home/bts/sparkdata/data
+      - ${SPARKD__RUNTIME_CONF_DIR:-sparkd_conf}:/home/bts/sparkdata/conf
    build:
-      dockerfile: src/DataStreamSuite/DataStreamPublisher/Dockerfile
-    depends_on:
-      - pubsub
-      - sparkd
-    command:
-      [
-        "dockerize",
-        "-wait",
-        "tcp://pubsub:22019",
-        "-wait",
-        "tcp://sparkd:8000",
-        "-timeout",
-        "10s",
-        "./DataStreamPublisher"
-      ]
-    environment:
-      - DOTNET_ENVIRONMENT=MarketData
-
-  webapp:
+      dockerfile: src/app_sparkd/Dockerfile
+      args:
+        - IMAGE=${SPARKD__IMAGE}
+        - SOURCE_DIR=${SPARKD__SOURCE_DIR}
+        - CONFIG_DIR=${SPARKD__CONFIG_DIR}
+        - PYTHON_REQUIREMENTS=${SPARKD__PYTHON_REQUIREMENTS}
+    shm_size: "${SPARKD__SHM_SIZE:-512mb}"
+    command: >
+      sh -c "
+        echo bts | sudo -S chown -R bts:bts /home/bts/sparkdata
+        if [ -e "/home/bts/sparkdata/data/logging.sparkd/log" ]; then
+          mv "/home/bts/sparkdata/data/logging.sparkd/log" "/home/bts/sparkdata/data/logging.sparkd/log_$(date '+%Y%m%d-%H%M')"
+        fi
+        ls -alh /home/bts/sparkdata/conf/
+        cat /home/bts/sparkdata/conf/.env.local
+        if [ -e "/home/bts/sparkdata/conf/.env.local" ]; then
+          echo 'Sourcing /home/bts/sparkdata/conf/.env.local'
+          set -a
+          . /home/bts/sparkdata/conf/.env.local
+          set +a
+        fi
+        ./sparkd run"
+    env_file:
+      - .env
+      - ${SPARKD__CONFIG_DIR}/.env
    ports:
-      - 8080:8080
-      - 8443:8443
-    build:
-      dockerfile: src/DataStreamSuite/Bts.DataStreamWebApp.Server/Dockerfile
-    command: [ "./Bts.DataStreamWebApp.Server" ]
-    environment:
-      - ASPNETCORE_ENVIRONMENT=Staging
+      - "${SPARKD__CONFIG_PORT}:${SPARKD__CONFIG_PORT}"
+      - "${SPARKD__QUICKBUS_PORT}:${SPARKD__QUICKBUS_PORT}"
+      - "${SPARKD__SIM_FIX_PORT}:${SPARKD__SIM_FIX_PORT}"
+    deploy:
+      restart_policy:
+        condition: any
+        delay: 3s
+      resources:
+        limits:
+          memory: "${SPARKD__MEMORY:-4gb}"

-  api-server:
-    profiles:
-      - api-server
-    depends_on:
-      - sparkd
-    ports:
-      - 8008:8008
-      - 9443:9443
-    build:
-      dockerfile: src/client/spark_api_server/Dockerfile
-    command: [ "./spark_api_server", "run" ]
-    environment:
-      - ASPNETCORE_ENVIRONMENT=Staging
+  sparkd-tail:
+    image: bluetradingsystems/linux_dotnet7
+    volumes:
+      - type: volume
+        read_only: true
+        source: sparkd_data
+        target: /home/bts/sparkdata/data/logging.sparkd
+        volume:
+          subpath: logging.sparkd
+    command: ["tail", "-f", "/home/bts/sparkdata/data/logging.sparkd/log"]

  sparkd-ping-test:
    depends_on:
@@ -70,3 +100,7 @@ services:
        USER "root"
        RUN apt-get update && apt-get -y install iputils-ping
    command: [ "ping", "sparkd" ]
+
+volumes:
+  sparkd_data:
+    name: "sparkd_${SPARKD__CONFIG_PORT:-8000}"
@@ -124,8 +124,8 @@ service OrderService {
 ```
 ### C++ Example
 ```cpp
-#include "lib/api/public/ServiceContext.h"
-#include "lib/api/services/OrderService.client.pb.h"
+#include "spark/api/public/ServiceContext.h"
+#include "spark/api/services/OrderService.client.pb.h"
 using namespace bts::api;
 ServiceContext ctx;
 auto orders = ctx.locate<orderman::OrderService>();
@@ -29,9 +29,9 @@ namespace sample {
 ```

 ```cpp
-#include "lib/api/public/ServiceContext.h"
-#include "spark/lib/domain_model/Security.h"
-#include "lib/api/services/MarketDataService.client.pb.h"
+#include "spark/api/public/ServiceContext.h"
+#include "spark/domain_model/Security.h"
+#include "spark/api/services/MarketDataService.client.pb.h"
 #include <thread>

 using namespace bts::api;
@@ -53,7 +53,7 @@ The status callback is called when the market for this security opens, closes, o
 // Subscribe to security trading status
 void IAlgoServices::subscribe_status(SecurityKey);
 // You implement
-virtual void on_status(SecurityKey, domain::protobufs::TradingStatus) override;
+virtual void on_status(SecurityKey, co2::TradingStatus) override;
 ```

 # Child Orders
@@ -2,13 +2,12 @@ FROM ubuntu:22.04

 ARG DEBIAN_FRONTEND=noninteractive

-RUN apt-get update \
-  && apt-get install -y pkg-config \
-  && apt-get install -y locales \
-  && apt-get install -y ca-certificates \
-  && apt-get install -y libssh2-1-dev \
-  && apt-get install -y software-properties-common \
-  && apt-get install -y sudo
+RUN apt-get update && apt-get install -y pkg-config
+RUN apt-get update && apt-get install -y locales
+RUN apt-get update && apt-get install -y ca-certificates
+RUN apt-get update && apt-get install -y libssh2-1-dev
+RUN apt-get update && apt-get install -y software-properties-common
+RUN apt-get update && apt-get install -y sudo

 # GCC
 ARG gcc_major=13
@@ -24,7 +23,6 @@ RUN locale-gen en_US.UTF-8
 ARG docker_user="bts"
 ARG docker_uid="1000"
 ARG docker_pass="bts"
-# ARG entrypoint

 RUN \
  groupadd -g "$docker_uid" "$docker_user" \
@@ -1,22 +1,42 @@
 astroid==2.7.3
 autopep8==1.5.7
+blinker==1.9.0
+boto3==1.37.9
+botocore==1.37.9
 certifi==2023.5.7
 charset-normalizer==3.1.0
+click==8.1.8
 colorama==0.4.4
+Flask==3.1.0
 idna==3.4
 isort==5.9.3
+itsdangerous==2.2.0
+Jinja2==3.1.6
+jmespath==1.0.1
 lazy-object-proxy==1.6.0
+MarkupSafe==3.0.2
 mccabe==0.6.1
+numpy==2.2.6
 platformdirs==2.3.0
 protobuf==4.21.12
 pybars3==0.9.7
+pybind11==2.13.6
+pybind11-stubgen==2.5.3
+pybit==5.11.0
 pycodestyle==2.7.0
 pycparser==2.20
+pycryptodome==3.23.0
 pylint==2.10.2
 PyMeta3==0.5.1
+python-dateutil==2.9.0.post0
+python-dotenv==1.0.1
 requests==2.31.0
+s3transfer==0.11.4
+six==1.17.0
 termcolor==1.1.0
 toml==0.10.2
 urllib3==2.0.3
+websocket-client==1.8.0
+Werkzeug==3.1.3
 wrapt==1.12.1
 yapf==0.31.0
@@ -8,6 +8,7 @@
   public int Integer;
   public double Real;
   public string Identifier;
+   public DateTime DateTime; 
   public ImmutableList<IArithmeticExpression> Arguments;
   public ImmutableList<Assignment> Assignments;
   public IArithmeticExpression Argument;
@@ -1,7 +1,7 @@
 <Project Sdk="Microsoft.NET.Sdk">

  <PropertyGroup>
-    <TargetFramework>net7.0</TargetFramework>
+    <TargetFramework>net8.0</TargetFramework>
    <ImplicitUsings>enable</ImplicitUsings>
    <Nullable>enable</Nullable>
  </PropertyGroup>
@@ -1,7 +1,7 @@
 <Project Sdk="Microsoft.NET.Sdk">

  <PropertyGroup>
-    <TargetFramework>net7.0</TargetFramework>
+    <TargetFramework>net8.0</TargetFramework>
    <ImplicitUsings>enable</ImplicitUsings>
    <Nullable>enable</Nullable>
  </PropertyGroup>
@@ -1,7 +1,7 @@
 <Project Sdk="Microsoft.NET.Sdk">

  <PropertyGroup>
-    <TargetFramework>net7.0</TargetFramework>
+    <TargetFramework>net8.0</TargetFramework>
    <ImplicitUsings>enable</ImplicitUsings>
    <Nullable>enable</Nullable>
  </PropertyGroup>
@@ -1,6 +1,4 @@
 using System;
-using System.Runtime.Versioning;
-
 using Microsoft.Win32;

 namespace BCG
@@ -12,7 +10,6 @@ namespace BCG
         rk.SetValue (name, value.ToString ());
      }

-      [SupportedOSPlatform("windows")]
      public static double getValue (RegistryKey rk, string name)
      {
         object v = rk.GetValue (name);
@@ -1,11 +1,19 @@
 namespace Bts.BooleanExpressions;

+using bts;
+
 class BooleanExpression : IBooleanExpression {
   readonly Func<IDictionary<string, object?>, bool> evaluate;

   public BooleanExpression(Func<IDictionary<string, object?>, bool> evaluate) =>
      this.evaluate = evaluate;

-   public bool Evaluate(IDictionary<string, object?> env) =>
-      evaluate(env);
+   public bool Evaluate(IDictionary<string, object?> env) {
+      try {
+         return evaluate(env);
+      } catch (Exception ex) {
+         Log.assert(false, $"BooleanExpression.Evaluate failed: {ex}");
+         throw;
+      }
+   }
 }
@@ -47,6 +47,11 @@ IsOfType                 '/'

 %%

+{StringConstant} {
+   yylval.String = yytext.Substring(1, yytext.Length - 2).Replace(@"\'", "'");
+   return (int)Token.STRING_CONSTANT;
+} 
+
 {BooleanConstant} {
   yylval.Boolean = yytext.ToLower() switch {
      "true" => true,
@@ -57,11 +62,6 @@ IsOfType                 '/'
   return (int)Token.BOOLEAN_CONSTANT;
 }

-{StringConstant} {
-   yylval.String = yytext.Substring(1, yytext.Length - 2).Replace(@"\'", "'");
-   return (int)Token.STRING_CONSTANT;
-} 
-
 {NullConstant} {
   yylval.Null = DBNull.Value;
   return (int)Token.NULL_CONSTANT;
@@ -1,14 +1,14 @@
 <Project Sdk="Microsoft.NET.Sdk">

  <PropertyGroup>
-    <TargetFramework>net7.0</TargetFramework>
+    <TargetFramework>net8.0</TargetFramework>
    <ImplicitUsings>enable</ImplicitUsings>
    <Nullable>enable</Nullable>
  </PropertyGroup>

  <ItemGroup>
    <PackageReference Include="Google.Protobuf" Version="3.24.3" />
-    <PackageReference Include="YaccLexTools" Version="1.2.0" />
+    <PackageReference Include="YaccLexTools" Version="1.2.3" />
  </ItemGroup>

  <ItemGroup>
@@ -24,8 +24,6 @@ namespace QUT.GplexBuffers
        public BufferException(string message) : base(message) { }
        public BufferException(string message, Exception innerException)
            : base(message, innerException) { }
-        protected BufferException(SerializationInfo info, StreamingContext context)
-            : base(info, context) { }
    }

    internal abstract class ScanBuff
@@ -60,12 +58,10 @@ namespace QUT.GplexBuffers
            return new BuildBuffer(source);
        }

-#if (!BYTEMODE)
        public static ScanBuff GetBuffer(Stream source, int fallbackCodePage)
        {
            return new BuildBuffer(source, fallbackCodePage);
        }
-#endif // !BYTEMODE
 #endif // !NOFILES
    }

@@ -353,17 +349,15 @@ namespace QUT.GplexBuffers
        {
            FileStream fStrm = (stream as FileStream);
            if (fStrm != null) FileName = fStrm.Name;
-            NextBlk = BlockReaderFactory.Raw(stream);
+            NextBlk = BlockReaderFactory.Get(stream);
        }

-#if (!BYTEMODE)
        public BuildBuffer(Stream stream, int fallbackCodePage)
        {
            FileStream fStrm = (stream as FileStream);
            if (fStrm != null) FileName = fStrm.Name;
            NextBlk = BlockReaderFactory.Get(stream, fallbackCodePage);
        }
-#endif

        /// <summary>
        /// Marks a conservative lower bound for the buffer,
@@ -434,43 +428,19 @@ namespace QUT.GplexBuffers
    //
    internal static class BlockReaderFactory
    {
-        public static BlockReader Raw(Stream stream)
-        {
-            return delegate(char[] block, int index, int number)
-            {
-                byte[] b = new byte[number];
-                int count = stream.Read(b, 0, number);
-                int i = 0;
-                int j = index;
-                for (; i < count; i++, j++)
-                    block[j] = (char)b[i];
-                return count;
-            };
-        }

-#if (!BYTEMODE)
-        public static BlockReader Get(Stream stream, int fallbackCodePage)
+        public static BlockReader Get(Stream stream, int fallbackCodePage = -2)
        {
            Encoding encoding;
            int preamble = Preamble(stream);

            if (preamble != 0)  // There is a valid BOM here!
                encoding = Encoding.GetEncoding(preamble);
-            else if (fallbackCodePage == -1) // Fallback is "raw" bytes
-                return Raw(stream);
-            else if (fallbackCodePage != -2) // Anything but "guess"
+            else if (fallbackCodePage >= 0) // Anything but "guess"
                encoding = Encoding.GetEncoding(fallbackCodePage);
            else // This is the "guess" option
-            {
-                int guess = new Guesser(stream).GuessCodePage();
-                stream.Seek(0, SeekOrigin.Begin);
-                if (guess == -1) // ==> this is a 7-bit file
-                    encoding = Encoding.ASCII;
-                else if (guess == 65001)
-                    encoding = Encoding.UTF8;
-                else             // ==> use the machine default
-                    encoding = Encoding.Default;
-            }
+                encoding = GuessEncoding(stream);
+
            StreamReader reader = new StreamReader(stream, encoding);
            return reader.Read;
        }
@@ -488,6 +458,14 @@ namespace QUT.GplexBuffers
            int b2 = stream.ReadByte();
            if (b0 == 0xef && b1 == 0xbb && b2 == 0xbf)
                return 65001; // UTF8
+
+            int b3 = stream.ReadByte();
+            // Check for UTF-32 BOM (Byte Order Mark)
+            if (b0 == 0x00 && b1 == 0x00 && b2 == 0xFE && b3 == 0xFF)
+                return 12001; // UTF-32BE
+            if (b0 == 0xFF && b1 == 0xFE && b2 == 0x00 && b3 == 0x00)
+                return 12000; // UTF-32LE
+
            //
            // There is no unicode preamble, so we
            // return denoter for the machine default.
@@ -495,7 +473,150 @@ namespace QUT.GplexBuffers
            stream.Seek(0, SeekOrigin.Begin);
            return 0;
        }
-#endif // !BYTEMODE
+
+        private static Encoding GuessEncoding(Stream stream)
+        {
+            stream.Seek(0, SeekOrigin.Begin);
+            if (IsUtf16(stream, out Encoding encoding))
+            {
+                stream.Seek(0, SeekOrigin.Begin);
+                return encoding;
+            }
+
+            stream.Seek(0, SeekOrigin.Begin);
+            if (IsUtf8(stream))
+            {
+                stream.Seek(0, SeekOrigin.Begin);
+                return Encoding.UTF8;
+            }
+
+            stream.Seek(0, SeekOrigin.Begin);
+            if (IsAscii(stream))
+            {
+                stream.Seek(0, SeekOrigin.Begin);
+                return Encoding.ASCII;
+            }
+
+#if !NET462_OR_GREATER
+            stream.Seek(0, SeekOrigin.Begin);
+            if (IsLatin1(stream))
+            {
+                stream.Seek(0, SeekOrigin.Begin);
+                return Encoding.Latin1;
+            }
+#endif
+            stream.Seek(0, SeekOrigin.Begin);
+            return Encoding.Default;
+        }
+
+        private static bool IsUtf8(Stream stream)
+        {
+            int b;
+            while ((b = stream.ReadByte()) != -1)
+            {
+                if ((b & 0x80) == 0) // 0xxxxxxx
+                {
+                    continue;
+                }
+                else if ((b & 0xE0) == 0xC0) // 110xxxxx 10xxxxxx
+                {
+                    if (!IsValidUtf8ContinuationByte(stream)) return false;
+                }
+                else if ((b & 0xF0) == 0xE0) // 1110xxxx 10xxxxxx 10xxxxxx
+                {
+                    if (!IsValidUtf8ContinuationByte(stream) || !IsValidUtf8ContinuationByte(stream)) return false;
+                }
+                else if ((b & 0xF8) == 0xF0) // 11110xxx 10xxxxxx 10xxxxxx 10xxxxxx
+                {
+                    if (!IsValidUtf8ContinuationByte(stream) || !IsValidUtf8ContinuationByte(stream) || !IsValidUtf8ContinuationByte(stream)) return false;
+                }
+                else
+                {
+                    return false;
+                }
+            }
+            return true;
+        }
+
+        private static bool IsValidUtf8ContinuationByte(Stream stream)
+        {
+            int b = stream.ReadByte();
+            return b != -1 && (b & 0xC0) == 0x80; // 10xxxxxx
+        }
+
+        private static bool IsUtf16(Stream stream, out Encoding encoding)
+        {
+            encoding = Encoding.Default;
+
+            if (stream.Length >= 2)
+            {
+                // Read the first two bytes
+                int b0 = stream.ReadByte();
+                int b1 = stream.ReadByte();
+
+                // Check for UTF-16 BOM (Byte Order Mark)
+                if (b0 == 0xFF && b1 == 0xFE)
+                {
+                    encoding = Encoding.Unicode;
+                    return true; // UTF-16LE
+                }
+                if (b0 == 0xFE && b1 == 0xFF)
+                {
+                    encoding = Encoding.BigEndianUnicode;
+                    return true; // UTF-16BE
+                }
+
+                // Reset the stream position
+                stream.Seek(0, SeekOrigin.Begin);
+
+                // Check for common UTF-16 patterns
+                byte[] buffer = new byte[2];
+                while (stream.Read(buffer, 0, 2) == 2)
+                {
+                    if (buffer[0] == 0x00 && buffer[1] != 0x00)
+                    {
+                        encoding = Encoding.BigEndianUnicode;
+                        return true; // Likely UTF-16BE
+                    }
+                    if (buffer[0] != 0x00 && buffer[1] == 0x00)
+                    {
+                        encoding = Encoding.Unicode;
+                        return true; // Likely UTF-16LE
+                    }
+                }
+            }
+
+            return false;
+        }
+
+        private static bool IsAscii(Stream stream)
+        {
+            int b;
+            while ((b = stream.ReadByte()) != -1)
+            {
+                // Latin1 includes byte values from 0x00 to 0xFF
+                if (b > 0x7F)
+                {
+                    return false;
+                }
+            }
+            return true;
+        }
+
+        private static bool IsLatin1(Stream stream)
+        {
+            int b;
+            while ((b = stream.ReadByte()) != -1)
+            {
+                // Latin1 includes byte values from 0x00 to 0xFF
+                if (b > 0xFF)
+                {
+                    return false;
+                }
+            }
+            return true;
+        }
+
    }
 #endif // !NOFILES
    #endregion Buffer classes
@@ -540,180 +661,6 @@ namespace QUT.GplexBuffers
            return 0;
        }
    }
-#region guesser
-#if (!BYTEMODE)
-    // ==============================================================
-    // ============          Encoding Guesser           =============
-    // ==============================================================
-
-    /// <summary>
-    /// This class provides a simple finite state automaton that
-    /// scans the file looking for (1) valid UTF-8 byte patterns,
-    /// (2) bytes >= 0x80 which are not part of a UTF-8 sequence.
-    /// The method then guesses whether it is UTF-8 or maybe some 
-    /// local machine default encoding.  This works well for the
-    /// various Latin encodings.
-    /// </summary>
-    internal class Guesser
-    {
-        ScanBuff buffer;
-
-        public int GuessCodePage() { return Scan(); }
-
-        const int maxAccept = 10;
-        const int initial = 0;
-        const int eofNum = 0;
-        const int goStart = -1;
-        const int INITIAL = 0;
-        const int EndToken = 0;
-
-        #region user code
-        /* 
-         *  Reads the bytes of a file to determine if it is 
-         *  UTF-8 or a single-byte code page file.
-         */
-        public long utfX;
-        public long uppr;
-        #endregion user code
-
-        int state;
-        int currentStart = startState[0];
-        int code;
-
-        #region ScannerTables
-        static int[] startState = new int[] { 11, 0 };
-
-        #region CharacterMap
-        static sbyte[] map = new sbyte[256] {
-/*     '\0' */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 
-/*   '\x10' */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 
-/*   '\x20' */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 
-/*      '0' */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 
-/*      '@' */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 
-/*      'P' */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 
-/*      '`' */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 
-/*      'p' */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 
-/*   '\x80' */ 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 
-/*   '\x90' */ 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 
-/*   '\xA0' */ 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 
-/*   '\xB0' */ 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 
-/*   '\xC0' */ 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 
-/*   '\xD0' */ 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 
-/*   '\xE0' */ 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 
-/*   '\xF0' */ 4, 4, 4, 4, 4, 4, 4, 4, 5, 5, 5, 5, 5, 5, 5, 5 };
-        #endregion
-
-        static sbyte[][] nextState = new sbyte[][] {
-            new sbyte[] {0, 0, 0, 0, 0, 0},
-            new sbyte[] {-1, -1, 10, -1, -1, -1},
-            new sbyte[] {-1, -1, -1, -1, -1, -1},
-            new sbyte[] {-1, -1, 8, -1, -1, -1},
-            new sbyte[] {-1, -1, 5, -1, -1, -1},
-            new sbyte[] {-1, -1, 6, -1, -1, -1},
-            new sbyte[] {-1, -1, 7, -1, -1, -1},
-            null,
-            new sbyte[] {-1, -1, 9, -1, -1, -1},
-            null,
-            null,
-            new sbyte[] {-1, 1, 2, 3, 4, 2}
-        };
-
-
-        [SuppressMessage("Microsoft.Performance", "CA1810:InitializeReferenceTypeStaticFieldsInline")]
-        // Reason for suppression: cannot have self-reference in array initializer.
-        static Guesser()
-        {
-            nextState[7] = nextState[2];
-            nextState[9] = nextState[2];
-            nextState[10] = nextState[2];
-        }
-
-        int NextState()
-        {
-            if (code == ScanBuff.EndOfFile)
-                return eofNum;
-            else
-                return nextState[state][map[code]];
-        }
-        #endregion
-
-        public Guesser(System.IO.Stream file) { SetSource(file); }
-
-        public void SetSource(System.IO.Stream source)
-        {
-            this.buffer = new BuildBuffer(source);
-            code = buffer.Read();
-        }
-
-        int Scan()
-        {
-            for (; ; )
-            {
-                int next;
-                state = currentStart;
-                while ((next = NextState()) == goStart)
-                    code = buffer.Read();
-
-                state = next;
-                code = buffer.Read();
-
-                while ((next = NextState()) > eofNum)
-                {
-                    state = next;
-                    code = buffer.Read();
-                }
-                if (state <= maxAccept)
-                {
-                    #region ActionSwitch
-#pragma warning disable 162
-                    switch (state)
-                    {
-                        case eofNum:
-                            switch (currentStart)
-                            {
-                                case 11:
-                                    if (utfX == 0 && uppr == 0) return -1; /* raw ascii */
-                                    else if (uppr * 10 > utfX) return 0;   /* default code page */
-                                    else return 65001;                     /* UTF-8 encoding */
-                                    break;
-                            }
-                            return EndToken;
-                        case 1: // Recognized '{Upper128}',	Shortest string "\xC0"
-                        case 2: // Recognized '{Upper128}',	Shortest string "\x80"
-                        case 3: // Recognized '{Upper128}',	Shortest string "\xE0"
-                        case 4: // Recognized '{Upper128}',	Shortest string "\xF0"
-                            uppr++;
-                            break;
-                        case 5: // Recognized '{Utf8pfx4}{Utf8cont}',	Shortest string "\xF0\x80"
-                            uppr += 2;
-                            break;
-                        case 6: // Recognized '{Utf8pfx4}{Utf8cont}{2}',	Shortest string "\xF0\x80\x80"
-                            uppr += 3;
-                            break;
-                        case 7: // Recognized '{Utf8pfx4}{Utf8cont}{3}',	Shortest string "\xF0\x80\x80\x80"
-                            utfX += 3;
-                            break;
-                        case 8: // Recognized '{Utf8pfx3}{Utf8cont}',	Shortest string "\xE0\x80"
-                            uppr += 2;
-                            break;
-                        case 9: // Recognized '{Utf8pfx3}{Utf8cont}{2}',	Shortest string "\xE0\x80\x80"
-                            utfX += 2;
-                            break;
-                        case 10: // Recognized '{Utf8pfx2}{Utf8cont}',	Shortest string "\xC0\x80"
-                            utfX++;
-                            break;
-                        default:
-                            break;
-                    }
-#pragma warning restore 162
-                    #endregion
-                }
-            }
-        }
-    } // end class Guesser
-    
-#endif // !BYTEMODE
-#endregion
 #endif // !NOFILES

 // End of code copied from embedded resource
@@ -1,7 +1,7 @@
 <Project Sdk="Microsoft.NET.Sdk">

  <PropertyGroup>
-    <TargetFramework>net7.0</TargetFramework>
+    <TargetFramework>net8.0</TargetFramework>
    <ImplicitUsings>enable</ImplicitUsings>
    <Nullable>enable</Nullable>
  </PropertyGroup>
@@ -1,7 +1,7 @@
 <Project Sdk="Microsoft.NET.Sdk">

  <PropertyGroup>
-    <TargetFramework>net7.0</TargetFramework>
+    <TargetFramework>net8.0</TargetFramework>
    <ImplicitUsings>enable</ImplicitUsings>
    <Nullable>enable</Nullable>
  </PropertyGroup>
@@ -1 +0,0 @@
-VITE_DATASTREAM_WEBSOCKET_URL=wss://localhost:443
@@ -1 +0,0 @@
-VITE_DATASTREAM_WEBSOCKET_URL=wss://localhost:443
@@ -1,32 +0,0 @@
-module.exports = {
-  root: true,
-  env: { browser: true, es2020: true },
-  extends: [
-    'eslint:recommended',
-    'plugin:react/recommended',
-    'plugin:react/jsx-runtime',
-    'plugin:react-hooks/recommended',
-  ],
-  ignorePatterns: ['dist', '.eslintrc.cjs'],
-  parserOptions: { ecmaVersion: 'latest', sourceType: 'module' },
-  settings: { react: { version: '18.3.1' } },
-  plugins: ['react-refresh'],
-  rules: {
-    "import/order": [
-      "error",
-      {
-        "groups": [
-          "builtin",
-          "external",
-          "internal",
-          ["parent", "sibling", "index"]
-        ],
-        "newlines-between": "always"
-      }
-    ],
-    'react-refresh/only-export-components': [
-      'warn',
-      { allowConstantExport: true },
-    ],
-  },
-}
@@ -1,24 +0,0 @@
-# Logs
-logs
-*.log
-npm-debug.log*
-yarn-debug.log*
-yarn-error.log*
-pnpm-debug.log*
-lerna-debug.log*
-
-node_modules
-dist
-dist-ssr
-*.local
-
-# Editor directories and files
-.vscode/*
-!.vscode/extensions.json
-.idea
-.DS_Store
-*.suo
-*.ntvs*
-*.njsproj
-*.sln
-*.sw?
@@ -1,11 +0,0 @@
-<Project Sdk="Microsoft.VisualStudio.JavaScript.Sdk/0.5.128-alpha">
-  <PropertyGroup>
-    <StartupCommand>npm run dev</StartupCommand>
-    <JavaScriptTestRoot>src\</JavaScriptTestRoot>
-    <JavaScriptTestFramework>Jest</JavaScriptTestFramework>
-    <!-- Allows the build (or compile) script located on package.json to run on Build -->
-    <ShouldRunBuildScript>true</ShouldRunBuildScript>
-    <!-- Folder where production build objects will be placed -->
-    <PublishAssetsDirectory>$(MSBuildProjectDirectory)\distill</PublishAssetsDirectory>
-  </PropertyGroup>
-</Project>
@@ -1,8 +0,0 @@
-# React + Vite
-
-This template provides a minimal setup to get React working in Vite with HMR and some ESLint rules.
-
-Currently, two official plugins are available:
-
- [@vitejs/plugin-react](https://github.com/vitejs/vite-plugin-react/blob/main/packages/plugin-react/README.md) uses [Babel](https://babeljs.io/) for Fast Refresh
- [@vitejs/plugin-react-swc](https://github.com/vitejs/vite-plugin-react-swc) uses [SWC](https://swc.rs/) for Fast Refresh
@@ -1,13 +0,0 @@
-<!doctype html>
-<html lang="en">
-  <head>
-    <meta charset="UTF-8" />
-    <link rel="icon" type="image/ico" href="/logo.ico" />
-    <meta name="viewport" content="width=device-width, initial-scale=1.0" />
-    <title>BTS Cloud Streams</title>
-  </head>
-  <body>
-    <div id="root"></div>
-    <script type="module" src="/src/main.jsx"></script>
-  </body>
-</html>
@@ -1,10 +0,0 @@
-<?xml version="1.0" encoding="utf-8"?>
-<configuration>
-  <packageSources>
-    <clear />
-    <add key="nuget.org" value="https://api.nuget.org/v3/index.json" />
-  </packageSources>
-  <disabledPackageSources>
-    <clear />
-  </disabledPackageSources>
-</configuration>
@@ -1,44 +0,0 @@
-{
-  "name": "bts.datastreamwebapp.client",
-  "private": true,
-  "version": "0.0.0",
-  "type": "module",
-  "scripts": {
-    "dev": "vite",
-    "build": "vite build",
-    "lint": "eslint . --ext js,jsx --report-unused-disable-directives --max-warnings 0",
-    "preview": "vite preview"
-  },
-  "dependencies": {
-    "@angular/material": "^18.0.4",
-    "@emotion/react": "^11.11.4",
-    "@emotion/styled": "^11.11.5",
-    "@fontsource/noto-sans": "^5.0.22",
-    "@material/data-table": "^15.0.0-canary.7f224ddd4.0",
-    "@mui/icons-material": "^5.15.20",
-    "@mui/material": "^5.15.20",
-    "@mui/x-data-grid": "^7.7.1",
-    "bulma": "^1.0.1",
-    "classnames": "^2.5.1",
-    "react": "^18.3.1",
-    "react-data-grid": "^7.0.0-beta.44",
-    "react-data-table-component": "^7.6.2",
-    "react-dom": "^18.3.1",
-    "react-json-view-lite": "^1.4.0",
-    "react-pro-sidebar": "^1.1.0",
-    "react-use-websocket": "^4.8.1",
-    "rpc-websockets": "^7.11.2"
-  },
-  "devDependencies": {
-    "@types/react": "^18.3.3",
-    "@types/react-dom": "^18.3.0",
-    "@vitejs/plugin-react": "^4.3.1",
-    "eslint": "^8.57.0",
-    "eslint-plugin-import": "^2.29.1",
-    "eslint-plugin-react": "^7.34.2",
-    "eslint-plugin-react-hooks": "^4.6.2",
-    "eslint-plugin-react-refresh": "^0.4.7",
-    "react-mui-sidebar": "^1.3.5",
-    "vite": "^5.2.13"
-  }
-}
@@ -1 +0,0 @@
-<svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" class="iconify iconify--logos" width="31.88" height="32" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 257"><defs><linearGradient id="IconifyId1813088fe1fbc01fb466" x1="-.828%" x2="57.636%" y1="7.652%" y2="78.411%"><stop offset="0%" stop-color="#41D1FF"></stop><stop offset="100%" stop-color="#BD34FE"></stop></linearGradient><linearGradient id="IconifyId1813088fe1fbc01fb467" x1="43.376%" x2="50.316%" y1="2.242%" y2="89.03%"><stop offset="0%" stop-color="#FFEA83"></stop><stop offset="8.333%" stop-color="#FFDD35"></stop><stop offset="100%" stop-color="#FFA800"></stop></linearGradient></defs><path fill="url(#IconifyId1813088fe1fbc01fb466)" d="M255.153 37.938L134.897 252.976c-2.483 4.44-8.862 4.466-11.382.048L.875 37.958c-2.746-4.814 1.371-10.646 6.827-9.67l120.385 21.517a6.537 6.537 0 0 0 2.322-.004l117.867-21.483c5.438-.991 9.574 4.796 6.877 9.62Z"></path><path fill="url(#IconifyId1813088fe1fbc01fb467)" d="M185.432.063L96.44 17.501a3.268 3.268 0 0 0-2.634 3.014l-5.474 92.456a3.268 3.268 0 0 0 3.997 3.378l24.777-5.718c2.318-.535 4.413 1.507 3.936 3.838l-7.361 36.047c-.495 2.426 1.782 4.5 4.151 3.78l15.304-4.649c2.372-.72 4.652 1.36 4.15 3.788l-11.698 56.621c-.732 3.542 3.979 5.473 5.943 2.437l1.313-2.028l72.516-144.72c1.215-2.423-.88-5.186-3.54-4.672l-25.505 4.922c-2.396.462-4.435-1.77-3.759-4.114l16.646-57.705c.677-2.35-1.37-4.583-3.769-4.113Z"></path></svg>
@@ -1,40 +0,0 @@
-import { Sidebar, Menu, MenuItem, Submenu, Logo } from "react-mui-sidebar";
-import './App.css';
-
-const AnimatedSideBar = () => {
-  return (
-    <Sidebar className={"sidebar-container"} width={"270px"}>
-      <Logo img="https://adminmart.com/wp-content/uploads/2024/03/logo-admin-mart-news.png">
-        AdminMart
-      </Logo>
-      <Menu subHeading="HOME">
-        <MenuItem link="/" badge="true">
-          Modern
-        </MenuItem>
-        <MenuItem>eCommerce</MenuItem>
-        <MenuItem>Analytical</MenuItem>
-      </Menu>
-      <Menu subHeading="APPS">
-        <MenuItem>Chat</MenuItem>
-        <MenuItem>Calendar</MenuItem>
-      </Menu>
-      <Menu subHeading="OTHERS">
-        <Submenu title="Menu Level">
-          <MenuItem>Post</MenuItem>
-          <MenuItem>Details</MenuItem>
-          <Submenu title="Level 2">
-            <MenuItem>new</MenuItem>
-            <MenuItem>Hello</MenuItem>
-          </Submenu>
-        </Submenu>
-
-        <MenuItem>Chip</MenuItem>
-        <MenuItem target="_blank" link="google.com">
-          External Link
-        </MenuItem>
-      </Menu>
-    </Sidebar>
-  );
-};
-
-export default AnimatedSideBar;
@@ -1,101 +0,0 @@
-@use "@material/checkbox";
-@use "@material/icon-button";
-@use "@material/data-table/data-table";
-
-.root-div {
-  display: flex;
-  flex-direction: row;
-  align-items: flex-start;
-}
-
-.main-content-div {
-  display: flex;
-  flex-direction: row;
-}
-
-.title-div {
-  width: 100%;
-}
-
-.container {
-  display: flex;
-}
-
-.container-column {
-  display: flex;
-  flex-direction: column;
-}
-
-.container-row {
-  display: flex;
-  flex-direction: row;
-  align-items: stretch;
-}
-
-box.query-input-form {
-   /* p: 2, display: "flex", flexFlow: "column", alignItems: "stretch"  */
-  display: flex;
-  flex-flow: column;
-  align-items: stretch;
-}
-
-/* li.ps-menuitem-root:hover {
-  background-color: #ff9922;  
-} */
-
-.container-row-centered {
-  display: flex;
-  flex-direction: row;
-  justify-content: center;
-  align-items: center;
-}
-
-h1#main-header {
-  padding: 4px 8px 4px 8px;
-  text-align: center;
-}
-
-td:hover {
-  background-color: cadetblue;
-}
-
-td#cell-decimal {
-  text-align: right;
-}
-
-td#cell-integer {
-  text-align: right;
-}
-
-td#cell-string {
-  text-align: center;
-}
-
-td#cell-date-time {
-  text-align: center;
-}
-
-/* Sidebar.css */
-/* .sidebar-container {
-  display: flex;
-  flex-direction: column;
-}
-
-.sidebar {
-  position: relative;
-  top: 0;
-  left: 0;
-  height: 100vh;
-  background-color: #333;
-  color: white;
-  overflow: hidden;
-  transition: transform 0.3s ease;
-}
-
-.sidebar.closed {
-  width: 0;
-}
-
-.sidebar.open {
-  width: 250px;
-} */
@@ -1,173 +0,0 @@
-import { ThemeProvider, CssBaseline, GlobalStyles } from '@mui/material';
-import QueryStatusView from './QueryStatusView';
-import useMediaQuery from '@mui/material/useMediaQuery';
-import QueryControl from './QueryControl';
-import DataTable from './DataTable';
-import { useState, useEffect, useMemo } from 'react';
-import useWebSocket, { ReadyState } from 'react-use-websocket';
-import Sidebar from './SideBar';
-import getTheme from './Theme';
-
-import '@fontsource/noto-sans';
-import './App.css';
-import './SideBar.css';
-
-const webSocketUrl = import.meta.env.VITE_DATASTREAM_WEBSOCKET_URL;
-// Always use the same subscription id (1), because we only show 1 table at a time.
-// The server will automatically dispose the previous subscription and start a new one
-// if we're currently subscribed to another query.
-const subscriptionId = "1";
-
-var rpcReqIdCounter = 1;
-
-function subscribeReq(query) {
-  return {
-    jsonrpc: "2.0",
-    id: rpcReqIdCounter++,
-    method: "subscribe",
-    params: {
-      subscription_id: subscriptionId,
-      query: query
-    }
-  };
-}
-
-function unsubscribeReq() {
-  return {
-    jsonrpc: "2.0",
-    id: rpcReqIdCounter++,
-    method: "unsubscribe",
-    params: {
-      subscription_id: subscriptionId,
-    }
-  };
-}
-
-const App = () => {
-  const prefersDarkMode = useMediaQuery('(prefers-color-scheme: dark)');
-  const theme = getTheme({ mode: prefersDarkMode ? 'dark' : 'light' });
-  const globalStyles = {
-    ':root': {
-      '--primary-color': theme.palette.primary.main,
-      '--primary-dark-color': theme.palette.primary.dark,
-      '--secondary-color': theme.palette.secondary.main,
-      '--secondary-dark-color': theme.palette.secondary.dark,
-      '--background-color': theme.palette.background.default,
-    },
-  };
-  const [blueTable, setBlueTable] = useState({ columns: null, rows: null });
-  const [queryError, setQueryError] = useState(null);
-  const [queryState, setQueryState] = useState("Unsubscribed");
-  const queryStateDetails = useMemo(() => {
-    if (queryState == "Unsubscribed") {
-      return "Enter query above, then click the play button to subscribe";
-    }
-    if (queryState == "Subscribed") {
-      return null;
-    }
-  }, [queryState]);
-  const [sidebarIsOpen, setSidebarIsOpen] = useState(false);
-  const [cannedQuery, setCannedQuery] = useState(null);
-
-  const onCannedQueryClicked = (cannedQuery) => {
-    console.info(`Executing canned query: ${JSON.stringify(cannedQuery)}`);
-    const queryText = cannedQuery.query.join("\n");
-    setCannedQuery({ queryText: queryText, clickTime: Date.now() });
-    subscribe(queryText);
-  };
-
-  const { sendJsonMessage, getWebSocket, readyState } = useWebSocket(
-    webSocketUrl,
-    {
-      onOpen: (e) => console.info(`WebSocket open. ${JSON.stringify(e)}`),
-      onClose: (e) => console.info(`WebSocket closed. ${JSON.stringify(e)}`),
-      onError: (e) => console.error(`WebSocket error: ${JSON.stringify(e)}`),
-      shouldReconnect: () => true,
-    });
-
-  useEffect(() => {
-    console.info(`WebSocket readyState: ${readyState}`)
-    if (readyState != ReadyState.OPEN) {
-      return;
-    }
-    const ws = getWebSocket();
-    ws.onmessage = e => {
-      console.info(`Received message: ${e.data}`);
-      const msg = JSON.parse(e.data);
-      setQueryError(msg.error);
-      if (msg.error != null) {
-        setQueryState("Error");
-      } else if (msg.id != null) {
-        setQueryState(msg.result.status);
-      } else {
-        console.assert(msg.params.subscription_id == subscriptionId);
-        console.info(`Received rows: ${msg.params.table.all}`);
-        setBlueTable({ columns: msg.params.table.columns, rows: msg.params.table.all });
-      }
-    };
-  }, [getWebSocket, readyState]);
-
-  const subscribe = (query) => {
-    const req = subscribeReq(query);
-    console.info(`Subscribing: ${JSON.stringify(subscribeReq(query))}`);
-    sendJsonMessage(req);
-  };
-
-  const unsubscribe = () => {
-    const req = unsubscribeReq();
-    console.info(`Unsubscribing: ${JSON.stringify(req)}`);
-    sendJsonMessage(req);
-  };
-
-  const onOpened = () => {
-    setSidebarIsOpen(true);
-  };
-
-  const onClosed = () => {
-    setSidebarIsOpen(false);
-  };
-
-  const sidebarWidth = sidebarIsOpen ? '250px' : '65px';
-  return (
-    <ThemeProvider theme={theme}>
-      <CssBaseline />
-      <GlobalStyles styles={globalStyles} />
-      <div className='root-div'>
-        <div style={{
-          display: 'flex',
-          flexDirection: 'column',
-          alignItems: 'flex-start',
-          flexWrap: 'nowrap',
-          width: sidebarWidth,
-          transition: 'width 0.3s ease',
-        }}>
-          <Sidebar onSelection={onCannedQueryClicked} onOpened={onOpened} onClosed={onClosed} />
-        </div>
-        <div style={{
-          display: 'flex',
-          flexDirection: 'column',
-          left: sidebarWidth,
-          width: '100%',
-          transition: 'width 0.3s ease',
-        }}>
-          <div style={{ width: '100%', display: 'flex', flexDirection: 'column', alignItems: 'flex-start', margin: theme.spacing(2, 0, 0, 0) }}>
-            <div style={{ width: '90%', rightMargin: '100px' }}>
-              <QueryControl
-                queryState={queryState}
-                queryText={cannedQuery?.queryText}
-                queryTextResetTime={cannedQuery?.clickTime}
-                subscribe={subscribe}
-                unsubscribe={unsubscribe} />
-            </div>
-            <div style={{ width: '90%', rightMargin: '100px' }}>
-              <QueryStatusView queryState={queryState} details={queryStateDetails} error={queryError} />
-              <DataTable columns={blueTable?.columns ?? []} rows={blueTable?.rows ?? []} />
-            </div>
-          </div>
-        </div>
-      </div>
-    </ThemeProvider>
-  );
-};
-
-export default App;
@@ -1,53 +0,0 @@
-@use "@material/checkbox";
-@use "@material/icon-button";
-@use "@material/data-table/data-table";
-
-.data-grid-container {
-  overflow: auto;
-  white-space: nowrap;
-}
-
-
-.data-grid {
-  max-height: 1000px;
-}
-
-div[role=columnheader] {
-  border-top: 0px;
-  border-bottom: 0px;
-  background-color: var(--primary-dark-color);
-}
-
-/* div[role=columnheader] {
-  border-top: 0px;
-  border-bottom: 0px;
-  background-color: var(--primary-dark-color);
-}
-
-.rdg-row {
-  border: 0px;
-  border-top: 0px;
-  border-bottom: 0px;
-  background-color: rgba(var(--primary-dark-color), 0.8);
-  color: var(--primary-color);
-}
-
-.rdg-row-odd {
-  border: 0px;
-  border-top: 0px;
-  border-bottom: 0px;
-  background-color: var(--primary-dark-color);
-  color: var(--primary-color);
-}
-
-.rdg-cell {
-  border-bottom: 0px;
-}
-
-.rdg-cell.even {
-  border-bottom: 5px;
-}
-
-.rdg-cell.odd {
-  border-bottom: 5px;
-} */
@@ -1,32 +0,0 @@
-@use "@material/checkbox";
-@use "@material/icon-button";
-@use "@material/data-table/data-table";
-
-th {
-  font-size: large;  
-  padding: 15px;
-}
-
-td {
-  padding: 15px;
-}
-
-td:hover {
-  background-color: cadetblue;
-}
-
-td#cell-decimal {
-  text-align: right;
-}
-
-td#cell-integer {
-  text-align: right;
-}
-
-td#cell-string {
-  text-align: center;
-}
-
-td#cell-date-time {
-  text-align: center;
-}
@@ -1,74 +0,0 @@
-import PropTypes from 'prop-types';
-import { useTheme } from '@mui/material/styles';
-import { DataGrid } from '@mui/x-data-grid';
-import './DataGrid.css'
-import { useMemo, useState } from 'react';
-
-export default function DataTable({ columns, rows }) {
-  const theme = useTheme();
-  const [columnWidths, setColumnWidths] = useState({});
-
-  const handleColumnResize = (params) => {
-    const newWidth = params.width;
-    setColumnWidths((prevWidths) => ({
-      ...prevWidths,
-      [params.colDef.field]: params.width,
-    }));
-  };
-
-  const getGridColDef = useMemo(() => {
-    return col => {
-      return {
-        field: col.name,
-        headerName: col.name,
-        width: columnWidths[col.name],
-      };
-    }
-  }, [columnWidths]);
-
-  return (
-    <DataGrid
-      className='data-grid-container'
-      autosizeOnMount={false}
-      columns={columns?.map(getGridColDef) ?? []}
-      onColumnResize={handleColumnResize}
-      rows={rows ?? []}
-      sx={{
-        margin: theme.spacing(2, 0, 0, 0),
-        borderColor: 'primary.main',
-        backgroundColor: 'primary.dark',
-        color: 'text.primary',
-        '& .MuiDataGrid-columnHeader': {
-          color: 'primary.main',
-          fontWeight: 700,
-          backgroundColor: 'primary.dark'
-        },
-        '& .MuiDataGrid-filler': {
-          color: 'primary.main',
-          fontWeight: 700,
-          backgroundColor: 'primary.dark'
-        },
-        '& .MuiDataGrid-cell': {
-          color: 'text.primary',
-        },
-        '& .MuiDataGrid-cell:hover': {
-          color: 'primary.main',
-        },
-        '& .MuiDataGrid-row': {
-          '--DataGrid-rowBorderColor': 'secondary.dark'
-        }
-      }}
-    />
-  );
-}
-
-DataTable.propTypes = {
-  columns: PropTypes.arrayOf(
-    PropTypes.shape({
-      name: PropTypes.string.isRequired,
-      type: PropTypes.string.isRequired,
-    })
-  ).isRequired,
-  rows: PropTypes.arrayOf(PropTypes.array).isRequired,
-  apiRef: PropTypes.object,
-};
@@ -1,119 +0,0 @@
-import PropTypes from 'prop-types';
-import DataTable from './DataTable';
-import { useRef, useState, useEffect } from 'react';
-import { useGridApiRef } from '@mui/x-data-grid';
-import Box from '@mui/material/Box';
-import Button from '@mui/material/Button';
-import Grid from '@mui/material/Grid';
-import Paper from '@mui/material/Paper';
-import PlayArrowIcon from '@mui/icons-material/PlayArrow';
-import StopIcon from '@mui/icons-material/Stop';
-import TextField from '@mui/material/TextField';
-import Tooltip from '@mui/material/Tooltip';
-
-const QueryControl = (props) => {
-  const { queryState, queryText, queryTextResetTime, subscribe, unsubscribe } = props;
-  // const theme = useTheme();
-  let textInput = useRef();
-  const [queryTextChangeTime, setQueryTextChangeTime] = useState(null);
-  useEffect(() => {
-    if (queryText == null || textInput.current == null) {
-      return;
-    }
-    // Init text field (unless a non-empty string is already present, or the user has modified
-    // the query - we don't want to overwrite their changes).
-    if (queryTextChangeTime == null || queryTextResetTime > queryTextChangeTime) {
-      console.info(`Initializing query text: ${queryText}`)
-      textInput.current.value = queryText;
-    }
-  }, [queryText, queryTextChangeTime, queryTextResetTime]);
-  const onClearClick = () => {
-    textInput.current.value = '';
-  };
-
-  const onRunClick = () => {
-    subscribe(textInput.current.value);
-  };
-
-  const onStopClick = () => {
-    unsubscribe();
-  };
-
-  const onKeyDown = (e) => {
-    // Detect keyboard shortcuts.
-    //   Ctrl+Enter : Run
-    if (e.ctrlKey && e.key === 'Enter') {
-      onRunClick();
-    }
-  };
-
-  const onChange = (e) => {
-    // So we know not to overwrite the changes with the canned query on re-render
-    setQueryTextChangeTime(Date.now());
-  };
-
-  return (
-    <Box sx={{ display: "flex", flexFlow: "column" }}>
-      <Paper elevation={3} sx={{ color: 'primary.main' }}>
-        <Box sx={{ p: 2 }}>
-          <TextField
-            fullWidth
-            multiline
-            rows={10}
-            variant="outlined"
-            onChange={onChange}
-            onKeyDown={onKeyDown}
-            inputRef={textInput}
-            placeholder="Enter Query..."
-          />
-        </Box>
-        <Box sx={{ p: 2, borderTop: 1 }}>
-          <Grid container spacing={2} justifyContent="flex-start">
-            <Grid item xs>
-              <Tooltip title="Clear Query Text">
-                <Button onClick={onClearClick} sx={{ padding: '11px', color: 'text.dark' }}>
-                  Clear
-                </Button>
-              </Tooltip>
-            </Grid>
-            <Grid item>
-              <Tooltip title="Run (Ctrl+Enter)">
-                <Button onClick={onRunClick} sx={{
-                  color: `queryStartButton.${queryState}.color`,
-                  bgcolor: `queryStartButton.${queryState}.backgroundColor`,
-                  padding: '11px'
-                }}>
-                  <PlayArrowIcon />
-                </Button>
-              </Tooltip>
-            </Grid>
-            <Grid item>
-              <Tooltip title="Stop">
-                <Button onClick={onStopClick} sx={{
-                  color: `queryStopButton.${queryState}.color`,
-                  bgcolor: `queryStopButton.${queryState}.backgroundColor`,
-                  padding: '11px'
-                }}>
-                  <StopIcon />
-                </Button>
-              </Tooltip>
-            </Grid>
-          </Grid>
-        </Box>
-      </Paper>
-    </Box>
-  );
-};
-
-QueryControl.propTypes = {
-  columns: PropTypes.array,
-  rows: PropTypes.array,
-  isRunning: PropTypes.bool,
-  queryText: PropTypes.string,
-  queryTextResetTime: PropTypes.number,
-  queryState: PropTypes.string,
-  subscribe: PropTypes.func,
-  unsubscribe: PropTypes.func,
-};
-
-export default QueryControl;
@@ -1,73 +0,0 @@
-import { useState } from 'react';
-import PropTypes from 'prop-types';
-import { Box, Typography, Collapse } from '@mui/material';
-import Alert from '@mui/material/Alert';
-import AlertTitle from '@mui/material/AlertTitle';
-import { useTheme } from '@mui/material/styles';
-import { alpha } from '@mui/system';
-
-const QueryStatusView = ({ queryState, details, error }) => {
-  const theme = useTheme();
-  const [open, setOpen] = useState(false);
-
-  let header = queryState;
-  let headerContent = null;
-  let detailElements = [];
-  let severity = queryState == "Subscribed" ? "success" : "info";
-  let cursor = details == null ? "default" : "pointer";
-  if (error != null) {
-    severity = "error";
-    cursor = "pointer";
-    header = `Error Code: ${error.code}`;
-    headerContent = error.message;
-    detailElements.push(<Typography variant="body1">{error.data.error_message}</Typography>);
-    detailElements.push(<Typography variant="subtitle1">Stack Trace:</Typography>);
-    (error.data.stack_trace ?? []).forEach(line => detailElements.push(
-      <Typography variant="body2" component="pre" sx={{ whiteSpace: 'pre-wrap' }}>
-        {line}
-      </Typography>));
-  } else if (details != null) {
-    detailElements.push(<Typography variant="body1">{details}</Typography>);
-  }
-  return (
-    <Box sx={{ margin: theme.spacing(2, 0, 0, 0) }}>
-      <Alert
-        severity={severity}
-        onClick={() => setOpen(!open)}
-        sx={{
-          margin: theme.spacing(2, 0, 0, 0),
-          cursor: cursor,
-        }}>
-        <AlertTitle>{header}</AlertTitle>
-        {headerContent}
-      </Alert>
-      {detailElements.length == 0
-        ? null
-        : <Collapse in={open}>
-          <Box
-            sx={{
-              backgroundColor: alpha(theme.palette[severity].light, 0.75),
-              color: theme.palette[severity].contrastText,
-              padding: theme.spacing(2),
-              marginTop: theme.spacing(0),
-              borderRadius: theme.shape.borderRadius,
-            }}
-          >
-            {detailElements}
-          </Box>
-        </Collapse>}
-    </Box>
-  );
-};
-
-QueryStatusView.propTypes = {
-  queryState: PropTypes.string,
-  details: PropTypes.string,
-  error: PropTypes.shape({
-    code: PropTypes.number,
-    message: PropTypes.string,
-    data: PropTypes.object,
-  })
-};
-
-export default QueryStatusView;
@@ -1,27 +0,0 @@
-import 'react-data-grid/lib/styles.css';
-import PropTypes from 'prop-types';
-import DataGrid from 'react-data-grid';
-import './DataGrid.css';
-
-function getGridColDef(col) {
-  return {
-    key: col.name,
-    name: col.name,
-  };
-}
-
-export default function ReactDataGrid({columns, rows}) {
-  return (
-    <DataGrid className={"data-grid"} rows={rows} columns={columns.map(getGridColDef)} />
-  )
-}
-
-ReactDataGrid.propTypes = {
-  columns: PropTypes.arrayOf(
-    PropTypes.shape({
-      name: PropTypes.string.isRequired,
-      type: PropTypes.string.isRequired,
-    })
-  ).isRequired,
-  rows: PropTypes.arrayOf(PropTypes.array).isRequired,
-};
@@ -1,98 +0,0 @@
-/* Sidebar.css */
-.app-container {
-  display: flex;
-  width: 100%;
-  overflow: hidden;
-  flex-direction: row;
-  justify-content: flex-start;
-  align-items: flex-start;
-}
-
-.sidebar {
-  display: flex;
-  flex-direction: row;
-  justify-content: flex-start;
-  align-items: flex-start;
-  background-color: transparent;
-  color: white;
-  overflow: hidden;
-  transition: width 0.3s ease;
-}
-
-.sidebar.open {
-  width: 100%;
-}
-
-.sidebar.closed {
-  width: 0%;
-}
-
-.sidebar-toggle-button {
-  border: 0px;
-  border-radius: 50%;
-  box-shadow: rgba(9, 30, 66, 0.08) 0px 0px 0px 1px, rgba(9, 30, 66, 0.08) 0px 2px 4px 1px;
-  color: var(--primary-color);
-  cursor: pointer;
-  height: 24px;
-  opacity: 1;
-  outline: 0px;
-  padding: 0px;
-  background-color: transparent;
-  transition: background-color 100ms linear 0s, color 100ms linear 0s, opacity 300ms cubic-bezier(0.2, 0, 0, 1) 0s, transform 300ms cubic-bezier(0.2, 0, 0, 1) 0s;
-  width: 20%;
-}
-
-.sidebar-toggle-button:hover {
-  transition: background-color 100ms linear 0s, color 100ms linear 0s, opacity 300ms cubic-bezier(0.2, 0, 0, 1) 0s, transform 300ms cubic-bezier(0.2, 0, 0, 1) 0s;
-}
-
-.main-content.open {
-  width: 165px;
-  transition: width 0.3s ease;
-  background-color: transparent;
-}
-
-.main-content.closed {
-  width: 0%;
-  /* min-width: 25px; */
-  transition: width 0.3s ease;
-  background-color: transparent;
-}
-
-.sidebar-menu-item {
-  width: 100%;
-  box-sizing: border-box;
-}
-
-.sidebar-menu-item.open {
-  width: 100%;
-  /* justify-content: flex-start; */
-  transition: width 0.3s ease;
-}
-
-.sidebar-menu-item.closed {
-  width: 0%;
-  min-width: 0px;
-  /* justify-content: flex-start; */
-  transition: width 0.3s ease;
-}
-
-.sidebar-menu-item-content.open {
-  width: 100%;
-  /* justify-content: flex-start; */
-  transition: width 0.3s ease;
-}
-
-.sidebar-menu-item-content.closed {
-  width: 0px;
-  /* justify-content: flex-start; */
-  transition: width 0.3s ease;
-}
-
-/* Ensure span can shrink */
-.button-text {
-  display: block;    /* Make span a block element */
-  width: 100%;       /* Ensure it takes the full width of its parent */
-  overflow: hidden;  /* Hide any overflow content */
-  white-space: nowrap; /* Prevent text wrapping */
-}
@@ -1,109 +0,0 @@
-import { useState, useEffect } from 'react';
-import Button from '@mui/material/Button';
-import { useTheme } from '@mui/material/styles';
-import PropTypes from 'prop-types';
-import Tooltip from '@mui/material/Tooltip';
-import ChevronLeftRounded from "@mui/icons-material/ChevronLeftRounded"
-import ChevronRightRounded from '@mui/icons-material/ChevronRightRounded';
-import FileCopyIcon from '@mui/icons-material/FileCopy';
-import './SideBar.css';
-
-const Sidebar = ({ onSelection, onOpened, onClosed }) => {
-  const theme = useTheme();
-  const [isOpen, setIsOpen] = useState(false);
-  const [cannedQueries, setCannedQueries] = useState([]);
-  const Icon = isOpen ? ChevronLeftRounded : ChevronRightRounded;
-
-  const toggleIsOpen = () => {
-    if (isOpen) {
-      setIsOpen(false);
-      if (onClosed != null) {
-        onClosed();
-      }
-    } else {
-      setIsOpen(true);
-      if (onOpened != null) {
-        onOpened();
-      }
-    }
-  };
-
-  const onMenuItemClick = (event, cannedQuery) => {
-    console.info(`menu item clicked: ${cannedQuery.name}`);
-    onSelection(cannedQuery);
-  };
-
-  // const onMenuItemClick = async (event, cannedQuery) => {
-  //   try {
-  //     await navigator.clipboard.writeText(cannedQuery.query.join("\n"));
-  //     alert(`Copied ${cannedQuery.name} to clipboard`);
-  //   } catch (err) {
-  //     console.error('Failed to copy query text: ', err);
-  //   }
-  // };
-
-  useEffect(() => {
-    const fetchCannedQueries = async () => {
-      try {
-        const response = await fetch('api/cannedqueries');
-        const data = await response.json();
-        setCannedQueries(data);
-      } catch (error) {
-        console.error('Error fetching WebSocket config:', error);
-      }
-    }
-    fetchCannedQueries();
-  }, []);
-
-  const mainContentClass = `main-content ${isOpen ? 'open' : 'closed'}`;
-  const menuItemClass = `sidebar-menu-item ${isOpen ? 'open' : 'closed'}`;
-  const menuItemContentClass = `sidebar-menu-item-content ${isOpen ? 'open' : 'closed'}`;
-  const toggleButtonTitle = `${isOpen ? 'Hide' : 'Show'} Common Queries`;
-
-  return (
-    <div
-      className={"sidebar"}
-      style={{
-        display: 'flex',
-        flexDirection: 'row',
-        justifyContent: 'flexStart',
-        alignItems: 'flexStart',
-        margin: theme.spacing(2, 0, 0, 0)
-      }}
-    >
-      <div className={mainContentClass}>
-        {cannedQueries.map(cq =>
-          <Button key={cq.name} className={menuItemClass} onClick={(event) => onMenuItemClick(event, cq)}>
-            <div className={menuItemContentClass}>
-              <span className='button-text'>{cq.name}</span>
-            </div>
-          </Button>)
-        }
-        {/* {cannedQueries.map(cq =>
-          <Button
-            variant="contained"
-            color="primary"
-            startIcon={<FileCopyIcon />}
-            onClick={onMenuItemClick}
-          >
-            <div className={menuItemContentClass}>
-              <span className='button-text'>{cq.name}</span>
-            </div>
-          </Button>)} */}
-      </div>
-      <Tooltip title={toggleButtonTitle}>
-        <Button onClick={toggleIsOpen}>
-          <Icon className="w-5 h-5" />
-        </Button>
-      </Tooltip>
-    </div>
-  );
-};
-
-export default Sidebar;
-
-Sidebar.propTypes = {
-  onSelection: PropTypes.func,
-  onOpened: PropTypes.func,
-  onClosed: PropTypes.func,
-};
@@ -1,112 +0,0 @@
-import { createTheme } from '@mui/material/styles';
-import PropTypes from 'prop-types';
-import '@fontsource/noto-sans';
-import './App.css';
-
-const getLightPalette = () => {
-  return {
-    mode: 'light',
-    primary: {
-      light: '#6573c3',
-      main: '#3f51b5',
-      dark: '#2c387e',
-      contrastText: '#ffffff',
-    },
-    secondary: {
-      light: '#f73378',
-      main: '#f50057',
-      dark: '#ab003c',
-      contrastText: '#ffffff',
-    },
-    text: {
-      primary: '#000000c4',
-      secondary: '#00000086',
-      disabled: '#0000005b',
-      hint: '#0000005b',
-    }
-  };
-};
-
-const getDarkPalette = () => {
-  return {
-    mode: 'dark',
-    primary: {
-      light: '#bcc5ff',
-      main: '#33baf9',
-      dark: '#202334',
-      contrastText: '#78a2ac',
-    },
-    secondary: {
-      light: '#9fd0b8',
-      main: '#5eb289',
-      dark: '#ffffff12',
-      contrastText: '#c8d4f7',
-    },
-    action: {
-      hoverOpacity: 0.08
-    },
-    text: {
-      primary: '#eaf8f8',
-      dark: '#eaf8f8a0'
-    },
-    queryStartButton: {
-      Subscribed: {
-        color: '#33baf9',
-        backgroundColor: '#ffffff40',
-      },
-      Unsubscribed: {
-        color: '#ffffff40',
-        backgroundColor: '#ffffff00',
-      }
-    },
-    queryStopButton: {
-      Subscribed: {
-        color: '#ff4444a0',
-        backgroundColor: '#ffffff40',
-      },
-      Unsubscribed: {
-        color: '#ff444450',
-        backgroundColor: '#ffffff00',
-      }
-    },
-    error: {
-      light: '#e57373',
-      main: '#f44336',
-      dark: '#d32f2f',
-      contrastText: '#fff',
-    },
-    warning: {
-      light: '#ffb74d',
-      main: '#ff9800',
-      dark: '#f57c00',
-      contrastText: '#000',
-    },
-    info: {
-      light: '#64b5f6',
-      main: '#2196f3',
-      dark: '#303030',
-      contrastText: '#fff',
-    },
-    success: {
-      light: '#81c784',
-      main: '#4caf50',
-      dark: '#388e3c',
-      contrastText: '#000',
-    },
-  };
-};
-
-const getTheme = ({ mode }) => {
-  return createTheme({
-    typography: {
-      fontFamily: '"Helvetica"'
-    },
-    palette: mode == 'dark' ? getDarkPalette() : getLightPalette()
-  });
-};
-
-export default getTheme;
-
-getTheme.propTypes = {
-  mode: PropTypes.string.isRequired,
-};
@@ -1 +0,0 @@
-<svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" class="iconify iconify--logos" width="35.93" height="32" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 228"><path fill="#00D8FF" d="M210.483 73.824a171.49 171.49 0 0 0-8.24-2.597c.465-1.9.893-3.777 1.273-5.621c6.238-30.281 2.16-54.676-11.769-62.708c-13.355-7.7-35.196.329-57.254 19.526a171.23 171.23 0 0 0-6.375 5.848a155.866 155.866 0 0 0-4.241-3.917C100.759 3.829 77.587-4.822 63.673 3.233C50.33 10.957 46.379 33.89 51.995 62.588a170.974 170.974 0 0 0 1.892 8.48c-3.28.932-6.445 1.924-9.474 2.98C17.309 83.498 0 98.307 0 113.668c0 15.865 18.582 31.778 46.812 41.427a145.52 145.52 0 0 0 6.921 2.165a167.467 167.467 0 0 0-2.01 9.138c-5.354 28.2-1.173 50.591 12.134 58.266c13.744 7.926 36.812-.22 59.273-19.855a145.567 145.567 0 0 0 5.342-4.923a168.064 168.064 0 0 0 6.92 6.314c21.758 18.722 43.246 26.282 56.54 18.586c13.731-7.949 18.194-32.003 12.4-61.268a145.016 145.016 0 0 0-1.535-6.842c1.62-.48 3.21-.974 4.76-1.488c29.348-9.723 48.443-25.443 48.443-41.52c0-15.417-17.868-30.326-45.517-39.844Zm-6.365 70.984c-1.4.463-2.836.91-4.3 1.345c-3.24-10.257-7.612-21.163-12.963-32.432c5.106-11 9.31-21.767 12.459-31.957c2.619.758 5.16 1.557 7.61 2.4c23.69 8.156 38.14 20.213 38.14 29.504c0 9.896-15.606 22.743-40.946 31.14Zm-10.514 20.834c2.562 12.94 2.927 24.64 1.23 33.787c-1.524 8.219-4.59 13.698-8.382 15.893c-8.067 4.67-25.32-1.4-43.927-17.412a156.726 156.726 0 0 1-6.437-5.87c7.214-7.889 14.423-17.06 21.459-27.246c12.376-1.098 24.068-2.894 34.671-5.345a134.17 134.17 0 0 1 1.386 6.193ZM87.276 214.515c-7.882 2.783-14.16 2.863-17.955.675c-8.075-4.657-11.432-22.636-6.853-46.752a156.923 156.923 0 0 1 1.869-8.499c10.486 2.32 22.093 3.988 34.498 4.994c7.084 9.967 14.501 19.128 21.976 27.15a134.668 134.668 0 0 1-4.877 4.492c-9.933 8.682-19.886 14.842-28.658 17.94ZM50.35 144.747c-12.483-4.267-22.792-9.812-29.858-15.863c-6.35-5.437-9.555-10.836-9.555-15.216c0-9.322 13.897-21.212 37.076-29.293c2.813-.98 5.757-1.905 8.812-2.773c3.204 10.42 7.406 21.315 12.477 32.332c-5.137 11.18-9.399 22.249-12.634 32.792a134.718 134.718 0 0 1-6.318-1.979Zm12.378-84.26c-4.811-24.587-1.616-43.134 6.425-47.789c8.564-4.958 27.502 2.111 47.463 19.835a144.318 144.318 0 0 1 3.841 3.545c-7.438 7.987-14.787 17.08-21.808 26.988c-12.04 1.116-23.565 2.908-34.161 5.309a160.342 160.342 0 0 1-1.76-7.887Zm110.427 27.268a347.8 347.8 0 0 0-7.785-12.803c8.168 1.033 15.994 2.404 23.343 4.08c-2.206 7.072-4.956 14.465-8.193 22.045a381.151 381.151 0 0 0-7.365-13.322Zm-45.032-43.861c5.044 5.465 10.096 11.566 15.065 18.186a322.04 322.04 0 0 0-30.257-.006c4.974-6.559 10.069-12.652 15.192-18.18ZM82.802 87.83a323.167 323.167 0 0 0-7.227 13.238c-3.184-7.553-5.909-14.98-8.134-22.152c7.304-1.634 15.093-2.97 23.209-3.984a321.524 321.524 0 0 0-7.848 12.897Zm8.081 65.352c-8.385-.936-16.291-2.203-23.593-3.793c2.26-7.3 5.045-14.885 8.298-22.6a321.187 321.187 0 0 0 7.257 13.246c2.594 4.48 5.28 8.868 8.038 13.147Zm37.542 31.03c-5.184-5.592-10.354-11.779-15.403-18.433c4.902.192 9.899.29 14.978.29c5.218 0 10.376-.117 15.453-.343c-4.985 6.774-10.018 12.97-15.028 18.486Zm52.198-57.817c3.422 7.8 6.306 15.345 8.596 22.52c-7.422 1.694-15.436 3.058-23.88 4.071a382.417 382.417 0 0 0 7.859-13.026a347.403 347.403 0 0 0 7.425-13.565Zm-16.898 8.101a358.557 358.557 0 0 1-12.281 19.815a329.4 329.4 0 0 1-23.444.823c-7.967 0-15.716-.248-23.178-.732a310.202 310.202 0 0 1-12.513-19.846h.001a307.41 307.41 0 0 1-10.923-20.627a310.278 310.278 0 0 1 10.89-20.637l-.001.001a307.318 307.318 0 0 1 12.413-19.761c7.613-.576 15.42-.876 23.31-.876H128c7.926 0 15.743.303 23.354.883a329.357 329.357 0 0 1 12.335 19.695a358.489 358.489 0 0 1 11.036 20.54a329.472 329.472 0 0 1-11 20.722Zm22.56-122.124c8.572 4.944 11.906 24.881 6.52 51.026c-.344 1.668-.73 3.367-1.15 5.09c-10.622-2.452-22.155-4.275-34.23-5.408c-7.034-10.017-14.323-19.124-21.64-27.008a160.789 160.789 0 0 1 5.888-5.4c18.9-16.447 36.564-22.941 44.612-18.3ZM128 90.808c12.625 0 22.86 10.235 22.86 22.86s-10.235 22.86-22.86 22.86s-22.86-10.235-22.86-22.86s10.235-22.86 22.86-22.86Z"></path></svg>
@@ -1,51 +0,0 @@
-.dark-high-contrast {
-  --md-sys-color-primary: rgb(251 250 255);
-  --md-sys-color-surface-tint: rgb(165 200 255);
-  --md-sys-color-on-primary: rgb(0 0 0);
-  --md-sys-color-primary-container: rgb(173 204 255);
-  --md-sys-color-on-primary-container: rgb(0 0 0);
-  --md-sys-color-secondary: rgb(250 250 255);
-  --md-sys-color-on-secondary: rgb(0 0 0);
-  --md-sys-color-secondary-container: rgb(159 208 255);
-  --md-sys-color-on-secondary-container: rgb(0 0 0);
-  --md-sys-color-tertiary: rgb(255 249 251);
-  --md-sys-color-on-tertiary: rgb(0 0 0);
-  --md-sys-color-tertiary-container: rgb(222 193 231);
-  --md-sys-color-on-tertiary-container: rgb(0 0 0);
-  --md-sys-color-error: rgb(255 249 249);
-  --md-sys-color-on-error: rgb(0 0 0);
-  --md-sys-color-error-container: rgb(255 186 177);
-  --md-sys-color-on-error-container: rgb(0 0 0);
-  --md-sys-color-background: rgb(17 19 24);
-  --md-sys-color-on-background: rgb(225 226 233);
-  --md-sys-color-surface: rgb(17 19 24);
-  --md-sys-color-on-surface: rgb(255 255 255);
-  --md-sys-color-surface-variant: rgb(67 71 78);
-  --md-sys-color-on-surface-variant: rgb(251 250 255);
-  --md-sys-color-outline: rgb(200 202 212);
-  --md-sys-color-outline-variant: rgb(200 202 212);
-  --md-sys-color-shadow: rgb(0 0 0);
-  --md-sys-color-scrim: rgb(0 0 0);
-  --md-sys-color-inverse-surface: rgb(225 226 233);
-  --md-sys-color-inverse-on-surface: rgb(0 0 0);
-  --md-sys-color-inverse-primary: rgb(0 42 83);
-  --md-sys-color-primary-fixed: rgb(219 231 255);
-  --md-sys-color-on-primary-fixed: rgb(0 0 0);
-  --md-sys-color-primary-fixed-dim: rgb(173 204 255);
-  --md-sys-color-on-primary-fixed-variant: rgb(0 22 49);
-  --md-sys-color-secondary-fixed: rgb(214 233 255);
-  --md-sys-color-on-secondary-fixed: rgb(0 0 0);
-  --md-sys-color-secondary-fixed-dim: rgb(159 208 255);
-  --md-sys-color-on-secondary-fixed-variant: rgb(0 24 42);
-  --md-sys-color-tertiary-fixed: rgb(249 222 255);
-  --md-sys-color-on-tertiary-fixed: rgb(0 0 0);
-  --md-sys-color-tertiary-fixed-dim: rgb(222 193 231);
-  --md-sys-color-on-tertiary-fixed-variant: rgb(33 14 42);
-  --md-sys-color-surface-dim: rgb(17 19 24);
-  --md-sys-color-surface-bright: rgb(55 57 62);
-  --md-sys-color-surface-container-lowest: rgb(12 14 19);
-  --md-sys-color-surface-container-low: rgb(25 28 32);
-  --md-sys-color-surface-container: rgb(29 32 36);
-  --md-sys-color-surface-container-high: rgb(40 42 47);
-  --md-sys-color-surface-container-highest: rgb(50 53 58);
-}
@@ -1,51 +0,0 @@
-.dark-medium-contrast {
-  --md-sys-color-primary: rgb(173 204 255);
-  --md-sys-color-surface-tint: rgb(165 200 255);
-  --md-sys-color-on-primary: rgb(0 22 49);
-  --md-sys-color-primary-container: rgb(112 146 198);
-  --md-sys-color-on-primary-container: rgb(0 0 0);
-  --md-sys-color-secondary: rgb(159 208 255);
-  --md-sys-color-on-secondary: rgb(0 24 42);
-  --md-sys-color-secondary-container: rgb(100 149 193);
-  --md-sys-color-on-secondary-container: rgb(0 0 0);
-  --md-sys-color-tertiary: rgb(222 193 231);
-  --md-sys-color-on-tertiary: rgb(33 14 42);
-  --md-sys-color-tertiary-container: rgb(162 136 171);
-  --md-sys-color-on-tertiary-container: rgb(0 0 0);
-  --md-sys-color-error: rgb(255 186 177);
-  --md-sys-color-on-error: rgb(55 0 1);
-  --md-sys-color-error-container: rgb(255 84 73);
-  --md-sys-color-on-error-container: rgb(0 0 0);
-  --md-sys-color-background: rgb(17 19 24);
-  --md-sys-color-on-background: rgb(225 226 233);
-  --md-sys-color-surface: rgb(17 19 24);
-  --md-sys-color-on-surface: rgb(251 250 255);
-  --md-sys-color-surface-variant: rgb(67 71 78);
-  --md-sys-color-on-surface-variant: rgb(200 202 212);
-  --md-sys-color-outline: rgb(160 163 171);
-  --md-sys-color-outline-variant: rgb(128 131 139);
-  --md-sys-color-shadow: rgb(0 0 0);
-  --md-sys-color-scrim: rgb(0 0 0);
-  --md-sys-color-inverse-surface: rgb(225 226 233);
-  --md-sys-color-inverse-on-surface: rgb(40 42 47);
-  --md-sys-color-inverse-primary: rgb(35 73 120);
-  --md-sys-color-primary-fixed: rgb(212 227 255);
-  --md-sys-color-on-primary-fixed: rgb(0 17 40);
-  --md-sys-color-primary-fixed-dim: rgb(165 200 255);
-  --md-sys-color-on-primary-fixed-variant: rgb(10 55 100);
-  --md-sys-color-secondary-fixed: rgb(206 229 255);
-  --md-sys-color-on-secondary-fixed: rgb(0 18 34);
-  --md-sys-color-secondary-fixed-dim: rgb(154 203 250);
-  --md-sys-color-on-secondary-fixed-variant: rgb(0 57 91);
-  --md-sys-color-tertiary-fixed: rgb(247 216 255);
-  --md-sys-color-on-tertiary-fixed: rgb(28 9 37);
-  --md-sys-color-tertiary-fixed-dim: rgb(218 189 226);
-  --md-sys-color-on-tertiary-fixed-variant: rgb(67 46 76);
-  --md-sys-color-surface-dim: rgb(17 19 24);
-  --md-sys-color-surface-bright: rgb(55 57 62);
-  --md-sys-color-surface-container-lowest: rgb(12 14 19);
-  --md-sys-color-surface-container-low: rgb(25 28 32);
-  --md-sys-color-surface-container: rgb(29 32 36);
-  --md-sys-color-surface-container-high: rgb(40 42 47);
-  --md-sys-color-surface-container-highest: rgb(50 53 58);
-}
@@ -1,51 +0,0 @@
-.dark {
-  --md-sys-color-primary: rgb(165 200 255);
-  --md-sys-color-surface-tint: rgb(165 200 255);
-  --md-sys-color-on-primary: rgb(0 49 94);
-  --md-sys-color-primary-container: rgb(34 72 118);
-  --md-sys-color-on-primary-container: rgb(212 227 255);
-  --md-sys-color-secondary: rgb(154 203 250);
-  --md-sys-color-on-secondary: rgb(0 51 83);
-  --md-sys-color-secondary-container: rgb(12 74 114);
-  --md-sys-color-on-secondary-container: rgb(206 229 255);
-  --md-sys-color-tertiary: rgb(218 189 226);
-  --md-sys-color-on-tertiary: rgb(61 41 70);
-  --md-sys-color-tertiary-container: rgb(85 63 93);
-  --md-sys-color-on-tertiary-container: rgb(247 216 255);
-  --md-sys-color-error: rgb(255 180 171);
-  --md-sys-color-on-error: rgb(105 0 5);
-  --md-sys-color-error-container: rgb(147 0 10);
-  --md-sys-color-on-error-container: rgb(255 218 214);
-  --md-sys-color-background: rgb(17 19 24);
-  --md-sys-color-on-background: rgb(225 226 233);
-  --md-sys-color-surface: rgb(17 19 24);
-  --md-sys-color-on-surface: rgb(225 226 233);
-  --md-sys-color-surface-variant: rgb(67 71 78);
-  --md-sys-color-on-surface-variant: rgb(195 198 207);
-  --md-sys-color-outline: rgb(141 145 153);
-  --md-sys-color-outline-variant: rgb(67 71 78);
-  --md-sys-color-shadow: rgb(0 0 0);
-  --md-sys-color-scrim: rgb(0 0 0);
-  --md-sys-color-inverse-surface: rgb(225 226 233);
-  --md-sys-color-inverse-on-surface: rgb(46 48 53);
-  --md-sys-color-inverse-primary: rgb(60 96 144);
-  --md-sys-color-primary-fixed: rgb(212 227 255);
-  --md-sys-color-on-primary-fixed: rgb(0 28 58);
-  --md-sys-color-primary-fixed-dim: rgb(165 200 255);
-  --md-sys-color-on-primary-fixed-variant: rgb(34 72 118);
-  --md-sys-color-secondary-fixed: rgb(206 229 255);
-  --md-sys-color-on-secondary-fixed: rgb(0 29 50);
-  --md-sys-color-secondary-fixed-dim: rgb(154 203 250);
-  --md-sys-color-on-secondary-fixed-variant: rgb(12 74 114);
-  --md-sys-color-tertiary-fixed: rgb(247 216 255);
-  --md-sys-color-on-tertiary-fixed: rgb(39 20 48);
-  --md-sys-color-tertiary-fixed-dim: rgb(218 189 226);
-  --md-sys-color-on-tertiary-fixed-variant: rgb(85 63 93);
-  --md-sys-color-surface-dim: rgb(17 19 24);
-  --md-sys-color-surface-bright: rgb(55 57 62);
-  --md-sys-color-surface-container-lowest: rgb(12 14 19);
-  --md-sys-color-surface-container-low: rgb(25 28 32);
-  --md-sys-color-surface-container: rgb(29 32 36);
-  --md-sys-color-surface-container-high: rgb(40 42 47);
-  --md-sys-color-surface-container-highest: rgb(50 53 58);
-}
@@ -1,51 +0,0 @@
-.light-high-contrast {
-  --md-sys-color-primary: rgb(0 34 70);
-  --md-sys-color-surface-tint: rgb(60 96 144);
-  --md-sys-color-on-primary: rgb(255 255 255);
-  --md-sys-color-primary-container: rgb(29 68 114);
-  --md-sys-color-on-primary-container: rgb(255 255 255);
-  --md-sys-color-secondary: rgb(0 36 61);
-  --md-sys-color-on-secondary: rgb(255 255 255);
-  --md-sys-color-secondary-container: rgb(3 70 110);
-  --md-sys-color-on-secondary-container: rgb(255 255 255);
-  --md-sys-color-tertiary: rgb(46 26 55);
-  --md-sys-color-on-tertiary: rgb(255 255 255);
-  --md-sys-color-tertiary-container: rgb(81 59 89);
-  --md-sys-color-on-tertiary-container: rgb(255 255 255);
-  --md-sys-color-error: rgb(78 0 2);
-  --md-sys-color-on-error: rgb(255 255 255);
-  --md-sys-color-error-container: rgb(140 0 9);
-  --md-sys-color-on-error-container: rgb(255 255 255);
-  --md-sys-color-background: rgb(249 249 255);
-  --md-sys-color-on-background: rgb(25 28 32);
-  --md-sys-color-surface: rgb(249 249 255);
-  --md-sys-color-on-surface: rgb(0 0 0);
-  --md-sys-color-surface-variant: rgb(224 226 236);
-  --md-sys-color-on-surface-variant: rgb(32 36 43);
-  --md-sys-color-outline: rgb(63 67 74);
-  --md-sys-color-outline-variant: rgb(63 67 74);
-  --md-sys-color-shadow: rgb(0 0 0);
-  --md-sys-color-scrim: rgb(0 0 0);
-  --md-sys-color-inverse-surface: rgb(46 48 53);
-  --md-sys-color-inverse-on-surface: rgb(255 255 255);
-  --md-sys-color-inverse-primary: rgb(228 236 255);
-  --md-sys-color-primary-fixed: rgb(29 68 114);
-  --md-sys-color-on-primary-fixed: rgb(255 255 255);
-  --md-sys-color-primary-fixed-dim: rgb(0 45 88);
-  --md-sys-color-on-primary-fixed-variant: rgb(255 255 255);
-  --md-sys-color-secondary-fixed: rgb(3 70 110);
-  --md-sys-color-on-secondary-fixed: rgb(255 255 255);
-  --md-sys-color-secondary-fixed-dim: rgb(0 47 77);
-  --md-sys-color-on-secondary-fixed-variant: rgb(255 255 255);
-  --md-sys-color-tertiary-fixed: rgb(81 59 89);
-  --md-sys-color-on-tertiary-fixed: rgb(255 255 255);
-  --md-sys-color-tertiary-fixed-dim: rgb(57 37 66);
-  --md-sys-color-on-tertiary-fixed-variant: rgb(255 255 255);
-  --md-sys-color-surface-dim: rgb(217 218 224);
-  --md-sys-color-surface-bright: rgb(249 249 255);
-  --md-sys-color-surface-container-lowest: rgb(255 255 255);
-  --md-sys-color-surface-container-low: rgb(243 243 250);
-  --md-sys-color-surface-container: rgb(237 237 244);
-  --md-sys-color-surface-container-high: rgb(231 232 238);
-  --md-sys-color-surface-container-highest: rgb(225 226 233);
-}
@@ -1,51 +0,0 @@
-.light-medium-contrast {
-  --md-sys-color-primary: rgb(29 68 114);
-  --md-sys-color-surface-tint: rgb(60 96 144);
-  --md-sys-color-on-primary: rgb(255 255 255);
-  --md-sys-color-primary-container: rgb(83 118 167);
-  --md-sys-color-on-primary-container: rgb(255 255 255);
-  --md-sys-color-secondary: rgb(3 70 110);
-  --md-sys-color-on-secondary: rgb(255 255 255);
-  --md-sys-color-secondary-container: rgb(71 121 163);
-  --md-sys-color-on-secondary-container: rgb(255 255 255);
-  --md-sys-color-tertiary: rgb(81 59 89);
-  --md-sys-color-on-tertiary: rgb(255 255 255);
-  --md-sys-color-tertiary-container: rgb(133 108 141);
-  --md-sys-color-on-tertiary-container: rgb(255 255 255);
-  --md-sys-color-error: rgb(140 0 9);
-  --md-sys-color-on-error: rgb(255 255 255);
-  --md-sys-color-error-container: rgb(218 52 46);
-  --md-sys-color-on-error-container: rgb(255 255 255);
-  --md-sys-color-background: rgb(249 249 255);
-  --md-sys-color-on-background: rgb(25 28 32);
-  --md-sys-color-surface: rgb(249 249 255);
-  --md-sys-color-on-surface: rgb(25 28 32);
-  --md-sys-color-surface-variant: rgb(224 226 236);
-  --md-sys-color-on-surface-variant: rgb(63 67 74);
-  --md-sys-color-outline: rgb(91 95 103);
-  --md-sys-color-outline-variant: rgb(119 123 131);
-  --md-sys-color-shadow: rgb(0 0 0);
-  --md-sys-color-scrim: rgb(0 0 0);
-  --md-sys-color-inverse-surface: rgb(46 48 53);
-  --md-sys-color-inverse-on-surface: rgb(240 240 247);
-  --md-sys-color-inverse-primary: rgb(165 200 255);
-  --md-sys-color-primary-fixed: rgb(83 118 167);
-  --md-sys-color-on-primary-fixed: rgb(255 255 255);
-  --md-sys-color-primary-fixed-dim: rgb(58 93 141);
-  --md-sys-color-on-primary-fixed-variant: rgb(255 255 255);
-  --md-sys-color-secondary-fixed: rgb(71 121 163);
-  --md-sys-color-on-secondary-fixed: rgb(255 255 255);
-  --md-sys-color-secondary-fixed-dim: rgb(43 96 137);
-  --md-sys-color-on-secondary-fixed-variant: rgb(255 255 255);
-  --md-sys-color-tertiary-fixed: rgb(133 108 141);
-  --md-sys-color-on-tertiary-fixed: rgb(255 255 255);
-  --md-sys-color-tertiary-fixed-dim: rgb(107 84 116);
-  --md-sys-color-on-tertiary-fixed-variant: rgb(255 255 255);
-  --md-sys-color-surface-dim: rgb(217 218 224);
-  --md-sys-color-surface-bright: rgb(249 249 255);
-  --md-sys-color-surface-container-lowest: rgb(255 255 255);
-  --md-sys-color-surface-container-low: rgb(243 243 250);
-  --md-sys-color-surface-container: rgb(237 237 244);
-  --md-sys-color-surface-container-high: rgb(231 232 238);
-  --md-sys-color-surface-container-highest: rgb(225 226 233);
-}
@@ -1,51 +0,0 @@
-.light {
-  --md-sys-color-primary: rgb(60 96 144);
-  --md-sys-color-surface-tint: rgb(60 96 144);
-  --md-sys-color-on-primary: rgb(255 255 255);
-  --md-sys-color-primary-container: rgb(212 227 255);
-  --md-sys-color-on-primary-container: rgb(0 28 58);
-  --md-sys-color-secondary: rgb(46 98 140);
-  --md-sys-color-on-secondary: rgb(255 255 255);
-  --md-sys-color-secondary-container: rgb(206 229 255);
-  --md-sys-color-on-secondary-container: rgb(0 29 50);
-  --md-sys-color-tertiary: rgb(110 86 118);
-  --md-sys-color-on-tertiary: rgb(255 255 255);
-  --md-sys-color-tertiary-container: rgb(247 216 255);
-  --md-sys-color-on-tertiary-container: rgb(39 20 48);
-  --md-sys-color-error: rgb(186 26 26);
-  --md-sys-color-on-error: rgb(255 255 255);
-  --md-sys-color-error-container: rgb(255 218 214);
-  --md-sys-color-on-error-container: rgb(65 0 2);
-  --md-sys-color-background: rgb(249 249 255);
-  --md-sys-color-on-background: rgb(25 28 32);
-  --md-sys-color-surface: rgb(249 249 255);
-  --md-sys-color-on-surface: rgb(25 28 32);
-  --md-sys-color-surface-variant: rgb(224 226 236);
-  --md-sys-color-on-surface-variant: rgb(67 71 78);
-  --md-sys-color-outline: rgb(116 119 127);
-  --md-sys-color-outline-variant: rgb(195 198 207);
-  --md-sys-color-shadow: rgb(0 0 0);
-  --md-sys-color-scrim: rgb(0 0 0);
-  --md-sys-color-inverse-surface: rgb(46 48 53);
-  --md-sys-color-inverse-on-surface: rgb(240 240 247);
-  --md-sys-color-inverse-primary: rgb(165 200 255);
-  --md-sys-color-primary-fixed: rgb(212 227 255);
-  --md-sys-color-on-primary-fixed: rgb(0 28 58);
-  --md-sys-color-primary-fixed-dim: rgb(165 200 255);
-  --md-sys-color-on-primary-fixed-variant: rgb(34 72 118);
-  --md-sys-color-secondary-fixed: rgb(206 229 255);
-  --md-sys-color-on-secondary-fixed: rgb(0 29 50);
-  --md-sys-color-secondary-fixed-dim: rgb(154 203 250);
-  --md-sys-color-on-secondary-fixed-variant: rgb(12 74 114);
-  --md-sys-color-tertiary-fixed: rgb(247 216 255);
-  --md-sys-color-on-tertiary-fixed: rgb(39 20 48);
-  --md-sys-color-tertiary-fixed-dim: rgb(218 189 226);
-  --md-sys-color-on-tertiary-fixed-variant: rgb(85 63 93);
-  --md-sys-color-surface-dim: rgb(217 218 224);
-  --md-sys-color-surface-bright: rgb(249 249 255);
-  --md-sys-color-surface-container-lowest: rgb(255 255 255);
-  --md-sys-color-surface-container-low: rgb(243 243 250);
-  --md-sys-color-surface-container: rgb(237 237 244);
-  --md-sys-color-surface-container-high: rgb(231 232 238);
-  --md-sys-color-surface-container-highest: rgb(225 226 233);
-}
@@ -1,107 +0,0 @@
-:root {
-  font-family: Inter, system-ui, Avenir, Helvetica, Arial, sans-serif;
-  line-height: 1.5;
-  font-weight: 400;
-
-  color-scheme: light dark;
-  color: rgba(255, 255, 255, 0.87);
-  background-color: #242424;
-
-  font-synthesis: none;
-  text-rendering: optimizeLegibility;
-  -webkit-font-smoothing: antialiased;
-  -moz-osx-font-smoothing: grayscale;
-}
-
-a {
-  font-weight: 500;
-  color: #646cff;
-  text-decoration: inherit;
-}
-a:hover {
-  color: #535bf2;
-}
-
-body {
-  margin: 0;
-  display: flex;
-  place-items: center;
-  min-width: 320px;
-  min-height: 100vh;
-}
-
-button {
-  border-radius: 8px;
-  border: 1px solid transparent;
-  padding: 0.6em 1.2em;
-  font-size: 1em;
-  font-weight: 500;
-  font-family: inherit;
-  background-color: #1a1a1a;
-  cursor: pointer;
-  transition: border-color 0.25s;
-}
-button:hover {
-  border-color: #646cff;
-}
-button:focus,
-button:focus-visible {
-  outline: 4px auto -webkit-focus-ring-color;
-}
-
-button#query-button {
-  height: 5em;
-  vertical-align: top;
-}
-
-button#play-button {
-  padding: '10px 20px';
-  font-size: '16px';
-  cursor: 'pointer';
-  background-color: '#28a745';
-  color: '#fff';
-  border: 'none';
-  border-radius: '5px';
-};
-
-button#pause-button {
-  background-color: #ffc107;
-}
-
-button#stop-button {
-  background-color: #dc3545;
-}
-
-h1 {
-  font-size: 3.2em;
-  line-height: 1.1;
-}
-
-p#footer {
-  float: none;
-  clear: both;
-  vertical-align: bottom;
-}
-
-table#query-output {
-  float: none;
-  clear: both;
-}
-
-textarea#query-input {
-  height: 5em;
-  width: 50em;
-}
-
-@media (prefers-color-scheme: light) {
-  :root {
-    color: #213547;
-    background-color: #ffffff;
-  }
-  a:hover {
-    color: #747bff;
-  }
-  button {
-    background-color: #f9f9f9;
-  }
-}
@@ -1,7 +0,0 @@
-import React from 'react';
-import ReactDOM from 'react-dom/client';
-import App from './App';
-
-ReactDOM.createRoot(document.getElementById('root')).render(
-  <App />
-)
@@ -1,68 +0,0 @@
-import { fileURLToPath, URL } from 'node:url';
-import { defineConfig, loadEnv } from 'vite';
-import plugin from '@vitejs/plugin-react';
-import fs from 'fs';
-import path from 'path';
-import child_process from 'child_process';
-import process, { env } from 'process';
-
-const baseFolder =
-  process.env.APPDATA !== undefined && process.env.APPDATA !== ''
-    ? `${process.env.APPDATA}/ASP.NET/https`
-    : `${process.env.HOME}/.aspnet/https`;
-
-const certificateName = "bts.datastreamwebapp.client";
-const certFilePath = path.join(baseFolder, `${certificateName}.pem`);
-const keyFilePath = path.join(baseFolder, `${certificateName}.key`);
-
-if (!fs.existsSync(certFilePath) || !fs.existsSync(keyFilePath)) {
-  if (0 !== child_process.spawnSync('dotnet', [
-    'dev-certs',
-    'https',
-    '--export-path',
-    certFilePath,
-    '--format',
-    'Pem',
-    '--no-password',
-  ], { stdio: 'inherit', }).status) {
-    throw new Error("Could not create certificate.");
-  }
-}
-
-export default defineConfig(({ $$command, mode, $$isSsrBuild, $$isPreview }) => {
-  const target = env.ASPNETCORE_HTTPS_PORT
-    ? `https://localhost:${env.ASPNETCORE_HTTPS_PORT}`
-    : env.ASPNETCORE_URLS
-      ? env.ASPNETCORE_URLS.split(';')[0]
-      : 'https://localhost:8443';
-  return {
-    plugins: [plugin()],
-    resolve: {
-      alias: {
-        '@': fileURLToPath(new URL('./src', import.meta.url))
-      }
-    },
-    build: {
-      minify: false,
-      sourcemap: true,
-      outDir: `../../../build/Bts.DataStreamWebApp.Client/${mode}/dist`
-    },
-    server: {
-      proxy: {
-        '^/api': {
-          target,
-          secure: false
-        },
-        '^/ws': {
-          target,
-          secure: false
-        }
-      },
-      port: 5173,
-      https: {
-        key: fs.readFileSync(keyFilePath),
-        cert: fs.readFileSync(certFilePath),
-      }
-    }
-  };
-});
@@ -1,40 +0,0 @@
-<Project Sdk="Microsoft.NET.Sdk.Web">
-
-  <PropertyGroup>
-    <TargetFramework>net7.0</TargetFramework>
-    <Nullable>enable</Nullable>
-    <ImplicitUsings>enable</ImplicitUsings>
-    <InvariantGlobalization>true</InvariantGlobalization>
-    <SpaRoot>..\Bts.DataStreamWebApp.Client</SpaRoot>
-    <SpaProxyLaunchCommand>npm run dev</SpaProxyLaunchCommand>
-    <SpaProxyServerUrl>https://localhost:5173</SpaProxyServerUrl>
-  </PropertyGroup>
-
-  <ItemGroup>
-    <PackageReference Include="CommandLineParser" Version="2.9.1" />
-    <PackageReference Include="Microsoft.AspNetCore.SpaProxy" Version="7.0.20" />
-    <PackageReference Include="Microsoft.AspNetCore.SpaServices.Extensions" Version="7.0.20" />
-    <PackageReference Include="Newtonsoft.Json" Version="13.0.3" />
-    <PackageReference Include="Swashbuckle.AspNetCore" Version="6.4.0" />
-  </ItemGroup>
-
-  <ItemGroup>
-    <ProjectReference Include="..\Bts.DataStreamWebApp.Client\Bts.DataStreamWebApp.Client.esproj">
-      <ReferenceOutputAssembly>false</ReferenceOutputAssembly>
-    </ProjectReference>
-    <ProjectReference Include="..\DataStream\DataStream.csproj" />
-    <ProjectReference Include="..\SharedStreams\SharedStreams.csproj" />
-    <ProjectReference Include="..\..\client\spark_api\spark_api.csproj" />
-    <ProjectReference Include="..\..\client\spark_core\spark_core.csproj" />
-    <ProjectReference Include="..\..\client\spark_websocket\spark_websocket.csproj" />
-  </ItemGroup>
-
-  <ItemGroup>
-    <ConfigurationProfile Include="appsettings.Development.json" />
-    <ConfigurationProfile Include="appsettings.Production.json" />
-    <ConfigurationProfile Include="appsettings.ProductionAws.json" />
-    <ConfigurationProfile Include="appsettings.Staging.json" />
-    <ConfigurationProfile Include="appsettings.json" />
-  </ItemGroup>
-
-</Project>
@@ -1,5 +0,0 @@
-namespace Bts.DataStreamWebApp.Server;
-
-public class CannedQueriesConfig {
-   public List<CannedQuery>? Items { get; set; }
-}
@@ -1,6 +0,0 @@
-namespace Bts.DataStreamWebApp.Server;
-
-public class CannedQuery {
-   public string? Name { get; set; }
-   public List<string>? Query { get; set; }
-}
@@ -1,18 +0,0 @@
-namespace Bts.DataStreamWebApp.Server;
-
-using Microsoft.AspNetCore.Mvc;
-
-[ApiController]
-[Route("api/[controller]")]
-public class CannedQueriesController : ControllerBase {
-   readonly IConfiguration config;
-
-   public CannedQueriesController(IConfiguration config) =>
-      this.config = config;
-
-   [HttpGet(Name = "GetCannedQueries")]
-   public IEnumerable<CannedQuery>? Get() {
-      var cannedQueries = config.GetSection("CannedQueries").Get<List<CannedQuery>>()?.ToArray() ?? Array.Empty<CannedQuery>();
-      return cannedQueries;
-   }
-}
@@ -1,22 +0,0 @@
-// using Bts.DataStreamWebApp.Server;
-
-// using Microsoft.AspNetCore.Mvc;
-
-// [ApiController]
-// [Route("api/[controller]")]
-// public class ConfigController : ControllerBase {
-//    private readonly IConfiguration _configuration;
-
-//    public ConfigController(IConfiguration configuration) {
-//       _configuration = configuration;
-//    }
-
-//    [HttpGet("websocket")]
-//    public ActionResult<DataStreamWebSocketConfig> GetWebSocketConfig() {
-//       var endpoint = $"{_configuration["Kestrel:Endpoints:Http:Url"]}/ws";
-//       var webSocketConfig = new DataStreamWebSocketConfig {
-//          Endpoint = endpoint // or Https based on your setup
-//       };
-//       return Ok(webSocketConfig);
-//    }
-// }
@@ -1,30 +0,0 @@
-namespace Bts.DataStreamWebApp.Server.Controllers;
-
-using Microsoft.AspNetCore.Mvc;
-
-[Route("/ws")]
-public class DataStreamSubscriptionController : ControllerBase {
-   readonly ILogger logger;
-
-   public DataStreamSubscriptionController(ILogger<DataStreamSubscriptionController> logger) {
-      this.logger = logger;
-      logger.LogInformation("Creating {ControllerType}", GetType().Name);
-   }
-
-   [Route("/ws")]
-   public async Task Get(CancellationToken cancellationToken) {
-      if (!HttpContext.WebSockets.IsWebSocketRequest) {
-         logger.LogError("Expected websocket request on path '{}'", HttpContext.Request.Path);
-         return;
-      }
-      logger.LogInformation("Establishing websocket connection...");
-      try {
-         using var webSocket = await HttpContext.WebSockets.AcceptWebSocketAsync();
-         logger.LogInformation("Websocket connection established.");
-         using var server = new DataStreamWebSocketServer(webSocket, logger);
-         await server.Run(cancellationToken);
-      } catch (Exception ex) {
-         logger.LogError("DataStreamSubscriptionController.Run threw exception: {Exception}", ex);
-      }
-   }
-}
@@ -1,133 +0,0 @@
-namespace Bts.DataStreamWebApp.Server.Controllers;
-
-using BCG;
-
-using bts.websocket_core.json_rpc;
-
-using Bts.DataStreamCore;
-using Bts.DataStreamWebApp.Server.Messages;
-
-using System;
-using System.Net.WebSockets;
-using System.Text;
-using System.Text.Json;
-using System.Text.Json.Serialization;
-
-class DataStreamSubscription : IDisposable, IHostedService {
-   public readonly ViewStream ViewStream;
-   public DataPacket? LastFired;
-
-   readonly ILogger logger;
-   readonly WebSocket webSocket;
-
-   public DataStreamSubscription(string id, JsonRpcId requestId, ViewStream viewStream, WebSocket webSocket, ILogger logger) {
-      ViewStream = viewStream;
-      RequestId = requestId;
-      Id = id;
-      this.webSocket = webSocket;
-      this.logger = logger;
-   }
-
-   public string Id { get; }
-   public JsonRpcId RequestId { get; }
-   public SubscriptionStatus Status { get; private set; }
-
-   public void Start() {
-      if (Status == SubscriptionStatus.Subscribed) {
-         logger.LogWarning("Ignoring redundant call to 'Start'. Id = {Id} Status = {Status}", Id, Status);
-         return;
-      }
-      logger.LogInformation("Subscribing to ViewStream. SubscriptionId = {SubscriptionId}", Id);
-      Status = SubscriptionStatus.Subscribed;
-      ViewStream.onChange += onChange;
-   }
-
-   public Task StartAsync(CancellationToken cancellationToken) {
-      cancellationToken.ThrowIfCancellationRequested();
-      Start();
-      return Task.CompletedTask;
-   }
-
-   public Task Clear() {
-      if (LastFired is null) {
-         return Task.CompletedTask;
-      }
-      logger.LogInformation("Clearing subscription (firing DataPacket to delete all rows). Id = {Id} Status = {Status}", Id, Status);
-      return sendDataPacket("clear", new(new BlueTable(LastFired.schema), new BlueTable(LastFired.schema), LastFired.newRows));
-   }
-
-   public void Stop() {
-      if (Status == SubscriptionStatus.Unsubscribed) {
-         logger.LogWarning("Ignoring redundant call to 'Stop'. Id = {Id} Status = {Status}", Id, Status);
-         return;
-      }
-      logger.LogInformation("Unsubscribing from ViewStream. SubscriptionId = {SubscriptionId}", Id);
-      Status = SubscriptionStatus.Unsubscribed;
-      ViewStream.onChange -= onChange;
-   }
-
-   public Task StopAsync(CancellationToken cancellationToken) {
-      cancellationToken.ThrowIfCancellationRequested();
-      Stop();
-      return Task.CompletedTask;
-   }
-
-   public void Dispose() =>
-      Stop();
-
-   async void onChange(object newValue) {
-      if (newValue is not DataPacket dp) {
-         logger.LogError("Expected DataPacket in stream callback (actually received {}).", newValue?.GetType());
-         return;
-      }
-      if (webSocket.State != WebSocketState.Open) {
-         logger.LogError(
-            "Unsubscribing from stream early because WebSocket state = {WebSocketState}", webSocket.State);
-         Dispose();
-         return;
-      }
-      await sendDataPacket("subscribe", dp);
-   }
-
-   async Task sendDataPacket(string method, DataPacket dp) {
-      var serializerOpts = new JsonSerializerOptions {
-         NumberHandling = JsonNumberHandling.AllowNamedFloatingPointLiterals
-      };
-      serializerOpts.Converters.Add(new DataPacketJsonConverter() {
-         IncludeCurrent = true,
-         UseObjectRows = true,
-      });
-      var res = JsonRpcRequest.CreateNotification(method, new DataStreamSubscriptionUpdate() {
-         SubscriptionId = Id,
-         Table = dp
-      });
-      var str = serialize(res, serializerOpts);
-      if (str is null) {
-         logger.LogError("Unsubscribing from stream early because we failed to serialize an outgoing DataPacket");
-         Dispose();
-         return;
-      }
-      logger.LogInformation(
-         "Sending DataPacket. NumRows = {NumRows} NumCols = {NumCols} MessageLength = {MessageLength}",
-         dp.newRows.Count,
-         dp.schema.columnNames.Length,
-         str.Length);
-      var bytes = Encoding.UTF8.GetBytes(str);
-      try {
-         await webSocket.SendAsync(bytes, WebSocketMessageType.Text, true, CancellationToken.None);
-         LastFired = dp;
-      } catch (WebSocketException ex) {
-         logger.LogError($"sendDataPacket failed on webSocket.SendAsync: {ex}");
-      }
-   }
-
-   string? serialize<T>(T message, JsonSerializerOptions? opts) {
-      opts ??= JsonSerializerOptions.Default;
-      try {
-         return JsonSerializer.Serialize(message, opts);
-      } catch (Exception ex) {
-         logger.LogError("Failed to serialize message of type {MessageType}: {Exception}", typeof(T), ex);
-      }
-      return null;
-   }
-}
@@ -1,13 +0,0 @@
-namespace Bts.DataStreamWebApp.Server;
-
-using BCG;
-
-using System.Text.Json.Serialization;
-
-public class DataStreamSubscriptionUpdate {
-   [JsonPropertyName("subscription_id")]
-   public string? SubscriptionId { get; set; }
-
-   [JsonPropertyName("table")]
-   public DataPacket? Table { get; set; }
-}
@@ -1,89 +0,0 @@
-namespace Bts.DataStreamWebApp.Server;
-
-using Bts.PubSub;
-
-using CommandLine;
-
-using System.Text.Json;
-using System.Text.Json.Serialization;
-
-class DataStreamWebAppServerConfig {
-   /// <summary>
-   /// The login user for sparkd
-   /// </summary>
-   [JsonPropertyName("user")]
-   [Option("user", HelpText = "The sparkd login user")]
-   public string? User { get; set; }
-
-   /// <summary>
-   /// The login password for sparkd
-   /// </summary>
-   [JsonPropertyName("password")]
-   [Option("pass", HelpText = "The sparkd login password")]
-   public string? Password { get; set; }
-
-   /// <summary>
-   /// The config server url for sparkd
-   /// </summary>
-   [JsonPropertyName("sparkd")]
-   [Option("sparkd", HelpText = "The sparkd config server")]
-   public string? Sparkd { get; set; }
-
-   [JsonPropertyName("pubsub")]
-   public PubSubConfig? PubSub { get; set; }
-
-   [Option("content-root-path")]
-   [JsonPropertyName("content_root_path")]
-   public string? ContentRootPath { get; set; }
-
-   [Option("web-root-path")]
-   [JsonPropertyName("web_root_path")]
-   public string? WebRootPath { get; set; }
-
-   [Option("environment-name")]
-   [JsonPropertyName("environment_name")]
-   public string? EnvironmentName { get; set; }
-
-   public DirectoryBrowserOptions? DirectoryBrowser { get; set; }
-
-   [Option(
-      "config",
-      MetaValue = "PATH",
-      HelpText = "Path to json file representing these options (DataStreamPublisherConfig)")]
-   [JsonIgnore]
-   public string? ConfigFile { get; set; }
-
-   public static DataStreamWebAppServerConfig Default() =>
-      new() {
-         User = "bts",
-         PubSub = PubSubConfig.Default(),
-      };
-
-   public void MergeWith(DataStreamWebAppServerConfig other) {
-      User ??= other.User;
-      Password ??= other.Password;
-      PubSub ??= other.PubSub;
-      Sparkd ??= other.Sparkd;
-      ContentRootPath ??= other.ContentRootPath;
-      WebRootPath ??= other.WebRootPath;
-      ConfigFile ??= other.ConfigFile;
-   }
-
-   /// <summary>
-   /// Merges options with those defined in `ConfigFile` (does not overwrite
-   /// non-null existing values).
-   /// </summary>
-   /// <exception cref="InvalidOperationException">
-   /// JsonSerializer.Deserialize returns null
-   /// </exception>
-   public void MergeWithConfigFile() {
-      if (ConfigFile is null) {
-         return;
-      }
-      var cfgString = File.ReadAllText(ConfigFile);
-      var parsedCfg = JsonSerializer.Deserialize<DataStreamWebAppServerConfig>(cfgString) ??
-         throw new InvalidOperationException($"Failed to parse json from {ConfigFile}");
-      MergeWith(parsedCfg);
-      ConfigFile = null;
-   }
-}
@@ -1,5 +0,0 @@
-namespace Bts.DataStreamWebApp.Server;
-
-public class DataStreamWebSocketConfig {
-   public string? Endpoint { get; set; }
-}
@@ -1,148 +0,0 @@
-namespace Bts.DataStreamWebApp.Server.Controllers;
-
-using bts.websocket_core.json_rpc;
-
-using Bts.DataStreamWebApp.Server.Messages;
-
-using System.Net.WebSockets;
-using System.Text;
-using System.Text.Json;
-using System.Text.Json.Nodes;
-
-public class DataStreamWebSocketServer : IDisposable, IAsyncDisposable {
-   readonly ILogger logger;
-   readonly WebSocket webSocket;
-   readonly Dictionary<string, DataStreamSubscription> subscriptions = new();
-
-   public DataStreamWebSocketServer(WebSocket webSocket, ILogger logger) {
-      this.webSocket = webSocket;
-      this.logger = logger;
-   }
-
-   public async Task<int> Run(CancellationToken cancellationToken) {
-      logger.LogInformation("DataStreamWebSocketServer running");
-      await foreach (var req in JsonRpcRequestAsyncEnumerable.Create(webSocket, logger, cancellationToken)) {
-         await (req.Method switch {
-            "subscribe" => onSubscribe(req, cancellationToken),
-            "unsubscribe" => onUnsubscribe(req.Deserialize<UnsubscribeRequest>(), cancellationToken),
-            _ => Task.CompletedTask,
-         });
-      }
-      logger.LogInformation("DataStreamWebSocketServer stopped");
-      return 0;
-   }
-
-   /// <summary>
-   /// Disposes internal subscriptions. To additionally notify the client of the subscription status
-   /// change(s), use <see cref="DisposeAsync"/>.
-   /// </summary>
-   public void Dispose() {
-      GC.SuppressFinalize(this);
-      disposeSubscriptions();
-   }
-
-   /// <summary>
-   /// Disposes internal subscriptions (like <see cref="Dispose"/>), then sends 'subscription status change'
-   /// notifications to the client (unlike <see cref="Dispose"/>).
-   /// </summary>
-   public async ValueTask DisposeAsync() {
-      GC.SuppressFinalize(this);
-      var finalSubs = subscriptions.Values.Select(s => (s.RequestId, s.Id)).ToList();
-      disposeSubscriptions();
-      // Send a 'subscription status change' msg to the client for each one.
-      foreach (var (requestId, id) in finalSubs) {
-         await sendSubscriptionStatus(requestId, id, "unsubscribe", SubscriptionStatus.Unsubscribed, CancellationToken.None);
-      }
-   }
-
-   async Task onSubscribe(JsonRpcRequest<JsonNode> req, CancellationToken cancellationToken) {
-      ArgumentNullException.ThrowIfNull(req.Id);
-      ArgumentException.ThrowIfNullOrEmpty(req.Method);
-      if (!SubscribeRequest.Parse(req, out var subReq, out var parseErr, out var errCode)) {
-         logger.LogError("Invalid 'subscribe' request: {Reason}", parseErr);
-         var rpcError = JsonRpcResponse.CreateError<SubscriptionError>(
-            req.Id, new() { Code = (long)errCode, Message = "Failed to parse subscription request", Data = parseErr });
-         await send(rpcError, cancellationToken);
-         return;
-      }
-      var subId = subReq.SubscriptionId ?? throw new InvalidOperationException("SubscriptionId is null (parse false)");
-      disposeSubscription(subId);
-      if (!subReq.ParseQuery(out var vs, out parseErr, out errCode)) {
-         logger.LogError("Invalid 'subscribe' request: {Reason}", parseErr);
-         var rpcError = JsonRpcResponse.CreateError<SubscriptionError>(
-            req.Id, new() { Code = (long)errCode, Message = "Failed to parse subscription request", Data = parseErr });
-         await send(rpcError, cancellationToken);
-         return;
-      }
-      if (subscriptions.TryGetValue(subId, out var oldSub)) {
-         logger.LogInformation("Disposing existing subscription with id {SubscriptionId}", subId);
-         oldSub.Dispose();
-      }
-      var sub = new DataStreamSubscription(subId, req.Id.Value, vs, webSocket, logger);
-      subscriptions[subId] = sub;
-      await sendSubscriptionStatus(req.Id.Value, subId, req.Method, SubscriptionStatus.Subscribed, cancellationToken);
-      vs.ErrorChanged += OnStreamError;
-      await sub.StartAsync(cancellationToken);
-
-      async void OnStreamError(Exception exception) {
-         logger.LogError("Error in ViewStream (disposing subscription): {Exception}", exception);
-         subscriptions.Remove(sub.Id);
-         sub.Dispose();
-         var error = new SubscriptionError() {
-            ErrorMessage = "Error in ViewStream (disposing subscription)"
-         };
-         if (exception is not null) {
-            error.ErrorMessage += ". " + exception.Message;
-            error.StackTrace = exception.StackTrace?.Split("\n")?.ToList();
-         }
-         var rpcError = JsonRpcResponse.CreateError<SubscriptionError>(
-            req.Id, new() { Code = (long)ErrorCode.ViewStreamError, Message = "Error in ViewStream (disposing subscription)", Data = error });
-         await send(rpcError, cancellationToken);
-      }
-   }
-
-   async Task onUnsubscribe(JsonRpcRequest<UnsubscribeRequest> req, CancellationToken cancellationToken) {
-      ArgumentNullException.ThrowIfNull(req.Id);
-      ArgumentException.ThrowIfNullOrEmpty(req.Method);
-      var subId = req.Params?.SubscriptionId ?? string.Empty;
-      if (!disposeSubscription(subId)) {
-         logger.LogInformation("Unable to dispose subscription. Subscription with id {Id} not found.", subId);
-      }
-      await sendSubscriptionStatus(req.Id.Value, subId, "unsubscribe", SubscriptionStatus.Unsubscribed, cancellationToken);
-   }
-
-   bool disposeSubscription(string subId) {
-      if (!subscriptions.TryGetValue(subId, out var sub)) {
-         return false;
-      }
-      logger.LogInformation("Disposing subscription with id {Id}", subId);
-      sub.Dispose();
-      subscriptions.Remove(subId);
-      return true;
-   }
-
-   Task sendSubscriptionStatus(JsonRpcId reqId, string subId, string method, SubscriptionStatus status, CancellationToken cancellationToken) {
-      logger.LogInformation("Sending {Method} ack. SubscriptionId = {SubscriptionId}", method, subId);
-      var response = JsonRpcResponse.Create(reqId, new SubscriptionStatusMessage() {
-         SubscriptionId = subId,
-         Status = status,
-      });
-      return send(response, cancellationToken);
-   }
-
-   Task send<T>(T message, CancellationToken cancellationToken) {
-      if (message is null) {
-         return Task.CompletedTask;
-      }
-      var str = JsonSerializer.Serialize(message);
-      var bytes = Encoding.UTF8.GetBytes(str);
-      logger.LogTrace("Sending data. Length = {Length}", bytes.Length);
-      return webSocket.SendAsync(bytes, WebSocketMessageType.Text, true, cancellationToken);
-   }
-
-   void disposeSubscriptions() {
-      logger.LogInformation("Disposing DataStreamWebSocketServer subscriptions. Count: {Count}", subscriptions.Count);
-      subscriptions.Values.DisposeAll();
-      subscriptions.Clear();
-   }
-}
@@ -1,7 +0,0 @@
-FROM bluetradingsystems/linux_dotnet7
-
-RUN dotnet dev-certs https --trust
-COPY --chown=bts:bts "build/Release/bin/net7.0/linux-x64/publish" "bin"
-COPY --chown=bts:bts "build/Bts.DataStreamWebApp.Client/production/dist" "bin/wwwroot"
-COPY --chown=bts:bts "src/DataStreamSuite/Bts.DataStreamWebApp.Server/spark.conf" ".local/share/bts/Bts.DataStreamWebApp.Server/conf/spark.conf"
-WORKDIR "/home/bts/bin"
@@ -1,5 +0,0 @@
-namespace Bts.DataStreamWebApp.Server;
-
-public class CannedQueriesResponse {
-   public List<CannedQuery>? Items { get; set; }
-}
@@ -1,7 +0,0 @@
-namespace Bts.DataStreamWebApp.Server.Messages;
-
-enum ErrorCode {
-   None = 0,
-   InvalidSubscriptionRequest,
-   ViewStreamError,
-}
@@ -1,72 +0,0 @@
-namespace Bts.DataStreamWebApp.Server.Messages;
-
-using BCG;
-
-using bts.websocket_core.json_rpc;
-
-using Bts.DataStreamExpressions;
-
-using System.Diagnostics.CodeAnalysis;
-using System.Text.Json;
-using System.Text.Json.Nodes;
-using System.Text.Json.Serialization;
-
-public class SubscribeRequest {
-   [JsonPropertyName("query")]
-   public string? Query { get; set; }
-
-   [JsonPropertyName("subscription_id")]
-   public string? SubscriptionId { get; set; }
-
-   internal static bool Parse(JsonRpcRequest<JsonNode> req, [NotNullWhen(true)]out SubscribeRequest? subscription, [NotNullWhen(false)]out SubscriptionError? error, out ErrorCode errorCode) {
-      subscription = null;
-      errorCode = ErrorCode.None;
-      var paramsJsonObj = req.Params?.AsObject();
-      error = null;
-      if (paramsJsonObj is null) {
-         errorCode = ErrorCode.InvalidSubscriptionRequest;
-         error = new() { ErrorMessage = "'params' object not found" };
-         return false;
-      }
-      subscription = paramsJsonObj.Deserialize<SubscribeRequest>();
-      if (subscription is null) {
-         errorCode = ErrorCode.InvalidSubscriptionRequest;
-         error = new() { ErrorMessage = "'params' object invalid" };
-         return false;
-      }
-      return true;
-   }
-
-   internal bool ParseQuery([NotNullWhen(true)] out ViewStream? stream, [NotNullWhen(false)] out SubscriptionError? error, out ErrorCode errorCode) {
-      stream = null;
-      errorCode = ErrorCode.None;
-      error = null;
-      if (Query is null) {
-         errorCode = ErrorCode.InvalidSubscriptionRequest;
-         error = new() { ErrorMessage = "'params.query' field not found (expected a string)" };
-         return false;
-      }
-      var exp = DataStreamExpressionsInterpreter.Evaluate(Query);
-      if (exp is null) {
-         errorCode = ErrorCode.InvalidSubscriptionRequest;
-         error = new() { ErrorMessage = "'params.query' expression has invalid syntax" };
-         return false;
-      }
-      Exception? exception = null;
-      try {
-         stream = exp.getView(new Dictionary<string, StatementExpression>(0));
-      } catch (Exception ex) {
-         exception = ex;
-      }
-      if (stream is null) {
-         errorCode = ErrorCode.InvalidSubscriptionRequest;
-         error = new() { ErrorMessage = "Failed to create ViewStream from 'params.query' expression (check for typos in arguments/typenames)" };
-         if (exception is not null) {
-            error.ErrorMessage += ". " + exception.Message;
-            error.StackTrace = exception.StackTrace?.Split("\n")?.ToList();
-         }
-         return false;
-      }
-      return true;
-   }
-}
				`@@ -1 +0,0 @@`
				`VITE_DATASTREAM_WEBSOCKET_URL=wss://localhost:443`
				`@@ -1 +0,0 @@`
				<svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" class="iconify iconify--logos" width="31.88" height="32" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 257"><defs><linearGradient id="IconifyId1813088fe1fbc01fb466" x1="-.828%" x2="57.636%" y1="7.652%" y2="78.411%"><stop offset="0%" stop-color="#41D1FF"></stop><stop offset="100%" stop-color="#BD34FE"></stop></linearGradient><linearGradient id="IconifyId1813088fe1fbc01fb467" x1="43.376%" x2="50.316%" y1="2.242%" y2="89.03%"><stop offset="0%" stop-color="#FFEA83"></stop><stop offset="8.333%" stop-color="#FFDD35"></stop><stop offset="100%" stop-color="#FFA800"></stop></linearGradient></defs><path fill="url(#IconifyId1813088fe1fbc01fb466)" d="M255.153 37.938L134.897 252.976c-2.483 4.44-8.862 4.466-11.382.048L.875 37.958c-2.746-4.814 1.371-10.646 6.827-9.67l120.385 21.517a6.537 6.537 0 0 0 2.322-.004l117.867-21.483c5.438-.991 9.574 4.796 6.877 9.62Z"></path><path fill="url(#IconifyId1813088fe1fbc01fb467)" d="M185.432.063L96.44 17.501a3.268 3.268 0 0 0-2.634 3.014l-5.474 92.456a3.268 3.268 0 0 0 3.997 3.378l24.777-5.718c2.318-.535 4.413 1.507 3.936 3.838l-7.361 36.047c-.495 2.426 1.782 4.5 4.151 3.78l15.304-4.649c2.372-.72 4.652 1.36 4.15 3.788l-11.698 56.621c-.732 3.542 3.979 5.473 5.943 2.437l1.313-2.028l72.516-144.72c1.215-2.423-.88-5.186-3.54-4.672l-25.505 4.922c-2.396.462-4.435-1.77-3.759-4.114l16.646-57.705c.677-2.35-1.37-4.583-3.769-4.113Z"></path></svg>