fix use dep injection in bts.datastreamwebapp.server

fix Add spark service modules designed for DependencyInjection
Fix mismatched types in PubsubHeartbeatStream schema
2024-07-11 11:06:29 -04:00 · 2024-07-11 11:05:23 -04:00 · 2024-07-11 10:59:41 -04:00 · 2024-07-11 10:43:24 -04:00 · 2024-07-11 10:38:30 -04:00 · 2024-07-11 10:37:29 -04:00
3002 changed files with 50793 additions and 105197 deletions
@@ -9,7 +9,7 @@
      ]
    },
    "fable": {
-      "version": "4.24.0",
+      "version": "4.0.6",
      "commands": [
        "fable"
      ]
@@ -25,12 +25,6 @@
      "commands": [
        "fake"
      ]
-    },
-    "dotnet-t4": {
-      "version": "3.0.0",
-      "commands": [
-        "t4"
-      ]
    }
  }
-}
+}
@@ -65,6 +65,7 @@ RUN \
   && apt-get install -y cmake

 # Dotnet
+RUN apt-get update && apt-get install -y dotnet7
 RUN apt-get update && apt-get install -y dotnet8

 # Node
@@ -87,15 +88,6 @@ RUN dpkg -i /emumba/emumba-deribit-connector-3.3.1-Linux.LICENSE_DISABLED.deb \
 # Pcap
 RUN apt-get update && apt-get install -y libpcap0.8-dev

-# Databento (deps)
-RUN apt-get update && apt-get -y install libzstd-dev
-
-# Docker in Docker
-RUN apt-get update && apt-get install -y \
-    docker.io \
-    docker-compose \
-    docker-compose-v2
-
 # Set up default executables (update-alternatives)
 RUN \
   update-alternatives --install /usr/bin/clang clang /usr/bin/clang-$clang_major 100; \
@@ -1,10 +1,15 @@
 {
  "dockerFile": "Dockerfile",
+  // Creates local dir to mount to HOME in docker 
+  "initializeCommand": "mkdir -p ${localEnv:HOME}/.bts/devcontainer-mounts/${devcontainerId}",
  "hostRequirements": {
-    "memory": "8gb",
-    "storage": "10gb"
+    "memory": "4gb",
+    "storage": "6gb"
  },
+  "postCreateCommand": "/home/bts/spark/docker/sparkd/dev/devcontainer_post_create.sh",
  "containerUser": "bts",
+  "workspaceMount": "source=${localWorkspaceFolder},target=/home/bts/spark,type=bind",
+  "workspaceFolder": "/home/bts/spark",
  "updateRemoteUserUID": true,
  "shutdownAction": "none",
  // Note: If running multiple containers on the same host, each must use a
@@ -12,19 +17,41 @@
  // example, if 8000-8003 is taken, go with 9000-9003, then 10000-10003, etc.
  "runArgs": [
    "--shm-size",
-    "8gb"
+    "2gb",
+    "-p",
+    "5000:5000",
+    "-p",
+    "5001:5001",
+    "-p",
+    "5173:5173",
+    "-p",
+    "5174:5174",
+    "-p",
+    "8000:8000",
+    "-p",
+    "8001:8001",
+    "-p",
+    "8002:8002",
+    "-p",
+    "8003:8003",
+    "-p",
+    "12345:12345",
  ],
  "mounts": [
-    "source=/var/run/docker.sock,target=/var/run/docker.sock,type=bind"
+    "source=${localEnv:HOME}/.bts/devcontainer-mounts/${devcontainerId},target=/home/bts,type=bind"
  ],
  "customizations": {
    "vscode": {
+      "settings": {
+        "clangd.arguments": [
+          "--compile-commands-dir=${workspaceFolder}/build/clang/debug"
+        ]
+      },
      "extensions": [
        "eamodio.gitlens",
        "llvm-vs-code-extensions.vscode-clangd",
        "ms-vscode.cmake-tools",
-        "ms-vscode.cpptools",
-        "xaver.clang-format"
+        "ms-vscode.cpptools"
      ]
    }
  }
@@ -1,49 +0,0 @@
-# This file defines the global default configuration for docker deployments.
-# It is tracked in source control. For temporary modifications, please create
-# a separate file called ".env.local" (it's in .gitignore, so won't be tracked).
-# If a variable is defined in both files, the value in .env.local will be used.
-#
-# About the deployment procedure:
-#   We use Docker Compose to build images and launch containers for various services
-#   that make up our software suite:
-#     - sparkd
-#     - TODO: pubsub
-#     - TODO: md-publisher
-#     - TODO: webapp (DSQA on the web)
-#     - TODO: api-server (websocket based)
-#   From inside a devcontainer, you can launch all services in separate containers
-#   on your machine by running:
-#
-#     tools/deploy.sh
-#
-#   This script builds release versions for each service, packaged (generally) in
-#   self-contained directories. Next, it runs a few "docker compose" commands to
-#   build docker images loaded with the self-contained binaries from the previous
-#   step, and start containers with those images to run each service. To view the
-#   commands that deploy.sh runs internally (without actually running them), run:
-#
-#     tools/deploy.sh --dry-run
-#
-#   For additional information/help, run:
-#
-#     tools/deploy.sh --help
-#
-
-### Sparkd ###
-# The internal image for sparkd to use. For more info, view:
-#   src/server/sparkd/Dockerfile
-SPARKD__IMAGE=sparkd
-# The config dir for this service to use
-SPARKD__CONFIG_DIR=src/app_sparkd/configurations/dev
-# Mount a specific volume or dir to "~/sparkdata/data" inside the docker.
-# Defaults to a volume named "sparkd_<config_port>" (e.g., sparkd_8000).
-# SPARKD__DATA_DIR=/home/bts/sparkdata/data
-SPARKD__SOURCE_DIR=install/sparkd-release-gcc
-SPARKD__RUNTIME_CONF_DIR=/home/ubuntu/8000/sparkdata/conf/
-SPARKD__CONFIG_PORT=8000
-SPARKD__QUICKBUS_PORT=8001
-SPARKD__SIM_FIX_PORT=8002
-SPARKD__SHM_SIZE=4gb
-SPARKD__PYTHON_REQUIREMENTS=requirements.txt
-
-# TODO: add config for other services (pubsub, md-publisher, webapp, api-server, etc.)
@@ -11,7 +11,6 @@ cmake-build-debug/
 **.vscode/*
 !.vscode/launch.json
 !.vscode/tasks.json
-!.vscode/*.env
 .vs
 obj/
 *.csproj.user
@@ -60,5 +59,3 @@ trace_data/
 .devcontainer/
 *.Generated.cs
 *.lst
-.env.local
-.secrets
@@ -1,65 +1,23 @@
 {
  "version": "0.2.0",
  "configurations": [
-    {
-      "name": "Python Debugger: Current File with Arguments",
-      "type": "debugpy",
-      "request": "launch",
-      "program": "${file}",
-      "console": "integratedTerminal",
-      "args": "${command:pickArgs}"
-    },
    {
      "name": ".NET Core Attach",
      "type": "coreclr",
      "request": "attach"
    },
-    {
-      "name": "Api Server",
-      "type": "coreclr",
-      "request": "launch",
-      "program": "",
-      "windows": {
-        "preLaunchTask": "Build Api Server (Debug)",
-        "program": "spark_api_server.exe",
-        "args": [
-          "run",
-          "--sparkd",
-          "http://127.0.0.1:8000/config"
-        ],
-        "cwd": "build/Debug/bin/net7.0",
-      },
-      "stopAtEntry": false,
-      "console": "integratedTerminal"
-    },
-    {
-      "name": "Api Server (Skip Build)",
-      "type": "coreclr",
-      "request": "launch",
-      "program": "",
-      "windows": {
-        "program": "spark_api_server.exe",
-        "args": [
-          "run",
-          "--sparkd",
-          "http://127.0.0.1:8000/config"
-        ],
-        "cwd": "build/Debug/bin/net7.0",
-      },
-      "stopAtEntry": false,
-      "console": "integratedTerminal"
-    },
    {
      "name": "Bts.DataStreamWebApp.Server",
      "type": "coreclr",
      "request": "launch",
      "preLaunchTask": "Build DataStream WebApp (Debug)",
-      "program": "${workspaceFolder}/build/Debug/bin/net8.0/Bts.DataStreamWebApp.Server.dll",
-      "cwd": "${workspaceFolder}/build/Debug/bin/net8.0",
+      "program": "${workspaceFolder}/build/Debug/bin/net7.0/Bts.DataStreamWebApp.Server.dll",
+      "args": [
+        "--config",
+        "${workspaceFolder}/src/DataStreamSuite/Bts.DataStreamWebApp.Server/Configurations/DevServer.json",
+      ],
      "env": {
-        "ASPNETCORE_HTTP_PORT": "8080",
-        "ASPNETCORE_HTTPS_PORT": "8443",
-        "ASPNETCORE_ENVIRONMENT": "${input:dataStreamWebAppServerEnvironment}",
+        "ASPNETCORE_ENVIRONMENT": "Development",
        "ASPNETCORE_HOSTINGSTARTUPASSEMBLIES": "Microsoft.AspNetCore.SpaProxy",
      }
    },
@@ -67,12 +25,13 @@
      "name": "Bts.DataStreamWebApp.Server (Skip Build)",
      "type": "coreclr",
      "request": "launch",
-      "program": "${workspaceFolder}/build/Debug/bin/net8.0/Bts.DataStreamWebApp.Server.dll",
-      "cwd": "${workspaceFolder}/build/Debug/bin/net8.0",
+      "program": "${workspaceFolder}/build/Debug/bin/net7.0/Bts.DataStreamWebApp.Server.dll",
+      "args": [
+        "--config",
+        "${workspaceFolder}/src/DataStreamSuite/Bts.DataStreamWebApp.Server/Configurations/DevServer.json",
+      ],
      "env": {
-        "ASPNETCORE_HTTP_PORT": "8080",
-        "ASPNETCORE_HTTPS_PORT": "8443",
-        "ASPNETCORE_ENVIRONMENT": "${input:dataStreamWebAppServerEnvironment}",
+        "ASPNETCORE_ENVIRONMENT": "Development",
        "ASPNETCORE_HOSTINGSTARTUPASSEMBLIES": "Microsoft.AspNetCore.SpaProxy",
      }
    },
@@ -88,10 +47,11 @@
      "windows": {
        "program": "DataStreamPublisher.exe",
      },
-      "cwd": "build/Debug/bin/net8.0",
-      "env": {
-        "DOTNET_ENVIRONMENT": "${input:dataStreamPublisherEnvironment}",
-      },
+      "args": [
+        "--config",
+        "${workspaceFolder}/src/DataStreamSuite/DataStreamPublisher/Configurations/${input:dataStreamPublisherConfig}.json"
+      ],
+      "cwd": "${workspaceFolder}/build/Debug/bin/net7.0",
      "stopAtEntry": false,
      "console": "integratedTerminal"
    },
@@ -106,57 +66,14 @@
      "windows": {
        "program": "DataStreamPublisher.exe",
      },
-      "cwd": "${workspaceFolder}/build/Debug/bin/net8.0",
-      "env": {
-        "DOTNET_ENVIRONMENT": "${input:dataStreamPublisherEnvironment}",
-      },
+      "args": [
+        "--config",
+        "${workspaceFolder}/src/DataStreamSuite/DataStreamPublisher/Configurations/${input:dataStreamPublisherConfig}.json"
+      ],
+      "cwd": "${workspaceFolder}/build/Debug/bin/net7.0",
      "stopAtEntry": false,
      "console": "integratedTerminal"
    },
-    {
-      "name": "(gdb) Launch",
-      "type": "cppdbg",
-      "request": "launch",
-      "program": "/home/bts/spark/build/sparkd-debug-clang/server/sparkd",
-      "args": [
-        "run"
-      ],
-      "stopAtEntry": false,
-      "cwd": "/home/bts/spark/build/sparkd-debug-clang/server",
-      "environment": [],
-      "externalConsole": false,
-      "MIMode": "gdb",
-    },
-    {
-      "name": "spreader tests2",
-      "type": "cppdbg",
-      "request": "launch",
-      "program": "/home/bts/spark/build/sparkd-debug-clang/server/sparkd",
-      "args": [
-        "test",
-        "[spreader2]"
-      ],
-      "stopAtEntry": false,
-      "cwd": "/home/bts/spark/build/sparkd-debug-clang/server",
-      "environment": [],
-      "externalConsole": false,
-      "MIMode": "gdb",
-    },
-    {
-      "name": "spreader tests",
-      "type": "cppdbg",
-      "request": "launch",
-      "program": "/home/bts/spark/build/sparkd-debug-clang/server/sparkd",
-      "args": [
-        "test",
-        "[spreader]"
-      ],
-      "stopAtEntry": false,
-      "cwd": "/home/bts/spark/build/sparkd-debug-clang/server",
-      "environment": [],
-      "externalConsole": false,
-      "MIMode": "gdb",
-    },
    {
      "name": "DataStreamQueryAnalyzer",
      "type": "coreclr",
@@ -169,7 +86,7 @@
          "--config",
          "${workspaceFolder}/src/DataStreamSuite/DataStreamQueryAnalyzer/Configurations/DataStreamQueryAnalyzer.json"
        ],
-        "cwd": "build/Debug/bin/net8.0-windows",
+        "cwd": "build/Debug/bin/net7.0-windows",
        "env": {
          "PYTHON_EXE": "${workspaceFolder}/.venv/Scripts/python.exe"
        },
@@ -189,10 +106,11 @@
      "windows": {
        "program": "PubSubServer.exe",
      },
-      "cwd": "build/Debug/bin/net8.0",
-      "env": {
-        "DOTNET_ENVIRONMENT": "${input:pubSubServerEnvironment}",
-      },
+      "args": [
+        "--config",
+        "${workspaceFolder}/src/DataStreamSuite/PubSubServer/Configurations/${input:pubSubServerConfig}.json"
+      ],
+      "cwd": "build/Debug/bin/net7.0",
      "stopAtEntry": false,
      "console": "integratedTerminal"
    },
@@ -207,10 +125,11 @@
      "windows": {
        "program": "PubSubServer.exe",
      },
+      "args": [
+        "--config",
+        "${workspaceFolder}/src/DataStreamSuite/PubSubServer/Configurations/${input:pubSubServerConfig}.json"
+      ],
      "cwd": "build/Debug/bin/net7.0",
-      "env": {
-        "DOTNET_ENVIRONMENT": "${input:pubSubServerEnvironment}",
-      },
      "stopAtEntry": false,
      "console": "integratedTerminal"
    },
@@ -222,19 +141,7 @@
      "windows": {
        "preLaunchTask": "Build Spark (Debug)",
        "program": "Spark.exe",
-        "cwd": "build/Debug/bin/net8.0-windows",
-      },
-      "stopAtEntry": false,
-      "console": "integratedTerminal"
-    },
-    {
-      "name": "Spark (Skip Build)",
-      "type": "coreclr",
-      "request": "launch",
-      "program": "",
-      "windows": {
-        "program": "Spark.exe",
-        "cwd": "build/Debug/bin/net8.0-windows",
+        "cwd": "build/Debug/bin/net7.0-windows",
      },
      "stopAtEntry": false,
      "console": "integratedTerminal"
@@ -251,138 +158,50 @@
          "--sparkd",
          "${input:sparkConfigServer}"
        ],
-        "cwd": "build/Debug/bin/net8.0-windows",
-      },
-      "stopAtEntry": false,
-      "console": "integratedTerminal"
-    },
-    {
-      "name": "Spark Test",
-      "type": "coreclr",
-      "request": "launch",
-      "program": "",
-      "windows": {
-        "preLaunchTask": "Build Spark (Debug)",
-        "program": "Spark.Console.exe",
-        "cwd": "build/Debug/bin/net8.0",
-        "args": [
-          "test",
-          "${input:sparkTest}"
-        ]
-      },
-      "stopAtEntry": false,
-      "console": "integratedTerminal"
-    },
-    {
-      "name": "Spark Test (Skip Build)",
-      "type": "coreclr",
-      "request": "launch",
-      "program": "",
-      "windows": {
-        "program": "Spark.Console.exe",
-        "cwd": "build/Debug/bin/net8.0",
-        "args": [
-          "test",
-          "${input:sparkTest}"
-        ]
+        "cwd": "build/Debug/bin/net7.0-windows",
      },
      "stopAtEntry": false,
      "console": "integratedTerminal"
    },
    {
      "name": "Sparkd",
-      "type": "cppdbg",
+      "type": "coreclr",
      "request": "launch",
+      "preLaunchTask": "Build Sparkd (Debug | Clang)",
      "program": "",
      "linux": {
-        "program": "${workspaceFolder}/build/sparkd-debug-clang/src/app_sparkd/sparkd",
+        "program": "sparkd",
      },
      "args": [
        "run"
      ],
-      "cwd": "${workspaceFolder}/build/sparkd-debug-clang/src/app_sparkd",
+      "cwd": "build/sparkd-debug-clang/server",
      "stopAtEntry": false,
-      "externalConsole": false,
-      "MIMode": "gdb",
-      "setupCommands": [
-        {
-          "description": "Enable pretty-printing for gdb",
-          "text": "-enable-pretty-printing",
-          "ignoreFailures": true
-        },
-        {
-          "description": "Set Disassembly Flavor to Intel",
-          "text": "-gdb-set disassembly-flavor intel",
-          "ignoreFailures": true
-        }
-      ]
-    },
-    {
-      "name": "Sparkd (Skip Build)",
-      "type": "cppdbg",
-      "request": "launch",
-      "program": "",
-      "linux": {
-        "program": "${workspaceFolder}/build/sparkd-debug-clang/src/app_sparkd/sparkd",
-      },
-      "args": [
-        "run"
-      ],
-      "cwd": "${workspaceFolder}/build/sparkd-debug-clang/src/app_sparkd",
-      "stopAtEntry": false,
-      "externalConsole": false,
-      "MIMode": "gdb",
-      "setupCommands": [
-        {
-          "description": "Enable pretty-printing for gdb",
-          "text": "-enable-pretty-printing",
-          "ignoreFailures": true
-        },
-        {
-          "description": "Set Disassembly Flavor to Intel",
-          "text": "-gdb-set disassembly-flavor intel",
-          "ignoreFailures": true
-        }
-      ]
-    },
+      "console": "integratedTerminal"
+    }
  ],
  "inputs": [
    {
-      "id": "dataStreamPublisherEnvironment",
+      "id": "dataStreamPublisherConfig",
      "type": "pickString",
      "description": "Publisher Configuration",
      "options": [
-        "Development",
-        "MarketData",
-        "Production",
-        "ProductionAws",
-        "Staging",
+        "MarketDataPublisher",
+        "SimpleEchoClient",
+        "SimpleTestDataPublisher",
+        "TestDataPublisher",
      ],
      "default": "MarketDataPublisher",
    },
    {
-      "id": "pubSubServerEnvironment",
+      "id": "pubSubServerConfig",
      "type": "pickString",
      "description": "PubSub Configuration",
      "options": [
-        "Development",
-        "EchoServer",
-        "Production",
-        "ProductionAws",
-        "Staging",
+        "PubSubServer",
+        "SimpleEchoClient",
      ],
-      "default": "Development",
-    },
-    {
-      "id": "dataStreamWebAppServerEnvironment",
-      "type": "pickString",
-      "description": "DataStream WebApp Environment",
-      "options": [
-        "Development",
-        "Production",
-        "ProductionAws",
-        "Staging",
-      ]
+      "default": "PubSubServer",
    },
    {
      "id": "sparkConfigServer",
@@ -390,25 +209,5 @@
      "description": "Config Server URL",
      "default": "http://localhost:8000/config",
    },
-    {
-      "id": "sparkTest",
-      "type": "promptString",
-      "description": "Test to run",
-    },
-    {
-      "id": "sparkdTest",
-      "type": "promptString",
-      "description": "Test to run",
-    },
-    {
-      "id": "isSparkApiEnabled",
-      "type": "pickString",
-      "description": "Enable Spark Api (connect to sparkd)",
-      "options": [
-        "True",
-        "False",
-      ],
-      "default": "True",
-    }
  ]
 }
@@ -1 +0,0 @@
-PYTHONPATH=/workspaces/spark/build/sparkd-debug-clang/src/lib_python_interop;/workspaces/spark/src/lib_python_interop/python_interop_plugins
@@ -212,26 +212,6 @@
        "color": "terminal.ansiCyan"
      },
    },
-    {
-      "label": "Build DataStream WebApp (Release)",
-      "group": "build",
-      "type": "process",
-      "command": "",
-      "linux": {
-        "command": "tools/build_data_stream_web_app.sh",
-      },
-      "windows": {
-        "command": "tools/build_data_stream_web_app.cmd",
-      },
-      "args": [
-        "--configuration",
-        "Release",
-      ],
-      "icon": {
-        "id": "terminal",
-        "color": "terminal.ansiCyan"
-      },
-    },
    {
      "label": "Build Spark",
      "group": "build",
@@ -490,7 +470,7 @@
          "${workspaceFolder}/src/DataStreamSuite/DataStreamQueryAnalyzer/Configurations/DataStreamQueryAnalyzer.json",
        ],
        "options": {
-          "cwd": "${workspaceFolder}/build/${input:buildConfiguration}/bin/net8.0-windows"
+          "cwd": "${workspaceFolder}/build/${input:buildConfiguration}/bin/net7.0-windows"
        },
      },
      "icon": {
@@ -510,7 +490,7 @@
          "${workspaceFolder}/src/DataStreamSuite/DataStreamQueryAnalyzer/Configurations/DataStreamQueryAnalyzer.json",
        ],
        "options": {
-          "cwd": "${workspaceFolder}/build/Debug/bin/net8.0-windows"
+          "cwd": "${workspaceFolder}/build/Debug/bin/net7.0-windows"
        },
      },
      "icon": {
@@ -530,7 +510,7 @@
          "${workspaceFolder}/src/DataStreamSuite/DataStreamQueryAnalyzer/Configurations/DataStreamQueryAnalyzer.json",
        ],
        "options": {
-          "cwd": "${workspaceFolder}/build/Release/bin/net8.0-windows"
+          "cwd": "${workspaceFolder}/build/Release/bin/net7.0-windows"
        },
      },
      "icon": {
@@ -554,7 +534,7 @@
        "${workspaceFolder}/src/DataStreamSuite/DataStreamPublisher/Configurations/${input:dataStreamPublisherConfig}.json",
      ],
      "options": {
-        "cwd": "${workspaceFolder}/build/${input:buildConfiguration}/bin/net8.0"
+        "cwd": "${workspaceFolder}/build/${input:buildConfiguration}/bin/net7.0"
      },
      "icon": {
        "id": "run",
@@ -577,7 +557,7 @@
        "${workspaceFolder}/src/DataStreamSuite/DataStreamPublisher/Configurations/${input:dataStreamPublisherConfig}.json",
      ],
      "options": {
-        "cwd": "${workspaceFolder}/build/Debug/bin/net8.0"
+        "cwd": "${workspaceFolder}/build/Debug/bin/net7.0"
      },
      "icon": {
        "id": "run",
@@ -600,90 +580,7 @@
        "${workspaceFolder}/src/DataStreamSuite/DataStreamPublisher/Configurations/${input:dataStreamPublisherConfig}.json",
      ],
      "options": {
-        "cwd": "${workspaceFolder}/build/Release/bin/net8.0"
-      },
-      "icon": {
-        "id": "run",
-        "color": "terminal.ansiGreen",
-      },
-    },
-    {
-      "label": "Run DataStream WebApp (Debug)",
-      "dependsOn": "Build DataStream WebApp (Debug)",
-      "command": "",
-      "linux": {
-        "command": "Bts.DataStreamWebApp.Server",
-      },
-      "windows": {
-        "command": "Bts.DataStreamWebApp.Server.exe",
-      },
-      "args": [
-        "--config",
-        "${workspaceFolder}/src/DataStreamSuite/Bts.DataStreamWebApp.Server/Configurations/DevServer.json",
-      ],
-      "options": {
-        "cwd": "${workspaceFolder}/build/Debug/bin/net7.0",
-        "env": {
-          "ASPNETCORE_HTTP_PORT": "8080",
-          "ASPNETCORE_HTTPS_PORT": "8443",
-          "ASPNETCORE_ENVIRONMENT": "Development",
-          "ASPNETCORE_HOSTINGSTARTUPASSEMBLIES": "Microsoft.AspNetCore.SpaProxy",
-        }
-      },
-      "icon": {
-        "id": "run",
-        "color": "terminal.ansiGreen",
-      },
-    },
-    {
-      "label": "Run DataStream WebApp (Release)",
-      "dependsOn": "Build DataStream WebApp (Release)",
-      "command": "",
-      "linux": {
-        "command": "Bts.DataStreamWebApp.Server",
-      },
-      "windows": {
-        "command": "Bts.DataStreamWebApp.Server.exe",
-      },
-      "args": [
-        "--config",
-        "${workspaceFolder}/src/DataStreamSuite/Bts.DataStreamWebApp.Server/Configurations/DevServer.json",
-      ],
-      "options": {
-        "cwd": "${workspaceFolder}/build/Release/bin/net7.0",
-        "env": {
-          "ASPNETCORE_HTTP_PORT": "8080",
-          "ASPNETCORE_HTTPS_PORT": "8443",
-          "ASPNETCORE_ENVIRONMENT": "Production",
-          "ASPNETCORE_HOSTINGSTARTUPASSEMBLIES": "Microsoft.AspNetCore.SpaProxy",
-        }
-      },
-      "icon": {
-        "id": "run",
-        "color": "terminal.ansiGreen",
-      },
-    },
-    {
-      "label": "Run DataStream WebApp (Release | Skip Build)",
-      "command": "",
-      "linux": {
-        "command": "Bts.DataStreamWebApp.Server",
-      },
-      "windows": {
-        "command": "Bts.DataStreamWebApp.Server.exe",
-      },
-      "args": [
-        "--config",
-        "${workspaceFolder}/src/DataStreamSuite/Bts.DataStreamWebApp.Server/Configurations/DevServer.json",
-      ],
-      "options": {
-        "cwd": "${workspaceFolder}/build/Release/bin/net7.0",
-        "env": {
-          "ASPNETCORE_HTTP_PORT": "8080",
-          "ASPNETCORE_HTTPS_PORT": "8443",
-          "ASPNETCORE_ENVIRONMENT": "Production",
-          "ASPNETCORE_HOSTINGSTARTUPASSEMBLIES": "Microsoft.AspNetCore.SpaProxy",
-        }
+        "cwd": "${workspaceFolder}/build/Release/bin/net7.0"
      },
      "icon": {
        "id": "run",
@@ -706,7 +603,7 @@
        "${workspaceFolder}/src/DataStreamSuite/PubSubServer/Configurations/${input:pubSubServerConfig}.json",
      ],
      "options": {
-        "cwd": "${workspaceFolder}/build/${input:buildConfiguration}/bin/net8.0"
+        "cwd": "${workspaceFolder}/build/${input:buildConfiguration}/bin/net7.0"
      },
      "icon": {
        "id": "run",
@@ -729,7 +626,7 @@
        "${workspaceFolder}/src/DataStreamSuite/PubSubServer/Configurations/${input:pubSubServerConfig}.json",
      ],
      "options": {
-        "cwd": "${workspaceFolder}/build/Debug/bin/net8.0"
+        "cwd": "${workspaceFolder}/build/Debug/bin/net7.0"
      },
      "icon": {
        "id": "run",
@@ -751,7 +648,7 @@
        "${workspaceFolder}/src/DataStreamSuite/PubSubServer/Configurations/${input:pubSubServerConfig}.json",
      ],
      "options": {
-        "cwd": "${workspaceFolder}/build/Debug/bin/net8.0"
+        "cwd": "${workspaceFolder}/build/Debug/bin/net7.0"
      },
      "icon": {
        "id": "run",
@@ -774,7 +671,7 @@
        "${workspaceFolder}/src/DataStreamSuite/PubSubServer/Configurations/${input:pubSubServerConfig}.json",
      ],
      "options": {
-        "cwd": "${workspaceFolder}/build/Release/bin/net8.0"
+        "cwd": "${workspaceFolder}/build/Release/bin/net7.0"
      },
      "icon": {
        "id": "run",
@@ -796,7 +693,7 @@
        "${workspaceFolder}/src/DataStreamSuite/PubSubServer/Configurations/${input:pubSubServerConfig}.json",
      ],
      "options": {
-        "cwd": "${workspaceFolder}/build/Release/bin/net8.0"
+        "cwd": "${workspaceFolder}/build/Release/bin/net7.0"
      },
      "icon": {
        "id": "run",
@@ -811,7 +708,7 @@
      "windows": {
        "command": "Spark.exe",
        "options": {
-          "cwd": "${workspaceFolder}/build/${input:buildConfiguration}/bin/net8.0-windows"
+          "cwd": "${workspaceFolder}/build/${input:buildConfiguration}/bin/net7.0-windows"
        },
      },
      "icon": {
@@ -827,7 +724,7 @@
      "windows": {
        "command": "Spark.exe",
        "options": {
-          "cwd": "${workspaceFolder}/build/Debug/bin/net8.0-windows"
+          "cwd": "${workspaceFolder}/build/Debug/bin/net7.0-windows"
        },
      },
      "icon": {
@@ -843,22 +740,7 @@
      "windows": {
        "command": "Spark.exe",
        "options": {
-          "cwd": "${workspaceFolder}/build/Release/bin/net8.0-windows"
-        },
-      },
-      "icon": {
-        "id": "run",
-        "color": "terminal.ansiGreen",
-      }
-    },
-    {
-      "label": "Run Spark (Skip Build)",
-      "type": "process",
-      "command": "",
-      "windows": {
-        "command": "Spark.exe",
-        "options": {
-          "cwd": "${workspaceFolder}/build/${input:buildConfiguration}/bin/net8.0-windows"
+          "cwd": "${workspaceFolder}/build/Release/bin/net7.0-windows"
        },
      },
      "icon": {
@@ -883,7 +765,7 @@
        "${input:sparkConfigQuery}",
      ],
      "options": {
-        "cwd": "${workspaceFolder}/build/${input:buildConfiguration}/bin/net8.0"
+        "cwd": "${workspaceFolder}/build/${input:buildConfiguration}/bin/net7.0"
      },
      "icon": {
        "id": "run",
@@ -909,24 +791,6 @@
        "color": "terminal.ansiGreen",
      },
    },
-    {
-      "label": "Run Sparkd (Skip Build)",
-      "type": "process",
-      "command": "",
-      "linux": {
-        "command": "sparkd",
-      },
-      "args": [
-        "run"
-      ],
-      "options": {
-        "cwd": "${workspaceFolder}/build/${input:sparkdCmakePreset}/server"
-      },
-      "icon": {
-        "id": "run",
-        "color": "terminal.ansiGreen",
-      },
-    },
    {
      "label": "Run Sparkd Config Query",
      "dependsOn": "Build Sparkd",
@@ -948,65 +812,6 @@
        "color": "terminal.ansiGreen",
      },
    },
-    {
-      "label": "Test Spark (Debug)",
-      "dependsOn": "Build Spark (Debug)",
-      "type": "process",
-      "icon": {
-        "id": "coverage",
-        "color": "terminal.ansiYellow",
-      },
-      "command": "",
-      "windows": {
-        "command": "Spark.Console.exe",
-        "args": [
-          "test",
-          "${input:sparkTest}"
-        ],
-        "options": {
-          "cwd": "${workspaceFolder}/build/Debug/bin/net8.0",
-        }
-      },
-    },
-    {
-      "label": "Test Sparkd",
-      "dependsOn": "Build Sparkd",
-      "type": "process",
-      "command": "",
-      "linux": {
-        "command": "sparkd",
-      },
-      "args": [
-        "test",
-        "${input:sparkdTest}",
-      ],
-      "options": {
-        "cwd": "${workspaceFolder}/build/${input:sparkdCmakePreset}/server"
-      },
-      "icon": {
-        "id": "coverage",
-        "color": "terminal.ansiYellow",
-      },
-    },
-    {
-      "label": "Test Sparkd (Skip Build)",
-      "type": "process",
-      "command": "",
-      "linux": {
-        "command": "sparkd",
-      },
-      "args": [
-        "test",
-        "${input:sparkdTest}",
-      ],
-      "options": {
-        "cwd": "${workspaceFolder}/build/${input:sparkdCmakePreset}/server"
-      },
-      "icon": {
-        "id": "coverage",
-        "color": "terminal.ansiYellow",
-      },
-    },
    {
      "label": "View DataStreamQueryAnalyzer Logs (Last Run)",
      "type": "process",
@@ -1125,16 +930,6 @@
      "id": "sparkdConfigQuery",
      "type": "promptString",
      "description": "Config Type(s)"
-    },
-    {
-      "id": "sparkTest",
-      "type": "promptString",
-      "description": "Test to run",
-    },
-    {
-      "id": "sparkdTest",
-      "type": "promptString",
-      "description": "Test to run",
-    },
+    }
  ]
 }
@@ -1,10 +1,7 @@
 cmake_minimum_required(VERSION 3.12)
 project(spark)
-include(GNUInstallDirs)
-include(${CMAKE_SOURCE_DIR}/cmake/GitVersion.cmake)
 set(CMAKE_CXX_STANDARD 20)

-get_version_from_git()
 if(WIN32)
  set(platform "windows")
 else()
@@ -21,10 +18,10 @@ set(
 # SHARED_MEMORY_PREFIX serves as a kind of namespace so multiple applications
 # can run simultaneously without fighting over the same shm files (e.g., sparkd,
 # api server, etc.).
-set(SHARED_MEMORY_PREFIX "\${BTS_SHARED_MEMORY_PREFIX}" CACHE STRING "Prefix to use when creating shared memory files")
-set(EXTERNALS_BIN "${EXTERNALS_ROOT}/bin")
-set(EXTERNALS_INCLUDE "${EXTERNALS_ROOT}/include")
-set(EXTERNALS_LIB "${EXTERNALS_ROOT}/lib/$<CONFIG>")
+set(SHARED_MEMORY_PREFIX "bts" CACHE STRING "Prefix to use when creating shared memory files")
+set(SPARK_BIN "${EXTERNALS_ROOT}/bin")
+set(SPARK_INCLUDE "${EXTERNALS_ROOT}/include")
+set(SPARK_LIB "${EXTERNALS_ROOT}/lib/$<CONFIG>")
 set(BUILD_DOCS OFF CACHE BOOL "Build documentation")
 set(BUILD_SDK OFF CACHE BOOL "Build the sdk")
 set(EXPORT_MD_CONSUMER OFF CACHE BOOL "Export (extern) C functions for MD consumer interop")
@@ -36,18 +33,25 @@ else()
  set(BUILD_ADMIN_WEB_APP ON CACHE BOOL "Build the Admin WebApp")
 endif()

-include_directories(BEFORE SYSTEM ${EXTERNALS_INCLUDE} ${EXTERNALS_ROOT}/include/zlib)
-link_directories(${EXTERNALS_LIB})
+include_directories(BEFORE SYSTEM ${SPARK_INCLUDE} ${EXTERNALS_ROOT}/include/zlib)
+link_directories(${SPARK_LIB})

 add_compile_definitions(SHARED_MEMORY_PREFIX="${SHARED_MEMORY_PREFIX}")
-if(DEFINED APP_DATA_DIR)
-  add_compile_definitions(APP_DATA_DIR="${APP_DATA_DIR}")
+add_compile_definitions(APP_DATA_DIR="${APP_DATA_DIR}")
+if(DEFINED APP_DATA_DIR_OVERRIDE)
+  add_compile_definitions(APP_DATA_DIR_OVERRIDE="${APP_DATA_DIR_OVERRIDE}")
 endif()
 add_compile_definitions(EXPORT_MD_CONSUMER="${EXPORT_MD_CONSUMER}")
 if(WIN32)
  # subdirs
-  add_subdirectory(src/lib_spark)
+  add_subdirectory(src/lib)
 else()
+  # make sparkd look for libspark.so in its directory
+  set(CMAKE_BUILD_WITH_INSTALL_RPATH TRUE)
+  set(CMAKE_INSTALL_RPATH ".")
+  set(CMAKE_INSTALL_PREFIX "${CMAKE_SOURCE_DIR}/install")
+  set(SPARK_INSTALL_PREFIX "${CMAKE_INSTALL_PREFIX}/$<CONFIG>")
+
  if(BUILD_SPARKD)
    # Note: All macros that begin w/ 'CCAPI' are ccapi's macros (defined and/or
    # read by their code). We override some below.
@@ -85,15 +89,9 @@ else()
    add_compile_definitions(BINANCE_US_PRICE_STATS_PATH="/api/v3/ticker/24hr")

    # ** Binance USDS Futures **
-    add_compile_definitions(CCAPI_BINANCE_USDS_FUTURES_URL_REST_BASE="https://fapi.binance.com")
-    add_compile_definitions(BINANCE_USDS_FUTURES_INSTRUMENTS_PATH="/fapi/v1/exchangeInfo")
-    add_compile_definitions(BINANCE_USDS_FUTURES_PRICE_STATS_PATH="/fapi/v1/ticker/24hr")
-    add_compile_definitions(BINANCE_USDS_FUTURES_USER_DATA_PATH="/api/v3/userDataStream")
-
-    # ** Binance Coin Futures **
-    add_compile_definitions(CCAPI_BINANCE_COIN_FUTURES_URL_REST_BASE="https://dapi.binance.com")
-    add_compile_definitions(BINANCE_COIN_FUTURES_INSTRUMENTS_PATH="/dapi/v1/exchangeInfo")
-    add_compile_definitions(BINANCE_COIN_FUTURES_PRICE_STATS_PATH="/dapi/v1/ticker/24hr")
+    add_compile_definitions(CCAPI_BINANCE_USDS_FUTURES_URL_REST_BASE="https://dapi.binance.com")
+    add_compile_definitions(BINANCE_USDS_FUTURES_INSTRUMENTS_PATH="/dapi/v1/exchangeInfo")
+    add_compile_definitions(BINANCE_USDS_FUTURES_PRICE_STATS_PATH="/dapi/v1/ticker/24hr")

    # ** Deribit **
    add_compile_definitions(DERIBIT_INSTRUMENT_PATH="/api/v2/public/get_instrument")
@@ -112,15 +110,11 @@ else()
  endif()

  # subdirs
-  add_subdirectory(src/lib_spark)
  if (BUILD_SPARKD)
-    add_subdirectory(src/lib_python_interop)
-    add_subdirectory(src/lib_onixs_fix)
-    add_subdirectory(src/lib_binance)
-    add_subdirectory(src/lib_okx_execution)
-    add_subdirectory(src/app_sparkd)
+    add_subdirectory(src/server/sparkd)
    add_subdirectory(src/data_files)
  endif()
+  add_subdirectory(src/lib)
  if (BUILD_ADMIN_WEB_APP)
    add_subdirectory(src/Spark.Admin.WebApp)
  endif()
@@ -130,4 +124,32 @@ else()
  if(BUILD_SDK)
    add_subdirectory(src/sdk)
  endif()
+
+  if(BUILD_SPARKD)
+    add_custom_command(
+      OUTPUT ${CMAKE_BINARY_DIR}/server
+      COMMAND mkdir -p ${CMAKE_BINARY_DIR}/server
+      COMMAND ln ARGS -sf ${CMAKE_BINARY_DIR}/src/server/sparkd/sparkd ${CMAKE_BINARY_DIR}/server
+      COMMAND ln ARGS -sf ${CMAKE_BINARY_DIR}/data_files ${CMAKE_BINARY_DIR}/server
+      COMMAND ln ARGS -sf ${CMAKE_BINARY_DIR}/src/lib/libspark.so ${CMAKE_BINARY_DIR}/server
+      COMMAND
+        ln ARGS -sf
+        ${EXTERNALS_ROOT}/lib/Release/libOnixS.IceImpactMulticastPriceFeedHandlerCpp.so.3.16.0.0
+        ${CMAKE_BINARY_DIR}/server
+      COMMAND ln ARGS -sf ${EXTERNALS_ROOT}/lib/Release/libACE-5.7.5.so ${CMAKE_BINARY_DIR}/server
+      COMMAND mkdir ARGS -p ${CMAKE_BINARY_DIR}/server/plugins
+      COMMENT "Creating server runtime layout"
+      DEPENDS sparkd data_files_dir)
+
+    add_custom_target(server_layout ALL DEPENDS ${CMAKE_BINARY_DIR}/server)
+
+    install(
+      DIRECTORY "${CMAKE_BINARY_DIR}/server/plugins"
+      DESTINATION "${SPARK_INSTALL_PREFIX}/bin")
+    install(
+      FILES
+      "${SPARK_LIB}/libACE-5.7.5.so"
+      "${SPARK_LIB}/libOnixS.IceImpactMulticastPriceFeedHandlerCpp.so.3.16.0.0"
+      DESTINATION "${SPARK_INSTALL_PREFIX}/bin")
+  endif()
 endif()
@@ -20,21 +20,23 @@
        "SPARK_CMAKE_CXX_FLAGS": "/MP /WX /wd4250 /wd4251 /wd4244 /wd4267 /wd4355 /wd4275 /wd4345 /bigobj /EHsc /DCURL_STATICLIB /D_WINSOCK_DEPRECATED_NO_WARNINGS /D__TBB_NO_IMPLICIT_LINKAGE /DBOOST_ALL_NO_LIB /std:c++20 /D_SCL_SECURE_NO_WARNINGS /D_CRT_SECURE_NO_WARNINGS /D_WIN32_WINNT=0x0600"
      },
      "cacheVariables": {
+        "APP_DATA_DIR": "${dollar}{APPDATA}/bts",
+        "APP_DATA_DIR_OVERRIDE": "${dollar}{SPARK_DATA_DIR}",
        "CMAKE_CXX_FLAGS": "$env{SPARK_CMAKE_CXX_FLAGS}",
        "CMAKE_CXX_FLAGS_DEBUG": "$env{SPARK_CMAKE_CXX_FLAGS} /MTd",
        "CMAKE_CXX_FLAGS_RELEASE": "$env{SPARK_CMAKE_CXX_FLAGS} /MT",
        "CMAKE_CXX_STANDARD": "20",
-        "CMAKE_INSTALL_PREFIX": "${sourceDir}/install/${presetName}",
        "CMAKE_SHARED_LINKER_FLAGS_DEBUG": "/DEBUG",
        "CMAKE_SHARED_LINKER_FLAGS_RELEASE": "/OPT:REF /OPT:ICF /LTCG",
        "CMAKE_TOOLCHAIN_FILE": "${sourceDir}/vcpkg/scripts/buildsystems/vcpkg.cmake",
        "EXTERNALS_ROOT": "${sourceDir}/externals/windows",
-        "CONFIG_DEFAULTS_PATH": "spark/config/profiles/default/*.conf",
-        "DOTNET_BINARY_DIR": "${sourceDir}/build/$<CONFIG>/bin/net8.0;${sourceDir}/build/$<CONFIG>/bin/net8.0-windows",
+        "CONFIG_DEFAULTS_PATH": "config/profiles/default/*.conf",
+        "DOTNET_BINARY_DIR": "${sourceDir}/build/$<CONFIG>/bin/net7.0;${sourceDir}/build/$<CONFIG>/bin/net7.0-windows",
        "EXPORT_MD_CONSUMER": {
          "type": "BOOL",
          "value": "ON"
-        }
+        },
+        "SHARED_MEMORY_PREFIX": "spark"
      }
    },

@@ -50,20 +52,16 @@
      "binaryDir": "${sourceDir}/build/${presetName}",
      "cacheVariables": {
        "platform": "",
+        "APP_DATA_DIR": "${dollar}{HOME}/sparkdata",
+        "APP_DATA_DIR_OVERRIDE": "${dollar}{SPARKD_DATA_DIR}",
        "CMAKE_BUILD_WITH_INSTALL_RPATH": {
-          "type": "BOOL",
-          "value": "OFF"
-        },
-        "CMAKE_CXX_STANDARD": "20",
-        "CMAKE_INSTALL_RPATH": ".",
-        "CMAKE_INSTALL_PREFIX": "${sourceDir}/install/${presetName}",
-        "CMAKE_PREFIX_PATH": "${sourceDir}/cmake/;${sourceDir}/.venv/lib/python3.10/site-packages/pybind11/share/cmake/pybind11",
-        "CMAKE_EXE_LINKER_FLAGS": "-Wno-undef",
-        "CMAKE_TOOLCHAIN_FILE": "${sourceDir}/vcpkg/scripts/buildsystems/vcpkg.cmake",
-        "DATABENTO_USE_EXTERNAL_JSON": {
          "type": "BOOL",
          "value": "ON"
        },
+        "CMAKE_CXX_STANDARD": "20",
+        "CMAKE_INSTALL_RPATH": ".",
+        "CMAKE_INSTALL_PREFIX": "${sourceDir}/install/${presetName}/sparkd",
+        "CMAKE_EXE_LINKER_FLAGS": "-Wno-undef",
        "EXTERNALS_ROOT": "${sourceDir}/externals/linux",
        "BUILD_SDK": {
          "type": "BOOL",
@@ -72,7 +70,8 @@
        "BUILD_ADMIN_WEB_APP": {
          "type": "BOOL",
          "value": "ON"
-        }
+        },
+        "SHARED_MEMORY_PREFIX": "sparkd"
      }
    },

@@ -87,7 +86,7 @@
          "value": "ON"
        },
        "CCAPI_LOG_LEVEL": "Info",
-        "CONFIG_DEFAULTS_PATH": "spark/config/profiles/md_dev/*.conf;spark/config/profiles/default/*.conf"
+        "CONFIG_DEFAULTS_PATH": "config/profiles/md_dev/*.conf;config/profiles/default/*.conf"
      }
    },

@@ -98,7 +97,7 @@
      "cacheVariables": {
        "CMAKE_BUILD_TYPE": "Release",
        "CCAPI_LOG_LEVEL": "Warn",
-        "CONFIG_DEFAULTS_PATH": "spark/config/profiles/md_prod/*.conf;spark/config/profiles/default/*.conf"
+        "CONFIG_DEFAULTS_PATH": "config/profiles/md_prod/*.conf;config/profiles/default/*.conf"
      }
    },

@@ -153,7 +152,11 @@
    {
      "name": "spark-api-server-windows",
      "inherits": "spark",
-      "displayName": "Spark Api Server (Windows)"
+      "displayName": "Spark Api Server (Windows)",
+      "cacheVariables": {
+        "APP_DATA_DIR_OVERRIDE": "${dollar}{SPARK_API_SERVER_DATA_DIR}",
+        "SHARED_MEMORY_PREFIX": "spark_api_server"
+      }
    },

    {
@@ -162,6 +165,7 @@
      "displayName": "Spark Api Server (Linux)",
      "generator": "Ninja Multi-Config",
      "cacheVariables": {
+        "APP_DATA_DIR_OVERRIDE": "${dollar}{SPARK_API_SERVER_DATA_DIR}",
        "BUILD_ADMIN_WEB_APP": {
          "type": "BOOL",
          "value": "OFF"
@@ -175,12 +179,8 @@
        "CMAKE_CXX_FLAGS_DEBUG": "-g -fPIC -Wall -pedantic -Wextra -Werror -Wno-missing-braces -Wno-ignored-qualifiers --std=c++20 -fdiagnostics-color -DDEBUG",
        "CMAKE_CXX_FLAGS_RELEASE": "-fPIC -Wall -pedantic -Wextra -Werror -Wno-missing-braces -Wno-ignored-qualifiers --std=c++20 -fdiagnostics-color -fno-omit-frame-pointer -DNDEBUG -O3",
        "CMAKE_INSTALL_PREFIX": "${sourceDir}/install/${presetName}",
-        "CMAKE_EXPORT_COMPILE_COMMANDS": {
-          "type": "BOOL",
-          "value": "ON"
-        },
-        "CONFIG_DEFAULTS_PATH": "spark/config/profiles/default/*.conf",
-        "DOTNET_BINARY_DIR": "${sourceDir}/build/Spark.Api.Server/$<CONFIG>/bin/net8.0",
+        "CONFIG_DEFAULTS_PATH": "config/profiles/default/*.conf",
+        "DOTNET_BINARY_DIR": "${sourceDir}/build/$<CONFIG>/bin/net7.0",
        "EXPORT_MD_CONSUMER": {
          "type": "BOOL",
          "value": "ON"
@@ -218,8 +218,11 @@
      "description": "Data Stream utils ported over from Edge",
      "binaryDir": "${sourceDir}/build/${presetName}",
      "cacheVariables": {
-        "CONFIG_DEFAULTS_PATH": "spark/config/profiles/default/*.conf;spark/config/profiles/data_stream/*.conf",
-        "DOTNET_BINARY_DIR": "${sourceDir}/build/$<CONFIG>/bin/net8.0;${sourceDir}/build/$<CONFIG>/bin/net8.0-windows"
+        "APP_DATA_DIR": "${dollar}{APPDATA}/bts/DataStream",
+        "APP_DATA_DIR_OVERRIDE": "${dollar}{DATA_STREAM_DATA_DIR}",
+        "CONFIG_DEFAULTS_PATH": "config/profiles/default/*.conf;config/profiles/data_stream/*.conf",
+        "DOTNET_BINARY_DIR": "${sourceDir}/build/$<CONFIG>/bin/net7.0;${sourceDir}/build/$<CONFIG>/bin/net7.0-windows;${sourceDir}/build/$<CONFIG>/bin/net8.0",
+        "SHARED_MEMORY_PREFIX": "data_stream"
      }
    },

@@ -235,6 +238,8 @@
      "inherits": "sparkd",
      "generator": "Ninja Multi-Config",
      "cacheVariables": {
+        "APP_DATA_DIR": "${dollar}{HOME}/.local/share/DataStream",
+        "APP_DATA_DIR_OVERRIDE": "${dollar}{DATA_STREAM_DATA_DIR}",
        "BUILD_ADMIN_WEB_APP": {
          "type": "BOOL",
          "value": "OFF"
@@ -251,12 +256,14 @@
        "CMAKE_CXX_COMPILER": "clang++",
        "CMAKE_CXX_FLAGS_DEBUG": "-g -fPIC -Wall -pedantic -Wextra -Werror -Wno-missing-braces -Wno-ignored-qualifiers --std=c++20 -fdiagnostics-color -DDEBUG",
        "CMAKE_CXX_FLAGS_RELEASE": "-fPIC -Wall -pedantic -Wextra -Werror -Wno-missing-braces -Wno-ignored-qualifiers --std=c++20 -fdiagnostics-color -fno-omit-frame-pointer -DNDEBUG -O3",
-        "CONFIG_DEFAULTS_PATH": "spark/config/profiles/default/*.conf;spark/config/profiles/data_stream/*.conf",
-        "DOTNET_BINARY_DIR": "${sourceDir}/build/$<CONFIG>/bin/net7.0",
+        "CMAKE_INSTALL_PREFIX": "${sourceDir}/install/${presetName}",
+        "CONFIG_DEFAULTS_PATH": "config/profiles/default/*.conf;config/profiles/data_stream/*.conf",
+        "DOTNET_BINARY_DIR": "${sourceDir}/build/$<CONFIG>/bin/net7.0;${sourceDir}/build/$<CONFIG>/bin/net8.0",
        "EXPORT_MD_CONSUMER": {
          "type": "BOOL",
          "value": "ON"
-        }
+        },
+        "SHARED_MEMORY_PREFIX": "data_stream"
      }
    }
  ],
@@ -295,7 +302,16 @@
        "lhs": "${hostSystemName}",
        "rhs": "Linux"
      },
-      "hidden": true
+      "hidden": true,
+      "targets": [
+        "data_files_dir",
+        "admin_web_app_bundle",
+        "install",
+        "libspark",
+        "rapidjson_project",
+        "server_layout",
+        "sparkd"
+      ]
    },

    {
@@ -14,34 +14,6 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "spark_test", "src\client\sp
 EndProject
 Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Spark.Console", "src\client\Spark.Console\Spark.Console.csproj", "{89D666E6-B500-498F-B026-4D167F68ADFC}"
 EndProject
-Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "src", "src", "{267F81EF-A8AB-49FD-9F56-D7EC3C1F7C7C}"
-EndProject
-Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "DataStreamSuite", "DataStreamSuite", "{C09ED521-D2DF-40BC-A76F-EB3CB870B2F6}"
-EndProject
-Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "DataStreamCore", "src\DataStreamSuite\DataStreamCore\DataStreamCore.csproj", "{EF26A475-6101-4C9C-A717-D60BB92A2302}"
-EndProject
-Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "SharedStreams", "src\DataStreamSuite\SharedStreams\SharedStreams.csproj", "{911B08A3-515A-441D-84C8-8A43740B5FC1}"
-EndProject
-Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "PubSub", "src\DataStreamSuite\PubSub\PubSub.csproj", "{14140310-2233-430A-821B-F3058A25234B}"
-EndProject
-Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "BaseUtilities", "src\DataStreamSuite\BaseUtilities\BaseUtilities.csproj", "{43146EA8-6FCC-46B8-88B0-42F9BB5E6E0A}"
-EndProject
-Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "ExpressionCore", "src\DataStreamSuite\ExpressionCore\ExpressionCore.csproj", "{6F22DB96-6EAB-4034-99D9-287021C36F01}"
-EndProject
-Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "ArithmeticExpressions", "src\DataStreamSuite\ArithmeticExpressions\ArithmeticExpressions.csproj", "{A5C33956-A41F-49B3-A41D-1082CBA59D02}"
-EndProject
-Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "DataStreamExpressions", "src\DataStreamSuite\DataStreamExpressions\DataStreamExpressions.csproj", "{2FE745C3-1EDF-4421-8CD7-A63EE0C1AE43}"
-EndProject
-Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Bts.ColumnSortExpressions", "src\DataStreamSuite\Bts.ColumnSortExpressions\Bts.ColumnSortExpressions.csproj", "{FDA791E0-2B91-4AD5-B91D-CF920A4C4E38}"
-EndProject
-Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "ColumnMapExpressions", "src\DataStreamSuite\ColumnMapExpressions\ColumnMapExpressions.csproj", "{B945A183-2174-4ACB-B289-8A5C546656FE}"
-EndProject
-Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "BooleanExpressions", "src\DataStreamSuite\BooleanExpressions\BooleanExpressions.csproj", "{9F6D7DE8-185B-4C1D-B809-3314D092DF28}"
-EndProject
-Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Bts.ColumnExpressions", "src\DataStreamSuite\Bts.ColumnExpressions\Bts.ColumnExpressions.csproj", "{6BD3F6D8-B6FC-44E8-B8FE-06352F638B9B}"
-EndProject
-Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "DataStream", "src\DataStreamSuite\DataStream\DataStream.csproj", "{39AEA075-6457-46C0-8821-1158BE6EFA57}"
-EndProject
 Global
 	GlobalSection(SolutionConfigurationPlatforms) = preSolution
 		Debug|x64 = Debug|x64
@@ -98,102 +70,6 @@ Global
 		{89D666E6-B500-498F-B026-4D167F68ADFC}.Release|x64.Build.0 = Release|x64
 		{89D666E6-B500-498F-B026-4D167F68ADFC}.RelWithDebInfo|x64.ActiveCfg = Debug|x64
 		{89D666E6-B500-498F-B026-4D167F68ADFC}.RelWithDebInfo|x64.Build.0 = Debug|x64
-		{EF26A475-6101-4C9C-A717-D60BB92A2302}.Debug|x64.ActiveCfg = Debug|Any CPU
-		{EF26A475-6101-4C9C-A717-D60BB92A2302}.Debug|x64.Build.0 = Debug|Any CPU
-		{EF26A475-6101-4C9C-A717-D60BB92A2302}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
-		{EF26A475-6101-4C9C-A717-D60BB92A2302}.MinSizeRel|x64.Build.0 = Debug|Any CPU
-		{EF26A475-6101-4C9C-A717-D60BB92A2302}.Release|x64.ActiveCfg = Release|Any CPU
-		{EF26A475-6101-4C9C-A717-D60BB92A2302}.Release|x64.Build.0 = Release|Any CPU
-		{EF26A475-6101-4C9C-A717-D60BB92A2302}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
-		{EF26A475-6101-4C9C-A717-D60BB92A2302}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
-		{911B08A3-515A-441D-84C8-8A43740B5FC1}.Debug|x64.ActiveCfg = Debug|Any CPU
-		{911B08A3-515A-441D-84C8-8A43740B5FC1}.Debug|x64.Build.0 = Debug|Any CPU
-		{911B08A3-515A-441D-84C8-8A43740B5FC1}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
-		{911B08A3-515A-441D-84C8-8A43740B5FC1}.MinSizeRel|x64.Build.0 = Debug|Any CPU
-		{911B08A3-515A-441D-84C8-8A43740B5FC1}.Release|x64.ActiveCfg = Release|Any CPU
-		{911B08A3-515A-441D-84C8-8A43740B5FC1}.Release|x64.Build.0 = Release|Any CPU
-		{911B08A3-515A-441D-84C8-8A43740B5FC1}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
-		{911B08A3-515A-441D-84C8-8A43740B5FC1}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
-		{14140310-2233-430A-821B-F3058A25234B}.Debug|x64.ActiveCfg = Debug|Any CPU
-		{14140310-2233-430A-821B-F3058A25234B}.Debug|x64.Build.0 = Debug|Any CPU
-		{14140310-2233-430A-821B-F3058A25234B}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
-		{14140310-2233-430A-821B-F3058A25234B}.MinSizeRel|x64.Build.0 = Debug|Any CPU
-		{14140310-2233-430A-821B-F3058A25234B}.Release|x64.ActiveCfg = Release|Any CPU
-		{14140310-2233-430A-821B-F3058A25234B}.Release|x64.Build.0 = Release|Any CPU
-		{14140310-2233-430A-821B-F3058A25234B}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
-		{14140310-2233-430A-821B-F3058A25234B}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
-		{43146EA8-6FCC-46B8-88B0-42F9BB5E6E0A}.Debug|x64.ActiveCfg = Debug|Any CPU
-		{43146EA8-6FCC-46B8-88B0-42F9BB5E6E0A}.Debug|x64.Build.0 = Debug|Any CPU
-		{43146EA8-6FCC-46B8-88B0-42F9BB5E6E0A}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
-		{43146EA8-6FCC-46B8-88B0-42F9BB5E6E0A}.MinSizeRel|x64.Build.0 = Debug|Any CPU
-		{43146EA8-6FCC-46B8-88B0-42F9BB5E6E0A}.Release|x64.ActiveCfg = Release|Any CPU
-		{43146EA8-6FCC-46B8-88B0-42F9BB5E6E0A}.Release|x64.Build.0 = Release|Any CPU
-		{43146EA8-6FCC-46B8-88B0-42F9BB5E6E0A}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
-		{43146EA8-6FCC-46B8-88B0-42F9BB5E6E0A}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
-		{6F22DB96-6EAB-4034-99D9-287021C36F01}.Debug|x64.ActiveCfg = Debug|Any CPU
-		{6F22DB96-6EAB-4034-99D9-287021C36F01}.Debug|x64.Build.0 = Debug|Any CPU
-		{6F22DB96-6EAB-4034-99D9-287021C36F01}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
-		{6F22DB96-6EAB-4034-99D9-287021C36F01}.MinSizeRel|x64.Build.0 = Debug|Any CPU
-		{6F22DB96-6EAB-4034-99D9-287021C36F01}.Release|x64.ActiveCfg = Release|Any CPU
-		{6F22DB96-6EAB-4034-99D9-287021C36F01}.Release|x64.Build.0 = Release|Any CPU
-		{6F22DB96-6EAB-4034-99D9-287021C36F01}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
-		{6F22DB96-6EAB-4034-99D9-287021C36F01}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
-		{A5C33956-A41F-49B3-A41D-1082CBA59D02}.Debug|x64.ActiveCfg = Debug|Any CPU
-		{A5C33956-A41F-49B3-A41D-1082CBA59D02}.Debug|x64.Build.0 = Debug|Any CPU
-		{A5C33956-A41F-49B3-A41D-1082CBA59D02}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
-		{A5C33956-A41F-49B3-A41D-1082CBA59D02}.MinSizeRel|x64.Build.0 = Debug|Any CPU
-		{A5C33956-A41F-49B3-A41D-1082CBA59D02}.Release|x64.ActiveCfg = Release|Any CPU
-		{A5C33956-A41F-49B3-A41D-1082CBA59D02}.Release|x64.Build.0 = Release|Any CPU
-		{A5C33956-A41F-49B3-A41D-1082CBA59D02}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
-		{A5C33956-A41F-49B3-A41D-1082CBA59D02}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
-		{2FE745C3-1EDF-4421-8CD7-A63EE0C1AE43}.Debug|x64.ActiveCfg = Debug|Any CPU
-		{2FE745C3-1EDF-4421-8CD7-A63EE0C1AE43}.Debug|x64.Build.0 = Debug|Any CPU
-		{2FE745C3-1EDF-4421-8CD7-A63EE0C1AE43}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
-		{2FE745C3-1EDF-4421-8CD7-A63EE0C1AE43}.MinSizeRel|x64.Build.0 = Debug|Any CPU
-		{2FE745C3-1EDF-4421-8CD7-A63EE0C1AE43}.Release|x64.ActiveCfg = Release|Any CPU
-		{2FE745C3-1EDF-4421-8CD7-A63EE0C1AE43}.Release|x64.Build.0 = Release|Any CPU
-		{2FE745C3-1EDF-4421-8CD7-A63EE0C1AE43}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
-		{2FE745C3-1EDF-4421-8CD7-A63EE0C1AE43}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
-		{FDA791E0-2B91-4AD5-B91D-CF920A4C4E38}.Debug|x64.ActiveCfg = Debug|Any CPU
-		{FDA791E0-2B91-4AD5-B91D-CF920A4C4E38}.Debug|x64.Build.0 = Debug|Any CPU
-		{FDA791E0-2B91-4AD5-B91D-CF920A4C4E38}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
-		{FDA791E0-2B91-4AD5-B91D-CF920A4C4E38}.MinSizeRel|x64.Build.0 = Debug|Any CPU
-		{FDA791E0-2B91-4AD5-B91D-CF920A4C4E38}.Release|x64.ActiveCfg = Release|Any CPU
-		{FDA791E0-2B91-4AD5-B91D-CF920A4C4E38}.Release|x64.Build.0 = Release|Any CPU
-		{FDA791E0-2B91-4AD5-B91D-CF920A4C4E38}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
-		{FDA791E0-2B91-4AD5-B91D-CF920A4C4E38}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
-		{B945A183-2174-4ACB-B289-8A5C546656FE}.Debug|x64.ActiveCfg = Debug|Any CPU
-		{B945A183-2174-4ACB-B289-8A5C546656FE}.Debug|x64.Build.0 = Debug|Any CPU
-		{B945A183-2174-4ACB-B289-8A5C546656FE}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
-		{B945A183-2174-4ACB-B289-8A5C546656FE}.MinSizeRel|x64.Build.0 = Debug|Any CPU
-		{B945A183-2174-4ACB-B289-8A5C546656FE}.Release|x64.ActiveCfg = Release|Any CPU
-		{B945A183-2174-4ACB-B289-8A5C546656FE}.Release|x64.Build.0 = Release|Any CPU
-		{B945A183-2174-4ACB-B289-8A5C546656FE}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
-		{B945A183-2174-4ACB-B289-8A5C546656FE}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
-		{39AEA075-6457-46C0-8821-1158BE6EFA57}.Debug|x64.ActiveCfg = Debug|Any CPU
-		{39AEA075-6457-46C0-8821-1158BE6EFA57}.Debug|x64.Build.0 = Debug|Any CPU
-		{39AEA075-6457-46C0-8821-1158BE6EFA57}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
-		{39AEA075-6457-46C0-8821-1158BE6EFA57}.MinSizeRel|x64.Build.0 = Debug|Any CPU
-		{39AEA075-6457-46C0-8821-1158BE6EFA57}.Release|x64.ActiveCfg = Release|Any CPU
-		{39AEA075-6457-46C0-8821-1158BE6EFA57}.Release|x64.Build.0 = Release|Any CPU
-		{39AEA075-6457-46C0-8821-1158BE6EFA57}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
-		{39AEA075-6457-46C0-8821-1158BE6EFA57}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
-		{9F6D7DE8-185B-4C1D-B809-3314D092DF28}.Debug|x64.ActiveCfg = Debug|Any CPU
-		{9F6D7DE8-185B-4C1D-B809-3314D092DF28}.Debug|x64.Build.0 = Debug|Any CPU
-		{9F6D7DE8-185B-4C1D-B809-3314D092DF28}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
-		{9F6D7DE8-185B-4C1D-B809-3314D092DF28}.MinSizeRel|x64.Build.0 = Debug|Any CPU
-		{9F6D7DE8-185B-4C1D-B809-3314D092DF28}.Release|x64.ActiveCfg = Release|Any CPU
-		{9F6D7DE8-185B-4C1D-B809-3314D092DF28}.Release|x64.Build.0 = Release|Any CPU
-		{9F6D7DE8-185B-4C1D-B809-3314D092DF28}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
-		{9F6D7DE8-185B-4C1D-B809-3314D092DF28}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
-		{6BD3F6D8-B6FC-44E8-B8FE-06352F638B9B}.Debug|x64.ActiveCfg = Debug|Any CPU
-		{6BD3F6D8-B6FC-44E8-B8FE-06352F638B9B}.Debug|x64.Build.0 = Debug|Any CPU
-		{6BD3F6D8-B6FC-44E8-B8FE-06352F638B9B}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
-		{6BD3F6D8-B6FC-44E8-B8FE-06352F638B9B}.MinSizeRel|x64.Build.0 = Debug|Any CPU
-		{6BD3F6D8-B6FC-44E8-B8FE-06352F638B9B}.Release|x64.ActiveCfg = Release|Any CPU
-		{6BD3F6D8-B6FC-44E8-B8FE-06352F638B9B}.Release|x64.Build.0 = Release|Any CPU
-		{6BD3F6D8-B6FC-44E8-B8FE-06352F638B9B}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
-		{6BD3F6D8-B6FC-44E8-B8FE-06352F638B9B}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
 	EndGlobalSection
 	GlobalSection(SolutionProperties) = preSolution
 		HideSolutionNode = FALSE
@@ -204,19 +80,4 @@ Global
 	GlobalSection(Performance) = preSolution
 		HasPerformanceSessions = true
 	EndGlobalSection
-	GlobalSection(NestedProjects) = preSolution
-		{C09ED521-D2DF-40BC-A76F-EB3CB870B2F6} = {267F81EF-A8AB-49FD-9F56-D7EC3C1F7C7C}
-		{EF26A475-6101-4C9C-A717-D60BB92A2302} = {C09ED521-D2DF-40BC-A76F-EB3CB870B2F6}
-		{911B08A3-515A-441D-84C8-8A43740B5FC1} = {C09ED521-D2DF-40BC-A76F-EB3CB870B2F6}
-		{14140310-2233-430A-821B-F3058A25234B} = {C09ED521-D2DF-40BC-A76F-EB3CB870B2F6}
-		{43146EA8-6FCC-46B8-88B0-42F9BB5E6E0A} = {C09ED521-D2DF-40BC-A76F-EB3CB870B2F6}
-		{6F22DB96-6EAB-4034-99D9-287021C36F01} = {C09ED521-D2DF-40BC-A76F-EB3CB870B2F6}
-		{A5C33956-A41F-49B3-A41D-1082CBA59D02} = {C09ED521-D2DF-40BC-A76F-EB3CB870B2F6}
-		{2FE745C3-1EDF-4421-8CD7-A63EE0C1AE43} = {C09ED521-D2DF-40BC-A76F-EB3CB870B2F6}
-		{FDA791E0-2B91-4AD5-B91D-CF920A4C4E38} = {C09ED521-D2DF-40BC-A76F-EB3CB870B2F6}
-		{B945A183-2174-4ACB-B289-8A5C546656FE} = {C09ED521-D2DF-40BC-A76F-EB3CB870B2F6}
-		{39AEA075-6457-46C0-8821-1158BE6EFA57} = {C09ED521-D2DF-40BC-A76F-EB3CB870B2F6}
-		{9F6D7DE8-185B-4C1D-B809-3314D092DF28} = {C09ED521-D2DF-40BC-A76F-EB3CB870B2F6}
-		{6BD3F6D8-B6FC-44E8-B8FE-06352F638B9B} = {C09ED521-D2DF-40BC-A76F-EB3CB870B2F6}
-	EndGlobalSection
 EndGlobal
@@ -1,7 +0,0 @@
-include_guard(GLOBAL)
-
-macro(export_to_parent)
-  foreach(var IN ITEMS ${ARGN})
-    set(${var} ${${var}} PARENT_SCOPE)
-  endforeach()
-endmacro()
@@ -1,48 +0,0 @@
-include_guard(GLOBAL)
-
-function(get_version_from_git)
-  find_package(Git QUIET)
-  if(NOT Git_FOUND)
-    message(WARNING "Git not found")
-    return()
-  endif()
-
-  execute_process(
-    COMMAND ${GIT_EXECUTABLE} describe --tags --always
-    WORKING_DIRECTORY ${CMAKE_SOURCE_DIR}
-    OUTPUT_VARIABLE GIT_TAG
-    OUTPUT_STRIP_TRAILING_WHITESPACE
-    RESULT_VARIABLE GIT_RESULT
-  )
-
-  if(NOT GIT_RESULT EQUAL 0)
-    message(WARNING "Failed to get git tag")
-    return()
-  endif()
-
-  execute_process(
-    COMMAND ${GIT_EXECUTABLE} rev-parse --short=7 HEAD
-    WORKING_DIRECTORY ${CMAKE_SOURCE_DIR}
-    OUTPUT_VARIABLE GIT_COMMIT_SHORT_HASH
-    OUTPUT_STRIP_TRAILING_WHITESPACE
-  )
-  set(PROJECT_VERSION_BUILD_METADATA ${GIT_COMMIT_SHORT_HASH})
-  set(PROJECT_VERSION_BUILD_METADATA ${GIT_COMMIT_SHORT_HASH} PARENT_SCOPE)
-
-  string(REGEX REPLACE "^v" "" CLEAN_TAG "${GIT_TAG}")
-  if(CLEAN_TAG MATCHES "^([0-9]+)\\.([0-9]+)\\.([0-9]+)(-.*)?$")
-    set(PROJECT_VERSION_MAJOR ${CMAKE_MATCH_1})
-    set(PROJECT_VERSION_MAJOR ${CMAKE_MATCH_1} PARENT_SCOPE)
-    set(PROJECT_VERSION_MINOR ${CMAKE_MATCH_2})
-    set(PROJECT_VERSION_MINOR ${CMAKE_MATCH_2} PARENT_SCOPE)
-    set(PROJECT_VERSION_PATCH ${CMAKE_MATCH_3})
-    set(PROJECT_VERSION_PATCH ${CMAKE_MATCH_3} PARENT_SCOPE)
-
-    set(PROJECT_VERSION "${CMAKE_MATCH_1}.${CMAKE_MATCH_2}.${CMAKE_MATCH_3}") 
-    set(PROJECT_VERSION "${CMAKE_MATCH_1}.${CMAKE_MATCH_2}.${CMAKE_MATCH_3}" PARENT_SCOPE)
-    set(PROJECT_VERSION_FULL "${CMAKE_MATCH_1}.${CMAKE_MATCH_2}.${CMAKE_MATCH_3}+${GIT_COMMIT_SHORT_HASH}")
-    set(PROJECT_VERSION_FULL "${CMAKE_MATCH_1}.${CMAKE_MATCH_2}.${CMAKE_MATCH_3}+${GIT_COMMIT_SHORT_HASH}" PARENT_SCOPE)
-  else()
-    message(WARNING "Tag '${CLEAN_TAG}' does not match semver format")
-  endif()
-endfunction()
@@ -1,142 +0,0 @@
-include_guard(DIRECTORY)
-include(${CMAKE_SOURCE_DIR}/cmake/ExportToParent.cmake)
-include(${CMAKE_SOURCE_DIR}/cmake/InstallBasic.cmake)
-include(${CMAKE_SOURCE_DIR}/cmake/ProtobufGenerateCpp.cmake)
-include(${CMAKE_SOURCE_DIR}/cmake/SetFirstDefined.cmake)
-include(${CMAKE_SOURCE_DIR}/cmake/safe_dep_glob.cmake)
-
-function(init_library)
-  set(_options SHARED STATIC)
-  set(_one_value_args
-    NAME
-    SOURCE_DIR
-    INCLUDE_DIR
-  )
-  set(_multi_value_args)
-  cmake_parse_arguments(_ARGS "${_options}" "${_one_value_args}" "${_multi_value_args}" ${ARGN})
-  if (_ARGS_SHARED AND _ARGS_STATIC)
-    message(FATAL_ERROR "Invalid args passed to init_library: cannot specify STATIC and SHARED")
-  endif()
-  set_first_defined(_name _ARGS_NAME PROJECT_NAME)
-  set_first_defined(_source_dir _ARGS_SOURCE_DIR _name)
-  set_first_defined(_include_dir _ARGS_INCLUDE_DIR _source_dir)
-
-  # Cpp sources
-  file(
-    GLOB_RECURSE ${_name}_SOURCES
-    RELATIVE ${CMAKE_CURRENT_SOURCE_DIR}
-    ${_source_dir}/*.cpp
-  )
-  # Note: cpp files inside 'tests' dir(s) are considered tests.
-  # Remove tests from main sources, and add them to test sources
-  foreach(item ${${_name}_SOURCES})
-    if(${item} MATCHES "tests/.*\.cpp$")
-      list(APPEND ${_name}_TEST_SOURCES ${item})
-    endif()
-  endforeach()
-  foreach(item ${${_name}_TEST_SOURCES})
-    list(REMOVE_ITEM ${_name}_SOURCES ${item})
-  endforeach()
-
-  # Proto sources 
-  file(
-    GLOB_RECURSE ${_name}_PROTO_SOURCES
-    RELATIVE ${CMAKE_CURRENT_SOURCE_DIR}
-    ${_source_dir}/protobufs/*.proto
-  )
-  update_deps_file("${${_name}_PROTO_SOURCES};")
-  protobuf_generate_cpp(
-    CXX_SOURCES ${_name}_CXX_PROTO_SOURCES
-    CXX_HEADERS ${_name}_CXX_PROTO_HEADERS
-    CXX_OUT_DIR ${CMAKE_CURRENT_BINARY_DIR}
-    PROTO_INCLUDE_DIRS
-      ${EXTERNALS_INCLUDE}
-      ${CMAKE_CURRENT_SOURCE_DIR}
-    PROTO_SOURCES
-      ${${_name}_PROTO_SOURCES}
-  )
-
-  export_to_parent(
-    ${_name}_SOURCES
-    ${_name}_TEST_SOURCES
-    ${_name}_PROTO_SOURCES
-    ${_name}_CXX_PROTO_SOURCES
-    ${_name}_CXX_PROTO_HEADERS
-  )
-
-  # Add main lib
-  if (_ARGS_STATIC)
-    add_library(${_name} STATIC ${${_name}_SOURCES} ${${_name}_CXX_PROTO_SOURCES})
-  elseif(_ARGS_SHARED)
-    add_library(${_name} SHARED ${${_name}_SOURCES} ${${_name}_CXX_PROTO_SOURCES})
-  else()
-    add_library(${_name} ${${_name}_SOURCES} ${${_name}_CXX_PROTO_SOURCES})
-  endif()
-  target_include_directories(
-    ${_name}
-    SYSTEM
-    BEFORE
-    PUBLIC
-      $<BUILD_INTERFACE:${EXTERNALS_ROOT}/include>
-      $<BUILD_INTERFACE:${EXTERNALS_ROOT}/include/zlib>
-  )
-  target_include_directories(
-    ${_name}
-    PUBLIC
-      $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}>
-      $<INSTALL_INTERFACE:${CMAKE_INSTALL_INCLUDEDIR}>
-  )
-  # Include protos (if any)
-  if (${_name}_PROTO_SOURCES)
-    target_include_directories(
-      ${_name}
-      PUBLIC
-        $<BUILD_INTERFACE:${CMAKE_CURRENT_BINARY_DIR}>
-    )
-  endif()
-  target_link_directories(
-    ${_name}
-    PUBLIC
-      $<BUILD_INTERFACE:${CMAKE_CURRENT_BINARY_DIR}>
-      $<BUILD_INTERFACE:${EXTERNALS_ROOT}/lib/$<CONFIG>>
-      $<INSTALL_INTERFACE:${CMAKE_INSTALL_LIBDIR}>
-  )
-  # Install main lib
-  foreach(_hdr ${${_name}_CXX_PROTO_HEADERS})
-    file(RELATIVE_PATH _relative_path ${CMAKE_CURRENT_BINARY_DIR} ${_hdr})
-    install(FILES ${_hdr} DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}/${_relative_path})
-  endforeach()
-  install_basic(
-    NAME ${_name}
-    INCLUDE_DIR ${_include_dir}
-  )
-
-  # Add tests executable (if there are any)
-  if (${_name}_TEST_SOURCES AND NOT WIN32)
-    add_executable(${_name}-tests ${${_name}_TEST_SOURCES})
-
-    get_target_property(_lib_type ${_name} TYPE)
-    if (_lib_type STREQUAL "SHARED_LIBRARY")
-      # Set RPATH so the tests exe knows where to find the shared lib.
-      set_target_properties(
-        ${_name}-tests
-        PROPERTIES
-          BUILD_RPATH "$ORIGIN"
-          INSTALL_RPATH "$ORIGIN/../${CMAKE_INSTALL_LIBDIR}"
-      )
-    endif()
-
-    # Link main lib
-    target_link_directories(
-      ${_name}-tests
-      PRIVATE
-        $<BUILD_INTERFACE:${CMAKE_CURRENT_BINARY_DIR}>
-        $<INSTALL_INTERFACE:${CMAKE_INSTALL_LIBDIR}>
-    )
-    target_link_libraries(${_name}-tests PRIVATE ${_name})
-
-    # Catch2
-    find_package(Catch2 CONFIG REQUIRED COMPONENTS Catch2WithMain)
-    target_link_libraries(${_name}-tests PRIVATE Catch2::Catch2WithMain)
-  endif()
-endfunction()
@@ -1,71 +0,0 @@
-include_guard(DIRECTORY)
-include(GNUInstallDirs)
-include(${CMAKE_SOURCE_DIR}/cmake/ExportToParent.cmake)
-include(${CMAKE_SOURCE_DIR}/cmake/SetFirstDefined.cmake)
-
-function(init_python_module)
-  set(_options)
-  set(_one_value_args
-    NAME
-  )
-  set(_multi_value_args
-    SOURCES
-  )
-  cmake_parse_arguments(_ARGS "${_options}" "${_one_value_args}" "${_multi_value_args}" ${ARGN})
-  set_first_defined(_name _ARGS_NAME PROJECT_NAME)
-  set_first_defined(_sources _ARGS_SOURCES ${_name}_SOURCES)
-
-  set(module_proj ${_name}_module)
-  set(${module_proj}_SOURCES ${_sources})
-  pybind11_add_module(${module_proj} SHARED ${_sources})
-  target_link_libraries(
-    ${module_proj}
-    PUBLIC
-      pybind11::embed
-      spark
-      ${_name}
-  )
-  target_include_directories(
-    ${module_proj}
-    PRIVATE
-      $<BUILD_INTERFACE:${pybind11_INCLUDE_DIRS}>
-  )
-  target_include_directories(
-    ${module_proj}
-    PUBLIC
-      ${Python3_INCLUDE_DIRS}
-      $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}>
-      $<INSTALL_INTERFACE:${CMAKE_INSTALL_INCLUDEDIR}>
-  )
-  add_custom_command(
-    OUTPUT ${CMAKE_CURRENT_BINARY_DIR}/stubs/${_name}.pyi
-    DEPENDS ${_name} ${module_proj}
-    COMMAND PYTHONPATH=${CMAKE_CURRENT_BINARY_DIR} pybind11-stubgen --numpy-array-use-type-var --ignore-invalid-expressions '.*' ${_name}_module
-    COMMENT "Generating py stubs"
-  )
-  add_custom_target(
-    "${_name}_py_stubs"
-    ALL
-    DEPENDS
-      ${CMAKE_CURRENT_BINARY_DIR}/stubs/${_name}.pyi
-  )
-  install(
-    TARGETS ${module_proj}
-    EXPORT ${module_proj}Targets
-    RUNTIME
-      DESTINATION ${CMAKE_INSTALL_BINDIR}
-    LIBRARY
-      DESTINATION ${CMAKE_INSTALL_LIBDIR}
-    ARCHIVE
-      DESTINATION ${CMAKE_INSTALL_LIBDIR}
-  )
-  foreach(dir ${pybind11_INCLUDE_DIRS})
-    install(DIRECTORY ${dir} DESTINATION ${CMAKE_INSTALL_INCLUDEDIR})
-  endforeach()
-  install(
-    EXPORT ${module_proj}Targets
-    FILE ${module_proj}Targets.cmake
-    NAMESPACE ${_name}::
-    DESTINATION ${CMAKE_INSTALL_LIBDIR}/cmake/${_name}
-  )
-endfunction()
@@ -1,65 +0,0 @@
-include_guard(DIRECTORY)
-include(GNUInstallDirs)
-
-function(init_python_plugin)
-  set(_options)
-  set(_one_value_args NAME PLUGIN_NAME)
-  set(_multi_value_args SOURCES)
-  cmake_parse_arguments(_ARGS "${_options}" "${_one_value_args}" "${_multi_value_args}" ${ARGN})
-  set_first_defined(_name _ARGS_NAME PROJECT_NAME)
-  set(_plugin_name ${_ARGS_PLUGIN_NAME})
-  if (NOT DEFINED _ARGS_PLUGIN_NAME)
-    set(_plugin_name ${_name}_plugins)
-  endif()
-  set(${_plugin_name}_SOURCES ${_ARGS_SOURCES})
-  if (NOT DEFINED _ARGS_SOURCES)
-    file(
-      GLOB_RECURSE ${_plugin_name}_SOURCES
-      RELATIVE ${CMAKE_CURRENT_SOURCE_DIR}
-      ${_plugin_name}/*.py)
-  endif()
-  set(src_outs)
-  foreach(_src ${${_plugin_name}_SOURCES})
-    set(_src_in ${CMAKE_CURRENT_SOURCE_DIR}/${_src})
-    set(_src_out ${CMAKE_CURRENT_BINARY_DIR}/${_src})
-    add_custom_command(
-      OUTPUT ${_src_out}
-      COMMAND ${CMAKE_COMMAND} -E copy ${_src_in} ${_src_out}
-      DEPENDS ${_src_in}
-      COMMENT "Copying ${_plugin_name} plugin ${_src} to ${_src_out}"
-    )
-    list(APPEND src_outs ${_src_out})
-  endforeach()
-  
-  add_custom_target(copy_${_plugin_name} ALL DEPENDS ${src_outs})
-  
-  add_library(${_plugin_name} INTERFACE)
-  add_library(${_name}::${_plugin_name} ALIAS ${_plugin_name})
-  
-  set_target_properties(
-    ${_plugin_name}
-    PROPERTIES
-      INTERFACE_PYTHON_SCRIPTS_DIR
-        $<BUILD_INTERFACE:${CMAKE_CURRENT_BINARY_DIR}/${_plugin_name}>$<INSTALL_INTERFACE:${CMAKE_INSTALL_BINDIR}/${_plugin_name}>
-  )
-  
-  set_target_properties(
-    ${_plugin_name}
-    PROPERTIES
-    INTERFACE_PYTHON_SCRIPTS $<BUILD_INTERFACE:${CMAKE_CURRENT_BINARY_DIR}/${${_plugin_name}_SOURCES}>$<INSTALL_INTERFACE:${CMAKE_INSTALL_BINDIR}/${${_plugin_name}_SOURCES}>
-  )
-  
-  install(
-    TARGETS ${_plugin_name}
-    EXPORT ${_plugin_name}Targets
-    DESTINATION ${CMAKE_INSTALL_BINDIR}
-  )
-  
-  install(
-    EXPORT ${_plugin_name}Targets
-    FILE ${_plugin_name}Targets.cmake
-    NAMESPACE ${_name}::
-    DESTINATION ${CMAKE_INSTALL_LIBDIR}/cmake/${_name}
-  )
-  install(DIRECTORY ${_plugin_name} DESTINATION ${CMAKE_INSTALL_BINDIR})
-endfunction()
@@ -1,39 +0,0 @@
-include_guard(DIRECTORY)
-include(GNUInstallDirs)
-include(${CMAKE_SOURCE_DIR}/cmake/PackBasic.cmake)
-include(${CMAKE_SOURCE_DIR}/cmake/SetFirstDefined.cmake)
-
-function(install_basic)
-  set(_options)
-  set(_one_value_args NAME INCLUDE_DIR)
-  set(_multi_value_args)
-  cmake_parse_arguments(_ARGS "${_options}" "${_one_value_args}" "${_multi_value_args}" ${ARGN})
-  set_first_defined(_name _ARGS_NAME PROJECT_NAME)
-  set_first_defined(_include_dir _ARGS_INCLUDE_DIR _name)
-
-  file(
-    GLOB_RECURSE ${_name}_HEADERS
-    RELATIVE ${CMAKE_CURRENT_SOURCE_DIR}
-    ${_include_dir}/*.h)
-
-  target_sources(${_name}
-    PUBLIC
-      FILE_SET HEADERS FILES ${${_name}_HEADERS})
-
-  install(
-    TARGETS ${_name}
-    EXPORT ${_name}Targets
-    RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR}
-    LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR}
-    ARCHIVE DESTINATION ${CMAKE_INSTALL_LIBDIR}
-    FILE_SET HEADERS DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}
-  )
-
-  install(
-    EXPORT ${_name}Targets
-    FILE ${_name}Targets.cmake
-    NAMESPACE ${_name}::
-    DESTINATION ${CMAKE_INSTALL_LIBDIR}/cmake/${_name})
-
-  pack_basic()
-endfunction()
@@ -1,44 +0,0 @@
-include_guard(DIRECTORY)
-include(CMakePackageConfigHelpers)
-include(${CMAKE_SOURCE_DIR}/cmake/SetFirstDefined.cmake)
-
-function(pack_basic)
-  set(_options)
-  set(_one_value_args NAME)
-  set(_multi_value_args)
-  cmake_parse_arguments(_ARGS "${_options}" "${_one_value_args}" "${_multi_value_args}" ${ARGN})
-  set_first_defined(_name _ARGS_NAME PROJECT_NAME)
-
-  find_package(Git QUIET)
-
-  if(GIT_FOUND)
-    execute_process(
-      COMMAND ${GIT_EXECUTABLE} describe --tags --always --dirty
-      WORKING_DIRECTORY ${CMAKE_SOURCE_DIR}
-      OUTPUT_VARIABLE PROJECT_VERSION
-      OUTPUT_STRIP_TRAILING_WHITESPACE
-    )
-  endif()
-
-  if(NOT GIT_FOUND OR PROJECT_VERSION STREQUAL "")
-    set(PROJECT_VERSION "0.0.0")
-  endif()
-
-  write_basic_package_version_file(
-    "${CMAKE_CURRENT_BINARY_DIR}/${_name}ConfigVersion.cmake"
-    VERSION ${PROJECT_VERSION}
-    COMPATIBILITY AnyNewerVersion
-  )
-
-  configure_package_config_file(
-    "${CMAKE_CURRENT_SOURCE_DIR}/Config.cmake.in"
-    "${CMAKE_CURRENT_BINARY_DIR}/${_name}Config.cmake"
-    INSTALL_DESTINATION ${CMAKE_INSTALL_LIBDIR}/cmake/${_name}
-  )
-
-  install(
-    FILES
-      "${CMAKE_CURRENT_BINARY_DIR}/${_name}Config.cmake"
-      "${CMAKE_CURRENT_BINARY_DIR}/${_name}ConfigVersion.cmake"
-    DESTINATION ${CMAKE_INSTALL_LIBDIR}/cmake/${_name})
-endfunction()
@@ -1,23 +1,12 @@
-set(Protobuf_PROTOC_EXECUTABLE "${EXTERNALS_BIN}/protoc")
-set(PROTOBUF_INCLUDE_DIRS "${EXTERNALS_INCLUDE}")
+set(Protobuf_PROTOC_EXECUTABLE "${SPARK_BIN}/protoc")
+set(PROTOBUF_INCLUDE_DIRS "${SPARK_INCLUDE}")
 set(PROTOBUF_STATIC_LIB "libprotobuf")
-set(_protobuf_src_dir ${CMAKE_SOURCE_DIR}/src/lib_spark)
-set(_protobuf_bin_dir ${CMAKE_BINARY_DIR}/src/lib_spark)

-function(Protobuf_Generate_Cpp)
-  set(_options)
-  set(_one_value_args CXX_SOURCES CXX_HEADERS CXX_OUT_DIR)
-  set(_multi_value_args PROTO_INCLUDE_DIRS PROTO_SOURCES)
-  cmake_parse_arguments(_ARGS "${_options}" "${_one_value_args}" "${_multi_value_args}" ${ARGN})
-  set(${_ARGS_CXX_SOURCES})
-  set(${_ARGS_CXX_HEADERS})
+function(Protobuf_Generate_Cpp SRCS HDRS)
+  set(${SRCS})
+  set(${HDRS})

-  set(_proto_include_cli_args)
-  foreach(_dir ${_ARGS_PROTO_INCLUDE_DIRS})
-    list(APPEND _proto_include_cli_args -I ${_dir})
-  endforeach()
-
-  foreach(FIL ${_ARGS_PROTO_SOURCES})
+  foreach(FIL ${ARGN})
    get_filename_component(ABS_FIL ${FIL} ABSOLUTE)
    get_filename_component(FIL_WE ${FIL} NAME_WE)
    get_filename_component(FIL_DIR ${FIL} DIRECTORY)
@@ -25,63 +14,48 @@ function(Protobuf_Generate_Cpp)
      set(FIL_WE "${FIL_DIR}/${FIL_WE}")
    endif()

-    list(APPEND ${_ARGS_CXX_SOURCES} "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.pb.cc")
-    list(APPEND ${_ARGS_CXX_HEADERS} "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.pb.h")
+    list(APPEND ${SRCS} "${CMAKE_CURRENT_BINARY_DIR}/${FIL_WE}.pb.cc")
+    list(APPEND ${HDRS} "${CMAKE_CURRENT_BINARY_DIR}/${FIL_WE}.pb.h")

    add_custom_command(
-      OUTPUT
-        "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.pb.cc"
-        "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.pb.h"
+      OUTPUT "${CMAKE_CURRENT_BINARY_DIR}/${FIL_WE}.pb.cc"
+             "${CMAKE_CURRENT_BINARY_DIR}/${FIL_WE}.pb.h"
      COMMAND
-        ${Protobuf_PROTOC_EXECUTABLE}
-      ARGS
-        --cpp_out=${_ARGS_CXX_OUT_DIR} ${_proto_include_cli_args} ${ABS_FIL}
+        ${Protobuf_PROTOC_EXECUTABLE} ARGS --cpp_out=${CMAKE_BINARY_DIR}/src -I
+        ${PROTOBUF_INCLUDE_DIRS} -I ${CMAKE_SOURCE_DIR}/src ${ABS_FIL}
      DEPENDS ${ABS_FIL}
      COMMENT "Running C++ protocol buffer compiler on ${FIL}"
-      VERBATIM
-    )
+      VERBATIM)
  endforeach()

-  set_source_files_properties(
-    ${${_ARGS_CXX_SOURCES}}
-    ${${_ARGS_CXX_HEADERS}}
-    PROPERTIES
-      GENERATED TRUE
-  )
+  set_source_files_properties(${${SRCS}} ${${HDRS}} PROPERTIES GENERATED TRUE)

  # remove some warnings from protobuf generated files
  if(WIN32)
-    set_source_files_properties(
-      ${${_ARGS_CXX_SOURCES}}
-      PROPERTIES
-        COMPILE_FLAGS "/wd4309 /wd4838"
-    )
+    set_source_files_properties(${${SRCS}} PROPERTIES COMPILE_FLAGS
+                                                      "/wd4309 /wd4838")
  else()
    set_source_files_properties(
-      ${${_ARGS_CXX_SOURCES}}
+      ${${SRCS}}
      PROPERTIES
-        COMPILE_FLAGS "-Wno-unused-parameter -Wno-effc++ -Wno-pedantic -Wno-sign-compare -Wno-uninitialized"
+        COMPILE_FLAGS
+        "-Wno-unused-parameter -Wno-effc++ -Wno-pedantic -Wno-sign-compare -Wno-uninitialized"
    )
  endif()

-  set(${_ARGS_CXX_SOURCES} ${${_ARGS_CXX_SOURCES}} PARENT_SCOPE)
-  set(${_ARGS_CXX_HEADERS} ${${_ARGS_CXX_HEADERS}} PARENT_SCOPE)
+  set(${SRCS}
+      ${${SRCS}}
+      PARENT_SCOPE)
+  set(${HDRS}
+      ${${HDRS}}
+      PARENT_SCOPE)
 endfunction()

-function(Service_Compiler_Generate_Cpp)
-  set(_options)
-  set(_one_value_args CXX_SOURCES CXX_HEADERS CXX_OUT_DIR)
-  set(_multi_value_args PROTO_INCLUDE_DIRS PROTO_SOURCES)
-  cmake_parse_arguments(_ARGS "${_options}" "${_one_value_args}" "${_multi_value_args}" ${ARGN})
-  set(${_ARGS_CXX_SOURCES})
-  set(${_ARGS_CXX_HEADERS})
+function(Service_Compiler_Generate_Cpp SRCS HDRS)
+  set(${SRCS})
+  set(${HDRS})

-  set(_proto_include_cli_args)
-  foreach(_dir ${_ARGS_PROTO_INCLUDE_DIRS})
-    list(APPEND _proto_include_cli_args -I ${_dir})
-  endforeach()
-
-  foreach(FIL ${_ARGS_PROTO_SOURCES})
+  foreach(FIL ${ARGN})
    get_filename_component(ABS_FIL ${FIL} ABSOLUTE)
    get_filename_component(FIL_WE ${FIL} NAME_WE)
    get_filename_component(FIL_DIR ${FIL} DIRECTORY)
@@ -89,94 +63,83 @@ function(Service_Compiler_Generate_Cpp)
      set(FIL_WE "${FIL_DIR}/${FIL_WE}")
    endif()

-    list(APPEND ${_ARGS_CXX_SOURCES} "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.pb.cc")
-    list(APPEND ${_ARGS_CXX_HEADERS} "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.pb.h")
+    list(APPEND ${SRCS} "${CMAKE_CURRENT_BINARY_DIR}/${FIL_WE}.pb.cc")
+    list(APPEND ${HDRS} "${CMAKE_CURRENT_BINARY_DIR}/${FIL_WE}.pb.h")

-    list(APPEND ${_ARGS_CXX_SOURCES} "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.client.pb.cc")
-    list(APPEND ${_ARGS_CXX_HEADERS} "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.client.pb.h")
+    list(APPEND ${SRCS} "${CMAKE_CURRENT_BINARY_DIR}/${FIL_WE}.client.pb.cc")
+    list(APPEND ${HDRS} "${CMAKE_CURRENT_BINARY_DIR}/${FIL_WE}.client.pb.h")

-    list(APPEND ${_ARGS_CXX_SOURCES} "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.server.pb.cc")
-    list(APPEND ${_ARGS_CXX_HEADERS} "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.server.pb.h")
+    list(APPEND ${SRCS} "${CMAKE_CURRENT_BINARY_DIR}/${FIL_WE}.server.pb.cc")
+    list(APPEND ${HDRS} "${CMAKE_CURRENT_BINARY_DIR}/${FIL_WE}.server.pb.h")

    set(TEMPLATES
-      ${CMAKE_SOURCE_DIR}/src/lib_spark/spark/api/service_compiler/templates/cpp/client.pb.cc
-      ${CMAKE_SOURCE_DIR}/src/lib_spark/spark/api/service_compiler/templates/cpp/client.pb.h
-      ${CMAKE_SOURCE_DIR}/src/lib_spark/spark/api/service_compiler/templates/cpp/server.pb.cc
-      ${CMAKE_SOURCE_DIR}/src/lib_spark/spark/api/service_compiler/templates/cpp/server.pb.h
+        ${CMAKE_CURRENT_SOURCE_DIR}/api/service_compiler/templates/cpp/client.pb.cc
+        ${CMAKE_CURRENT_SOURCE_DIR}/api/service_compiler/templates/cpp/client.pb.h
+        ${CMAKE_CURRENT_SOURCE_DIR}/api/service_compiler/templates/cpp/server.pb.cc
+        ${CMAKE_CURRENT_SOURCE_DIR}/api/service_compiler/templates/cpp/server.pb.h
    )

-    set(_plugin_py ${CMAKE_SOURCE_DIR}/src/lib_spark/spark/api/service_compiler/plugin.py)
    if(WIN32)
-      set(_plugin_cmd ${CMAKE_SOURCE_DIR}/src/lib_spark/spark/api/service_compiler/plugin.cmd)
      add_custom_command(
-        OUTPUT
-          "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.pb.cc"
-          "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.pb.h"
-          "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.client.pb.cc"
-          "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.client.pb.h"
-          "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.server.pb.cc"
-          "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.server.pb.h"
+        OUTPUT "${CMAKE_CURRENT_BINARY_DIR}/${FIL_WE}.pb.cc"
+               "${CMAKE_CURRENT_BINARY_DIR}/${FIL_WE}.pb.h"
+               "${CMAKE_CURRENT_BINARY_DIR}/${FIL_WE}.client.pb.cc"
+               "${CMAKE_CURRENT_BINARY_DIR}/${FIL_WE}.client.pb.h"
+               "${CMAKE_CURRENT_BINARY_DIR}/${FIL_WE}.server.pb.cc"
+               "${CMAKE_CURRENT_BINARY_DIR}/${FIL_WE}.server.pb.h"
        COMMAND SET PYTHONPATH=${SPARK_PYTHON}
        COMMAND ${CMAKE_SOURCE_DIR}/.venv/Scripts/activate.bat
        COMMAND
-          ${Protobuf_PROTOC_EXECUTABLE}
-          ARGS
-            --cpp_out=${_ARGS_CXX_OUT_DIR}
-            --plugin=protoc-gen-service=${_plugin_cmd}
-            --service_out=lang=cpp:${_ARGS_CXX_OUT_DIR}
-            -I ${PROTOBUF_INCLUDE_DIRS}
-            -I ${_protobuf_src_dir}
-            ${ABS_FIL}
-        DEPENDS
-          ${ABS_FIL}
-          ${TEMPLATES}
-          ${_plugin_cmd}
-          ${_plugin_py}
+          ${Protobuf_PROTOC_EXECUTABLE} ARGS --cpp_out=${CMAKE_BINARY_DIR}/src
+          --plugin=protoc-gen-service=${CMAKE_SOURCE_DIR}/src/lib/api/service_compiler/plugin.cmd
+          --service_out=lang=cpp:${CMAKE_BINARY_DIR}/src -I
+          ${PROTOBUF_INCLUDE_DIRS} -I ${CMAKE_SOURCE_DIR}/src ${ABS_FIL}
+        DEPENDS ${ABS_FIL} ${TEMPLATES}
+                ${CMAKE_CURRENT_SOURCE_DIR}/api/service_compiler/plugin.py
        COMMENT "Running C++ service compiler on ${FIL}"
-        VERBATIM
-      )
+        VERBATIM)
    else()
      add_custom_command(
-        OUTPUT
-          "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.pb.cc"
-          "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.pb.h"
-          "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.client.pb.cc"
-          "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.client.pb.h"
-          "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.server.pb.cc"
-          "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.server.pb.h"
+        OUTPUT "${CMAKE_CURRENT_BINARY_DIR}/${FIL_WE}.pb.cc"
+               "${CMAKE_CURRENT_BINARY_DIR}/${FIL_WE}.pb.h"
+               "${CMAKE_CURRENT_BINARY_DIR}/${FIL_WE}.client.pb.cc"
+               "${CMAKE_CURRENT_BINARY_DIR}/${FIL_WE}.client.pb.h"
+               "${CMAKE_CURRENT_BINARY_DIR}/${FIL_WE}.server.pb.cc"
+               "${CMAKE_CURRENT_BINARY_DIR}/${FIL_WE}.server.pb.h"
        COMMAND ${CMAKE_SOURCE_DIR}/tools/venv_exec.sh
        ARGS
          ${Protobuf_PROTOC_EXECUTABLE}
-          --cpp_out=${_ARGS_CXX_OUT_DIR}
-          --plugin=protoc-gen-service=${_plugin_py}
-          --service_out=lang=cpp,make_file_name=${ABS_FIL}:${_ARGS_CXX_OUT_DIR}
-          ${_proto_include_cli_args} ${ABS_FIL}
-        DEPENDS
-          ${ABS_FIL}
-          ${TEMPLATES}
-          ${_plugin_py}
+          --cpp_out=${CMAKE_BINARY_DIR}/src
+          --plugin=protoc-gen-service=${CMAKE_SOURCE_DIR}/src/lib/api/service_compiler/plugin.py
+          --service_out=lang=cpp,make_file_name=${ABS_FIL}:${CMAKE_BINARY_DIR}/src
+          -I ${PROTOBUF_INCLUDE_DIRS} -I ${CMAKE_SOURCE_DIR}/src ${ABS_FIL}
+        DEPENDS ${ABS_FIL} ${TEMPLATES}
+                ${CMAKE_CURRENT_SOURCE_DIR}/api/service_compiler/plugin.py
        COMMENT "Running C++ service compiler on ${FIL}"
-        VERBATIM
-      )
+        VERBATIM)
    endif()
+
  endforeach()

  # remove some warnings from protobuf generated files
  if(WIN32)
-    set_source_files_properties(
-      ${${_ARGS_CXX_SOURCES}}
-      PROPERTIES
-        COMPILE_FLAGS "/wd4309 /wd4838"
-    )
+    set_source_files_properties(${${SRCS}} PROPERTIES COMPILE_FLAGS
+                                                      "/wd4309 /wd4838")
  else()
    set_source_files_properties(
-      ${${_ARGS_CXX_SOURCES}}
+      ${${SRCS}}
      PROPERTIES
-        COMPILE_FLAGS "-Wno-unused-parameter -Wno-effc++ -Wno-pedantic -Wno-sign-compare -Wno-uninitialized"
+        COMPILE_FLAGS
+        "-Wno-unused-parameter -Wno-effc++ -Wno-pedantic -Wno-sign-compare -Wno-uninitialized"
    )
  endif()

-  set(${_ARGS_CXX_SOURCES} ${${_ARGS_CXX_SOURCES}} PARENT_SCOPE)
-  set(${_ARGS_CXX_HEADERS} ${${_ARGS_CXX_HEADERS}} PARENT_SCOPE)
+  set(${SRCS}
+      ${${SRCS}}
+      PARENT_SCOPE)
+  set(${HDRS}
+      ${${HDRS}}
+      PARENT_SCOPE)
+
 endfunction()

@@ -1,83 +0,0 @@
-include_guard(GLOBAL)
-
-function(protobuf_generate_cpp)
-  set(_options)
-  set(_one_value_args
-    CXX_SOURCES
-    CXX_HEADERS
-    CXX_OUT_DIR
-    PROTOC
-  )
-  set(_multi_value_args
-    PROTO_INCLUDE_DIRS
-    PROTO_SOURCES
-  )
-  cmake_parse_arguments(_ARGS "${_options}" "${_one_value_args}" "${_multi_value_args}" ${ARGN})
-
-  if (NOT DEFINED _ARGS_PROTOC)
-    if (WIN32)
-      set(_ARGS_PROTOC "${EXTERNALS_BIN}/protoc.exe")
-    else()
-      set(_ARGS_PROTOC "${EXTERNALS_BIN}/protoc")
-    endif()
-  endif()
-  set(${_ARGS_CXX_SOURCES})
-  set(${_ARGS_CXX_HEADERS})
-
-  set(_proto_include_cli_args)
-  foreach(_dir ${_ARGS_PROTO_INCLUDE_DIRS})
-    list(APPEND _proto_include_cli_args -I ${_dir})
-  endforeach()
-
-  foreach(_fil ${_ARGS_PROTO_SOURCES})
-    get_filename_component(_abs_fil ${_fil} ABSOLUTE)
-    get_filename_component(_fil_we ${_fil} NAME_WE)
-    get_filename_component(_fil_dir ${_fil} DIRECTORY)
-    if(_fil_dir)
-      set(_fil_we "${_fil_dir}/${_fil_we}")
-    endif()
-    list(APPEND ${_ARGS_CXX_SOURCES} "${_ARGS_CXX_OUT_DIR}/${_fil_we}.pb.cc")
-    list(APPEND ${_ARGS_CXX_HEADERS} "${_ARGS_CXX_OUT_DIR}/${_fil_we}.pb.h")
-    message(STATUS "add_custom_command for ${_fil} -> ${_ARGS_CXX_OUT_DIR}/${_fil_we}.pb.h")
-    # TODO: Pass proto sources by relative path (so this fn can be called from
-    # arbitrary subdirs).
-    add_custom_command(
-      OUTPUT
-        "${_ARGS_CXX_OUT_DIR}/${_fil_we}.pb.cc"
-        "${_ARGS_CXX_OUT_DIR}/${_fil_we}.pb.h"
-      COMMAND
-        ${_ARGS_PROTOC}
-      ARGS
-        --cpp_out=${_ARGS_CXX_OUT_DIR} ${_proto_include_cli_args} ${_abs_fil}
-      DEPENDS ${_abs_fil}
-      COMMENT "Running C++ protocol buffer compiler on ${_fil}"
-      VERBATIM
-    )
-  endforeach()
-
-  set_source_files_properties(
-    ${${_ARGS_CXX_SOURCES}}
-    ${${_ARGS_CXX_HEADERS}}
-    PROPERTIES
-      GENERATED TRUE
-  )
-
-  # remove some warnings from protobuf generated files
-  if(WIN32)
-    set_source_files_properties(
-      ${${_ARGS_CXX_SOURCES}}
-      PROPERTIES
-        COMPILE_FLAGS "/wd4309 /wd4838"
-    )
-  else()
-    set_source_files_properties(
-      ${${_ARGS_CXX_SOURCES}}
-      PROPERTIES
-        COMPILE_FLAGS "-Wno-unused-parameter -Wno-effc++ -Wno-pedantic -Wno-sign-compare -Wno-uninitialized"
-    )
-  endif()
-
-  set(${_ARGS_CXX_SOURCES} ${${_ARGS_CXX_SOURCES}} PARENT_SCOPE)
-  set(${_ARGS_CXX_HEADERS} ${${_ARGS_CXX_HEADERS}} PARENT_SCOPE)
-endfunction()
-
@@ -1,61 +0,0 @@
-include_guard(GLOBAL)
-
-function(protobuf_generate_cpp_config)
-  set(_options)
-  set(_one_value_args
-    PROTO_SOURCES_HEADER
-    PROTO_DEFAULTS_HEADER
-  )
-  set(_multi_value_args
-    PROTO_SOURCES
-    PROTO_DEFAULTS
-  )
-  cmake_parse_arguments(_ARGS "${_options}" "${_one_value_args}" "${_multi_value_args}" ${ARGN})
-
-  # Configure the 'config_proto_files' header. It provides an array of strings populated with the
-  # files in PROTO_SOURCES. At runtime, we use this array to generate a special 'config' proto
-  # message with fields for each message defined in said files. A global instance of this 'config'
-  # proto is then constructed, and populated with [compile-time] defaults, and [runtime] overrides
-  # (i.e., in spark.conf). Calls to `bts::config::get<T>()` retrieve the value from the field
-  # associated with type `T` in this global instance.
-  message(STATUS "PROTO_SOURCES: ${_ARGS_PROTO_SOURCES}")
-  message(STATUS "PROTO_SOURCES_HEADER: ${_ARGS_PROTO_SOURCES_HEADER}")
-  message(STATUS "PROTO_DEFAULTS_HEADER: ${_ARGS_PROTO_DEFAULTS_HEADER}")
-  if (_ARGS_PROTO_SOURCES_HEADER)
-    set(PROTO_SOURCES)
-    set(PROTO_SOURCES_CXX_ARRAY_ITEMS "")
-    foreach(_proto_path ${_ARGS_PROTO_SOURCES})
-      list(APPEND PROTO_SOURCES ${_proto_path})
-      string(APPEND PROTO_SOURCES_CXX_ARRAY_ITEMS "\"${_proto_path}\",\n")
-    endforeach()
-    list(LENGTH PROTO_SOURCES PROTO_SOURCES_LENGTH)
-    message(STATUS "CONFIGURING PROTO_SOURCES_HEADER): ${_ARGS_PROTO_SOURCES_HEADER}")
-    configure_file(${CMAKE_SOURCE_DIR}/cmake/config_proto_files.h.in ${_ARGS_PROTO_SOURCES_HEADER})
-  endif()
-
-  # Generate static defaults. Ideally, this would be a single string, but we use an array because
-  # the compiler complains if the string is longer than 65k.
-  set(_cfg_defaults_str "inline const char * config_static_defaults[] = { ")
-
-  foreach(_cfg_file_path ${_ARGS_PROTO_DEFAULTS})
-    file(READ ${_cfg_file_path} _cfg_file_content)
-    string(REPLACE "\"" "\\\"" _cfg_file_content_escaped ${_cfg_file_content})
-    string(REPLACE "\n" ";" _cfg_lines ${_cfg_file_content_escaped})
-
-    foreach(_line ${_cfg_lines})
-      string(APPEND _cfg_defaults_str "\n  \"${_line}\",")
-    endforeach(_line)
-  endforeach(_cfg_file_path)
-
-  string(APPEND _cfg_defaults_str "};")
-  set(_old_cfg_defaults_str "")
-
-  if(EXISTS ${_ARGS_PROTO_DEFAULTS_HEADER})
-    file(READ ${_ARGS_PROTO_DEFAULTS_HEADER} _old_cfg_defaults_str)
-  endif()
-
-  if(NOT _cfg_defaults_str STREQUAL _old_cfg_defaults_str)
-    file(WRITE "${_ARGS_PROTO_DEFAULTS_HEADER}" "${_cfg_defaults_str}")
-  endif()
-endfunction()
-
@@ -1,49 +0,0 @@
-include_guard(DIRECTORY)
-
-function(protobuf_generate_fs)
-  set(_options)
-  set(_one_value_args TARGET_NAME OUT_DIR PROTOC)
-  set(_multi_value_args PROTO_INCLUDE_DIRS PROTO_SOURCES)
-  cmake_parse_arguments(_ARGS "${_options}" "${_one_value_args}" "${_multi_value_args}" ${ARGN})
-
-  if (NOT DEFINED _ARGS_TARGET_NAME)
-    set(_ARGS_TARGET_NAME "${PROJECT_NAME}_generate_fs_protos")
-  endif()
-  if (NOT DEFINED _ARGS_PROTOC)
-    set(_ARGS_PROTOC "${EXTERNALS_BIN}/protoc")
-  endif()
-
-  set(_proto_include_cli_args)
-  foreach(_dir ${_ARGS_PROTO_INCLUDE_DIRS})
-    list(APPEND _proto_include_cli_args -I ${_dir})
-  endforeach()
-
-  set(_outputs)
-  foreach(_proto_source ${_ARGS_PROTO_SOURCES})
-    # get_filename_component(_abs_proto_source ${_proto_source} ABSOLUTE)
-    get_filename_component(_proto_source_we ${_proto_source} NAME_WE)
-    # get_filename_component(_proto_source_dir ${_proto_source} DIRECTORY)
-    list(APPEND _outputs ${_ARGS_OUT_DIR}/${_proto_source_we}.fable.fs)
-  endforeach()
-
-  set(_venv_exec ${CMAKE_SOURCE_DIR}/tools/venv_exec.sh)
-  set(_plugin_py ${CMAKE_SOURCE_DIR}/src/lib_spark/spark/api/service_compiler/plugin.py)
-  set(_template_fs ${CMAKE_SOURCE_DIR}/src/lib_spark/spark/api/service_compiler/templates/fs/fable.fs)
-  add_custom_command(
-    OUTPUT ${_outputs}
-    WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}
-    COMMAND ${_venv_exec}
-      ${_ARGS_PROTOC}
-      --plugin=protoc-gen-service=${_plugin_py}
-      --service_out=lang=fs,strip_out_path=1:${_ARGS_OUT_DIR}
-      ${_proto_include_cli_args}
-      ${_ARGS_PROTO_SOURCES}
-    DEPENDS
-      ${_venv_exec}
-      ${_plugin_py}
-      ${_template_fs}
-      ${CMAKE_CURRENT_SOURCE_DIR}/package-lock.json
-      ${_ARGS_PROTO_SOURCES}
-  )
-  add_custom_target(${_ARGS_TARGET_NAME} ALL DEPENDS "${_outputs}")
-endfunction()
@@ -1,33 +0,0 @@
-include_guard(DIRECTORY)
-
-function(protobuf_generate_js)
-  set(_options)
-  set(_one_value_args TARGET_NAME OUTPUT)
-  set(_multi_value_args PROTO_INCLUDE_DIRS PROTO_SOURCES)
-  cmake_parse_arguments(_ARGS "${_options}" "${_one_value_args}" "${_multi_value_args}" ${ARGN})
-
-  if (NOT DEFINED _ARGS_TARGET_NAME)
-    set(_ARGS_TARGET_NAME "${PROJECT_NAME}_generate_js_protos")
-  endif()
-
-  set(_proto_include_cli_args)
-  foreach(_dir ${_ARGS_PROTO_INCLUDE_DIRS})
-    list(APPEND _proto_include_cli_args -p ${_dir})
-  endforeach()
-
-  add_custom_command(
-    OUTPUT ${_ARGS_OUTPUT}
-    WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}
-    COMMAND npx
-      --no-install pbjs
-      -t json-module
-      -w commonjs
-      -o ${_ARGS_OUTPUT}
-      ${_proto_include_cli_args}
-      ${_ARGS_PROTO_SOURCES}
-    DEPENDS
-      ${CMAKE_CURRENT_SOURCE_DIR}/package-lock.json
-      ${_ARGS_PROTO_SOURCES}
-  )
-  add_custom_target(${_ARGS_TARGET_NAME} ALL DEPENDS ${_ARGS_OUTPUT})
-endfunction()
@@ -1,10 +0,0 @@
-include_guard(GLOBAL)
-
-macro(set_first_defined RESULT)
-  foreach(_arg ${ARGN})
-    if (DEFINED ${_arg})
-      set(${RESULT} ${${_arg}})
-      break()
-    endif()
-  endforeach()
-endmacro()
@@ -1,49 +0,0 @@
-include_guard(DIRECTORY)
-include(${CMAKE_SOURCE_DIR}/cmake/TernaryOperator.cmake)
-
-function(sym_link_dir)
-  set(_options)
-  set(_one_value_args TARGET_NAME SOURCE DEST COMMENT)
-  set(_multi_value_args)
-  cmake_parse_arguments(_ARGS "${_options}" "${_one_value_args}" "${_multi_value_args}" ${ARGN})
-  add_custom_command(
-    OUTPUT ${_ARGS_DEST}
-    COMMAND ln ARGS -sf ${_ARGS_SOURCE} ${_ARGS_DEST}
-  )
-  set_ternary(_comment _ARGS_COMMENT "${_ARGS_COMMENT}" "linking dir ${_ARGS_SOURCE} -> ${_ARGS_DEST}")
-  add_custom_target(
-    ${_ARGS_TARGET_NAME} ALL
-    DEPENDS ${_ARGS_DEST}
-    COMMENT "${_comment}"
-  )
-endfunction()
-
-function(sym_link_file)
-  set(_options)
-  set(_one_value_args SOURCE DEST INSTALL_DEST)
-  set(_multi_value_args)
-  cmake_parse_arguments(_ARGS "${_options}" "${_one_value_args}" "${_multi_value_args}" ${ARGN})
-  file(
-    CREATE_LINK
-      ${_ARGS_SOURCE}
-      ${_ARGS_DEST}
-    SYMBOLIC
-  )
-  if (DEFINED _ARGS_INSTALL_DEST)
-    install(
-      FILES
-      ${_ARGS_SOURCE}
-      DESTINATION ${_ARGS_INSTALL_DEST}
-    )
-  endif()
-endfunction()
-
-function(sym_link_local_lib LIB_NAME)
-  set(_lib_file_name "${CMAKE_SHARED_LIBRARY_PREFIX}${LIB_NAME}.so")
-  file(
-    CREATE_LINK
-      "${CMAKE_BINARY_DIR}/src/lib_${LIB_NAME}/${_lib_file_name}"
-      "${CMAKE_CURRENT_BINARY_DIR}/${_lib_file_name}"
-    SYMBOLIC
-  )
-endfunction()
@@ -1,39 +0,0 @@
-include_guard(DIRECTORY)
-include(${CMAKE_SOURCE_DIR}/cmake/SymLink.cmake)
-
-function(sym_link_python_module)
-  set(_options)
-  set(_one_value_args LIB_NAME MODULE_NAME PLUGIN_NAME)
-  set(_multi_value_args)
-  cmake_parse_arguments(_ARGS "${_options}" "${_one_value_args}"
-                        "${_multi_value_args}" ${ARGN})
-  if(NOT DEFINED _ARGS_LIB_NAME)
-    message(
-      FATAL_ERROR "LIB_NAME must be specified for sym_link_python_module()")
-  endif()
-  set(_lib_name ${_ARGS_LIB_NAME})
-  set(_module_name ${_ARGS_MODULE_NAME})
-  if(NOT DEFINED _ARGS_MODULE_NAME)
-    set(_module_name ${_lib_name}_module)
-  endif()
-  set(_plugin_name ${_ARGS_PLUGIN_NAME})
-  set(_deps ${_lib_name} ${_module_name})
-  if(DEFINED _ARGS_PLUGIN_NAME)
-    list(APPEND _deps ${_plugin_name} copy_${_plugin_name})
-  endif()
-  add_dependencies(${PROJECT_NAME} ${_deps})
-  sym_link_local_lib(${_lib_name})
-  set(_module_suffix cpython-310-x86_64-linux-gnu.so)
-  sym_link_file(
-    SOURCE
-    ${CMAKE_BINARY_DIR}/src/lib_${_lib_name}/${_module_name}.${_module_suffix}
-    DEST
-    ${CMAKE_CURRENT_BINARY_DIR}/${_module_name}.${_module_suffix}
-    INSTALL_DEST
-    ${CMAKE_INSTALL_BINDIR}/)
-  if(DEFINED _ARGS_PLUGIN_NAME)
-    get_target_property(plugins ${_plugin_name} INTERFACE_PYTHON_SCRIPTS_DIR)
-    sym_link_dir(TARGET_NAME ${PROJECT_NAME}_link_${_plugin_name} SOURCE
-                 ${plugins} DEST ${CMAKE_CURRENT_BINARY_DIR}/${_plugin_name})
-  endif()
-endfunction()
@@ -1,9 +0,0 @@
-include_guard(GLOBAL)
-
-macro(set_ternary RESULT CONDITION TRUE_VALUE FALSE_VALUE)
-  if(${CONDITION})
-    set(${RESULT} ${TRUE_VALUE})
-  else()
-    set(${RESULT} ${FALSE_VALUE})
-  endif()
-endmacro()
@@ -1,11 +0,0 @@
-#pragma once
-
-#include <array>
-
-namespace bts::@PROJECT_NAME@::protobufs {
-
-inline std::array<const char *, @PROTO_SOURCES_LENGTH@> config_proto_files = {
-@PROTO_SOURCES_CXX_ARRAY_ITEMS@
-};
-
-} // namespace bts::@PROJECT_NAME@::protobufs
@@ -1,11 +0,0 @@
-#ifndef BTS_SPARK_VERSION_H
-#define BTS_SPARK_VERSION_H
-
-#define BTS_SPARK_VERSION_MAJOR @PROJECT_VERSION_MAJOR@
-#define BTS_SPARK_VERSION_MINOR @PROJECT_VERSION_MINOR@
-#define BTS_SPARK_VERSION_PATCH @PROJECT_VERSION_PATCH@
-#define BTS_SPARK_VERSION_BUILD_METADATA @PROJECT_VERSION_BUILD_METADATA@
-#define BTS_SPARK_VERSION "@PROJECT_VERSION@"
-#define BTS_SPARK_VERSION_FULL "@PROJECT_VERSION_FULL@"
-
-#endif
@@ -0,0 +1,24 @@
+cmake_minimum_required(VERSION 3.5)
+
+# Explicitly set policy to convert relative paths to absolute (although this is a default policy,
+# setting it explicitly prevents a compile-warning)
+cmake_policy(SET CMP0076 NEW)
+
+# add protos to the library to facilitate source_group
+target_sources(libspark PUBLIC ${Config_Protos} ${NonConfig_Protos} ${Service_Protos})
+
+function(assign_source_group)
+  foreach(_source IN ITEMS ${ARGN})
+    if(IS_ABSOLUTE "${_source}")
+      file(RELATIVE_PATH _source_rel "${CMAKE_CURRENT_SOURCE_DIR}" "${_source}")
+    else()
+      set(_source_rel "${_source}")
+    endif()
+    get_filename_component(_source_path "${_source_rel}" PATH)
+    string(REPLACE "/" "\\" _source_path_msvc "${_source_path}")
+    source_group("${_source_path_msvc}" FILES "${_source}")
+    set(SOURCE_FILES "${_source}")
+  endforeach()
+endfunction(assign_source_group)
+
+assign_source_group(${SparkLib_Sources} ${Config_Protos} ${NonConfig_Protos} ${Service_Protos})
@@ -1,165 +0,0 @@
-services:
-  sparkd:
-    volumes:
-      - ${SPARKD__DATA_DIR:-sparkd_data}:/home/bts/sparkdata/data
-    build:
-      dockerfile: src/app_sparkd/Dockerfile
-      args:
-        - IMAGE=${SPARKD__IMAGE}
-        - SOURCE_DIR=${SPARKD__SOURCE_DIR}
-        - CONFIG_DIR=${SPARKD__CONFIG_DIR}
-        - PYTHON_REQUIREMENTS=${SPARKD__PYTHON_REQUIREMENTS}
-    shm_size: "${SPARKD__SHM_SIZE:-512mb}"
-    command: >
-      sh -c "
-        echo bts | sudo -S chown -R bts:bts /home/bts/sparkdata/data
-        if [ -e "/home/bts/sparkdata/data/logging.sparkd/log" ]; then
-          mv "/home/bts/sparkdata/data/logging.sparkd/log" "/home/bts/sparkdata/data/logging.sparkd/log_$(date '+%Y%m%d-%H%M')"
-        fi
-        if [ -e "/home/bts/sparkdata/conf/.env.local" ]; then
-          echo 'Sourcing /home/bts/sparkdata/conf/.env.local'
-          set -a
-          . /home/bts/sparkdata/conf/.env.local
-          set +a
-        fi
-        if [ -d /home/bts/sparkdata/conf/plugins ]; then
-          echo 'Copying /home/bts/sparkdata/conf/plugins to runtime dir (/home/bts/bin)'
-          cp -r /home/bts/sparkdata/conf/plugins /home/bts/bin/
-        fi
-        ./sparkd run"
-    env_file:
-      - .env
-      - ${SPARKD__CONFIG_DIR}/.env
-    ports:
-      - "${SPARKD__CONFIG_PORT}:${SPARKD__CONFIG_PORT}"
-      - "${SPARKD__QUICKBUS_PORT}:${SPARKD__QUICKBUS_PORT}"
-      - "${SPARKD__SIM_FIX_PORT}:${SPARKD__SIM_FIX_PORT}"
-    deploy:
-      resources:
-        limits:
-          memory: "${SPARKD__MEMORY:-4gb}"
-
-  sparkd-symbols:
-    volumes:
-      - ${SPARKD__DATA_DIR:-sparkd_data}:/home/bts/sparkdata/data
-      - ${SPARKD__RUNTIME_CONF_DIR:-sparkd_conf}:/home/bts/sparkdata/conf
-    build:
-      dockerfile: src/app_sparkd/Dockerfile
-      args:
-        - IMAGE=${SPARKD__IMAGE}
-        - SOURCE_DIR=${SPARKD__SOURCE_DIR}
-        - CONFIG_DIR=${SPARKD__CONFIG_DIR}
-        - PYTHON_REQUIREMENTS=${SPARKD__PYTHON_REQUIREMENTS}
-    shm_size: "${SPARKD__SHM_SIZE:-512mb}"
-    command: >
-      sh -c "
-        echo bts | sudo -S chown -R bts:bts /home/bts/sparkdata
-        if [ -e "/home/bts/sparkdata/data/logging.sparkd/log" ]; then
-          mv "/home/bts/sparkdata/data/logging.sparkd/log" "/home/bts/sparkdata/data/logging.sparkd/log_$(date '+%Y%m%d-%H%M')"
-        fi
-        ls -alh /home/bts/sparkdata/conf/
-        cat /home/bts/sparkdata/conf/.env.local
-        if [ -e "/home/bts/sparkdata/conf/.env.local" ]; then
-          echo 'Sourcing /home/bts/sparkdata/conf/.env.local'
-          set -a
-          . /home/bts/sparkdata/conf/.env.local
-          set +a
-        fi
-        if [ -d /home/bts/sparkdata/conf/plugins ]; then
-          echo 'Copying /home/bts/sparkdata/conf/plugins to runtime dir (/home/bts/bin)'
-          cp -r /home/bts/sparkdata/conf/plugins /home/bts/bin/
-        fi
-        ./sparkd run"
-    env_file:
-      - .env
-      - ${SPARKD__CONFIG_DIR}/.env
-    ports:
-      - "${SPARKD__CONFIG_PORT}:${SPARKD__CONFIG_PORT}"
-      - "${SPARKD__QUICKBUS_PORT}:${SPARKD__QUICKBUS_PORT}"
-      - "${SPARKD__SIM_FIX_PORT}:${SPARKD__SIM_FIX_PORT}"
-    deploy:
-      restart_policy:
-        condition: any
-        delay: 3s
-      resources:
-        limits:
-          memory: "${SPARKD__MEMORY:-4gb}"
-
-  sparkd-tail:
-    image: bluetradingsystems/linux_dotnet7
-    volumes:
-      - type: volume
-        read_only: true
-        source: sparkd_data
-        target: /home/bts/sparkdata/data/logging.sparkd
-        volume:
-          subpath: logging.sparkd
-    command: ["tail", "-f", "/home/bts/sparkdata/data/logging.sparkd/log"]
-
-  sparkd-ping-test:
-    depends_on:
-      - sparkd
-    profiles:
-      - ping-test
-    build:
-      context: .
-      dockerfile_inline: |
-        FROM bluetradingsystems/linux_dotnet7
-        USER "root"
-        RUN apt-get update && apt-get -y install iputils-ping
-    command: [ "ping", "sparkd" ]
-
-  pubsub:
-    environment:
-      - DOTNET_ENVIRONMENT=Staging
-    build:
-      dockerfile: src/DataStreamSuite/PubSubServer/Dockerfile
-      args:
-        - SOURCE_DIR=build/Release/bin/net8.0/linux-x64/publish
-        - CONF_FILE=src/DataStreamSuite/PubSubServer/spark.conf
-    ports:
-      - 22019:22019
-    command: [ "./PubSubServer" ]
-
-  md-publisher:
-    build:
-      dockerfile: src/DataStreamSuite/DataStreamPublisher/Dockerfile
-      args:
-        - SOURCE_DIR=build/Release/bin/net8.0/linux-x64/publish
-        - CONF_FILE=src/DataStreamSuite/DataStreamPublisher/spark.conf
-    depends_on:
-      - pubsub
-      - sparkd
-    command: [ "dockerize", "-wait", "tcp://pubsub:22019", "-wait", "tcp://sparkd:8000", "-timeout", "10s", "./DataStreamPublisher" ]
-    environment:
-      - DOTNET_ENVIRONMENT=MarketData
-
-  webapp:
-    ports:
-      - 8080:8080
-      - 8443:8443
-    build:
-      dockerfile: src/DataStreamSuite/Bts.DataStreamWebApp.Server/Dockerfile
-      args:
-        - SOURCE_DIR=build/Release/bin/net8.0/linux-x64/publish
-        - WEB_ROOT=build/Bts.DataStreamWebApp.Client/production/dist
-        - CONF_FILE=src/DataStreamSuite/Bts.DataStreamWebApp.Server/spark.conf
-    command: [ "./Bts.DataStreamWebApp.Server" ]
-    environment:
-      - ASPNETCORE_ENVIRONMENT=Staging
-
-  api-server:
-    depends_on:
-      - sparkd
-    ports:
-      - 8008:8008
-      - 9443:9443
-    build:
-      dockerfile: src/client/spark_api_server/Dockerfile
-    command: [ "dockerize", "-wait", "tcp://sparkd:8000", "-timeout", "10s", "./spark_api_server", "run" ]
-    environment:
-      - ASPNETCORE_ENVIRONMENT=Staging
-
-volumes:
-  sparkd_data:
-    name: "sparkd_${SPARKD__CONFIG_PORT:-8000}"
@@ -124,8 +124,8 @@ service OrderService {
 ```
 ### C++ Example
 ```cpp
-#include "spark/api/public/ServiceContext.h"
-#include "spark/api/services/OrderService.client.pb.h"
+#include "lib/api/public/ServiceContext.h"
+#include "lib/api/services/OrderService.client.pb.h"
 using namespace bts::api;
 ServiceContext ctx;
 auto orders = ctx.locate<orderman::OrderService>();
@@ -29,9 +29,9 @@ namespace sample {
 ```

 ```cpp
-#include "spark/api/public/ServiceContext.h"
-#include "spark/domain_model/Security.h"
-#include "spark/api/services/MarketDataService.client.pb.h"
+#include "lib/api/public/ServiceContext.h"
+#include "spark/lib/domain_model/Security.h"
+#include "lib/api/services/MarketDataService.client.pb.h"
 #include <thread>

 using namespace bts::api;
@@ -1,38 +0,0 @@
-FROM ubuntu:22.04
-
-ARG DEBIAN_FRONTEND=noninteractive
-
-RUN apt-get update && apt-get install -y pkg-config
-RUN apt-get update && apt-get install -y locales
-RUN apt-get update && apt-get install -y ca-certificates
-RUN apt-get update && apt-get install -y libssh2-1-dev
-RUN apt-get update && apt-get install -y software-properties-common
-RUN apt-get update && apt-get install -y sudo
-
-# GCC
-ARG gcc_major=13
-RUN add-apt-repository ppa:ubuntu-toolchain-r/test \
-  && apt-get update \
-  && apt-get install -y g++-$gcc_major
-
-RUN apt-get update \
-  && apt-get install -y dotnet7
-
-RUN locale-gen en_US.UTF-8
-
-ARG docker_user="bts"
-ARG docker_uid="1000"
-ARG docker_pass="bts"
-
-RUN \
-  groupadd -g "$docker_uid" "$docker_user" \
-  && useradd \
-  -m \
-  -u "$docker_uid" \
-  -g "$docker_uid" \
-  -G sudo \
-  --password "$(openssl passwd -1 "$docker_pass")" \
-  "$docker_user"
-
-USER bts
-WORKDIR /home/bts
@@ -1,39 +0,0 @@
-FROM ubuntu:22.04
-
-ARG DEBIAN_FRONTEND=noninteractive
-
-RUN apt-get update \
-  && apt-get install -y pkg-config \
-  && apt-get install -y locales \
-  && apt-get install -y ca-certificates \
-  && apt-get install -y libssh2-1-dev \
-  && apt-get install -y software-properties-common \
-  && apt-get install -y sudo
-
-# GCC
-ARG gcc_major=13
-RUN add-apt-repository ppa:ubuntu-toolchain-r/test \
-  && apt-get update \
-  && apt-get install -y g++-$gcc_major
-
-RUN apt-get update \
-  && apt-get install -y dotnet7
-
-RUN locale-gen en_US.UTF-8
-
-ARG docker_user="bts"
-ARG docker_uid="1000"
-ARG docker_pass="bts"
-
-RUN \
-  groupadd -g "$docker_uid" "$docker_user" \
-  && useradd \
-  -m \
-  -u "$docker_uid" \
-  -g "$docker_uid" \
-  -G sudo \
-  --password "$(openssl passwd -1 "$docker_pass")" \
-  "$docker_user"
-
-USER bts
-WORKDIR /home/bts
@@ -1,42 +1,22 @@
 astroid==2.7.3
 autopep8==1.5.7
-blinker==1.9.0
-boto3==1.37.9
-botocore==1.37.9
 certifi==2023.5.7
 charset-normalizer==3.1.0
-click==8.1.8
 colorama==0.4.4
-Flask==3.1.0
 idna==3.4
 isort==5.9.3
-itsdangerous==2.2.0
-Jinja2==3.1.6
-jmespath==1.0.1
 lazy-object-proxy==1.6.0
-MarkupSafe==3.0.2
 mccabe==0.6.1
-numpy==2.2.6
 platformdirs==2.3.0
 protobuf==4.21.12
 pybars3==0.9.7
-pybind11==2.13.6
-pybind11-stubgen==2.5.3
-pybit==5.11.0
 pycodestyle==2.7.0
 pycparser==2.20
-pycryptodome==3.23.0
 pylint==2.10.2
 PyMeta3==0.5.1
-python-dateutil==2.9.0.post0
-python-dotenv==1.0.1
 requests==2.31.0
-s3transfer==0.11.4
-six==1.17.0
 termcolor==1.1.0
 toml==0.10.2
 urllib3==2.0.3
-websocket-client==1.8.0
-Werkzeug==3.1.3
 wrapt==1.12.1
 yapf==0.31.0
@@ -18,14 +18,6 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "spark_websocket", "src\clie
 EndProject
 Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "spark_api_server", "src\client\spark_api_server\spark_api_server.csproj", "{2B8C5D01-FF78-4BFA-B332-5E430586DAF9}"
 EndProject
-Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "DataStreamCore", "src\DataStreamSuite\DataStreamCore\DataStreamCore.csproj", "{4E4F491F-C688-43C4-88DA-AC92BC5FA1B8}"
-EndProject
-Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "DataStream", "src\DataStreamSuite\DataStream\DataStream.csproj", "{2AC54ADC-3246-427E-9CA9-FEA2DBCB9432}"
-EndProject
-Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "SharedStreams", "src\DataStreamSuite\SharedStreams\SharedStreams.csproj", "{4E96E5CC-7791-40B3-A8A1-C34FBB736B3B}"
-EndProject
-Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "PubSub", "src\DataStreamSuite\PubSub\PubSub.csproj", "{51C40BA9-F226-4A35-939D-1ED6F5FF7299}"
-EndProject
 Global
 	GlobalSection(SolutionConfigurationPlatforms) = preSolution
 		Debug|x64 = Debug|x64
@@ -98,38 +90,6 @@ Global
 		{2B8C5D01-FF78-4BFA-B332-5E430586DAF9}.Release|x64.Build.0 = Release|x64
 		{2B8C5D01-FF78-4BFA-B332-5E430586DAF9}.RelWithDebInfo|x64.ActiveCfg = Release|x64
 		{2B8C5D01-FF78-4BFA-B332-5E430586DAF9}.RelWithDebInfo|x64.Build.0 = Release|x64
-		{4E4F491F-C688-43C4-88DA-AC92BC5FA1B8}.Debug|x64.ActiveCfg = Debug|Any CPU
-		{4E4F491F-C688-43C4-88DA-AC92BC5FA1B8}.Debug|x64.Build.0 = Debug|Any CPU
-		{4E4F491F-C688-43C4-88DA-AC92BC5FA1B8}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
-		{4E4F491F-C688-43C4-88DA-AC92BC5FA1B8}.MinSizeRel|x64.Build.0 = Debug|Any CPU
-		{4E4F491F-C688-43C4-88DA-AC92BC5FA1B8}.Release|x64.ActiveCfg = Release|Any CPU
-		{4E4F491F-C688-43C4-88DA-AC92BC5FA1B8}.Release|x64.Build.0 = Release|Any CPU
-		{4E4F491F-C688-43C4-88DA-AC92BC5FA1B8}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
-		{4E4F491F-C688-43C4-88DA-AC92BC5FA1B8}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
-		{2AC54ADC-3246-427E-9CA9-FEA2DBCB9432}.Debug|x64.ActiveCfg = Debug|Any CPU
-		{2AC54ADC-3246-427E-9CA9-FEA2DBCB9432}.Debug|x64.Build.0 = Debug|Any CPU
-		{2AC54ADC-3246-427E-9CA9-FEA2DBCB9432}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
-		{2AC54ADC-3246-427E-9CA9-FEA2DBCB9432}.MinSizeRel|x64.Build.0 = Debug|Any CPU
-		{2AC54ADC-3246-427E-9CA9-FEA2DBCB9432}.Release|x64.ActiveCfg = Release|Any CPU
-		{2AC54ADC-3246-427E-9CA9-FEA2DBCB9432}.Release|x64.Build.0 = Release|Any CPU
-		{2AC54ADC-3246-427E-9CA9-FEA2DBCB9432}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
-		{2AC54ADC-3246-427E-9CA9-FEA2DBCB9432}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
-		{4E96E5CC-7791-40B3-A8A1-C34FBB736B3B}.Debug|x64.ActiveCfg = Debug|Any CPU
-		{4E96E5CC-7791-40B3-A8A1-C34FBB736B3B}.Debug|x64.Build.0 = Debug|Any CPU
-		{4E96E5CC-7791-40B3-A8A1-C34FBB736B3B}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
-		{4E96E5CC-7791-40B3-A8A1-C34FBB736B3B}.MinSizeRel|x64.Build.0 = Debug|Any CPU
-		{4E96E5CC-7791-40B3-A8A1-C34FBB736B3B}.Release|x64.ActiveCfg = Release|Any CPU
-		{4E96E5CC-7791-40B3-A8A1-C34FBB736B3B}.Release|x64.Build.0 = Release|Any CPU
-		{4E96E5CC-7791-40B3-A8A1-C34FBB736B3B}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
-		{4E96E5CC-7791-40B3-A8A1-C34FBB736B3B}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
-		{51C40BA9-F226-4A35-939D-1ED6F5FF7299}.Debug|x64.ActiveCfg = Debug|Any CPU
-		{51C40BA9-F226-4A35-939D-1ED6F5FF7299}.Debug|x64.Build.0 = Debug|Any CPU
-		{51C40BA9-F226-4A35-939D-1ED6F5FF7299}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
-		{51C40BA9-F226-4A35-939D-1ED6F5FF7299}.MinSizeRel|x64.Build.0 = Debug|Any CPU
-		{51C40BA9-F226-4A35-939D-1ED6F5FF7299}.Release|x64.ActiveCfg = Release|Any CPU
-		{51C40BA9-F226-4A35-939D-1ED6F5FF7299}.Release|x64.Build.0 = Release|Any CPU
-		{51C40BA9-F226-4A35-939D-1ED6F5FF7299}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
-		{51C40BA9-F226-4A35-939D-1ED6F5FF7299}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
 	EndGlobalSection
 	GlobalSection(SolutionProperties) = preSolution
 		HideSolutionNode = FALSE
@@ -1,7 +1,7 @@
 <Project Sdk="Microsoft.NET.Sdk">

  <PropertyGroup>
-    <TargetFramework>net8.0</TargetFramework>
+    <TargetFramework>net7.0</TargetFramework>
    <ImplicitUsings>enable</ImplicitUsings>
    <Nullable>enable</Nullable>
  </PropertyGroup>
@@ -1,7 +1,7 @@
 <Project Sdk="Microsoft.NET.Sdk">

  <PropertyGroup>
-    <TargetFramework>net8.0</TargetFramework>
+    <TargetFramework>net7.0</TargetFramework>
    <ImplicitUsings>enable</ImplicitUsings>
    <Nullable>enable</Nullable>
  </PropertyGroup>
@@ -1,7 +1,7 @@
 <Project Sdk="Microsoft.NET.Sdk">

  <PropertyGroup>
-    <TargetFramework>net8.0</TargetFramework>
+    <TargetFramework>net7.0</TargetFramework>
    <ImplicitUsings>enable</ImplicitUsings>
    <Nullable>enable</Nullable>
  </PropertyGroup>
@@ -1,11 +1,19 @@
 namespace Bts.BooleanExpressions;

+using bts;
+
 class BooleanExpression : IBooleanExpression {
   readonly Func<IDictionary<string, object?>, bool> evaluate;

   public BooleanExpression(Func<IDictionary<string, object?>, bool> evaluate) =>
      this.evaluate = evaluate;

-   public bool Evaluate(IDictionary<string, object?> env) =>
-      evaluate(env);
+   public bool Evaluate(IDictionary<string, object?> env) {
+      try {
+         return evaluate(env);
+      } catch (Exception ex) {
+         Log.assert(false, $"BooleanExpression.Evaluate failed: {ex}");
+         throw;
+      }
+   }
 }
@@ -47,6 +47,11 @@ IsOfType                 '/'

 %%

+{StringConstant} {
+   yylval.String = yytext.Substring(1, yytext.Length - 2).Replace(@"\'", "'");
+   return (int)Token.STRING_CONSTANT;
+} 
+
 {BooleanConstant} {
   yylval.Boolean = yytext.ToLower() switch {
      "true" => true,
@@ -57,11 +62,6 @@ IsOfType                 '/'
   return (int)Token.BOOLEAN_CONSTANT;
 }

-{StringConstant} {
-   yylval.String = yytext.Substring(1, yytext.Length - 2).Replace(@"\'", "'");
-   return (int)Token.STRING_CONSTANT;
-} 
-
 {NullConstant} {
   yylval.Null = DBNull.Value;
   return (int)Token.NULL_CONSTANT;
@@ -1,14 +1,14 @@
 <Project Sdk="Microsoft.NET.Sdk">

  <PropertyGroup>
-    <TargetFramework>net8.0</TargetFramework>
+    <TargetFramework>net7.0</TargetFramework>
    <ImplicitUsings>enable</ImplicitUsings>
    <Nullable>enable</Nullable>
  </PropertyGroup>

  <ItemGroup>
    <PackageReference Include="Google.Protobuf" Version="3.24.3" />
-    <PackageReference Include="YaccLexTools" Version="1.2.3" />
+    <PackageReference Include="YaccLexTools" Version="1.2.0" />
  </ItemGroup>

  <ItemGroup>
@@ -24,6 +24,8 @@ namespace QUT.GplexBuffers
        public BufferException(string message) : base(message) { }
        public BufferException(string message, Exception innerException)
            : base(message, innerException) { }
+        protected BufferException(SerializationInfo info, StreamingContext context)
+            : base(info, context) { }
    }

    internal abstract class ScanBuff
@@ -58,10 +60,12 @@ namespace QUT.GplexBuffers
            return new BuildBuffer(source);
        }

+#if (!BYTEMODE)
        public static ScanBuff GetBuffer(Stream source, int fallbackCodePage)
        {
            return new BuildBuffer(source, fallbackCodePage);
        }
+#endif // !BYTEMODE
 #endif // !NOFILES
    }

@@ -349,15 +353,17 @@ namespace QUT.GplexBuffers
        {
            FileStream fStrm = (stream as FileStream);
            if (fStrm != null) FileName = fStrm.Name;
-            NextBlk = BlockReaderFactory.Get(stream);
+            NextBlk = BlockReaderFactory.Raw(stream);
        }

+#if (!BYTEMODE)
        public BuildBuffer(Stream stream, int fallbackCodePage)
        {
            FileStream fStrm = (stream as FileStream);
            if (fStrm != null) FileName = fStrm.Name;
            NextBlk = BlockReaderFactory.Get(stream, fallbackCodePage);
        }
+#endif

        /// <summary>
        /// Marks a conservative lower bound for the buffer,
@@ -428,19 +434,43 @@ namespace QUT.GplexBuffers
    //
    internal static class BlockReaderFactory
    {
+        public static BlockReader Raw(Stream stream)
+        {
+            return delegate(char[] block, int index, int number)
+            {
+                byte[] b = new byte[number];
+                int count = stream.Read(b, 0, number);
+                int i = 0;
+                int j = index;
+                for (; i < count; i++, j++)
+                    block[j] = (char)b[i];
+                return count;
+            };
+        }

-        public static BlockReader Get(Stream stream, int fallbackCodePage = -2)
+#if (!BYTEMODE)
+        public static BlockReader Get(Stream stream, int fallbackCodePage)
        {
            Encoding encoding;
            int preamble = Preamble(stream);

            if (preamble != 0)  // There is a valid BOM here!
                encoding = Encoding.GetEncoding(preamble);
-            else if (fallbackCodePage >= 0) // Anything but "guess"
+            else if (fallbackCodePage == -1) // Fallback is "raw" bytes
+                return Raw(stream);
+            else if (fallbackCodePage != -2) // Anything but "guess"
                encoding = Encoding.GetEncoding(fallbackCodePage);
            else // This is the "guess" option
-                encoding = GuessEncoding(stream);
-
+            {
+                int guess = new Guesser(stream).GuessCodePage();
+                stream.Seek(0, SeekOrigin.Begin);
+                if (guess == -1) // ==> this is a 7-bit file
+                    encoding = Encoding.ASCII;
+                else if (guess == 65001)
+                    encoding = Encoding.UTF8;
+                else             // ==> use the machine default
+                    encoding = Encoding.Default;
+            }
            StreamReader reader = new StreamReader(stream, encoding);
            return reader.Read;
        }
@@ -458,14 +488,6 @@ namespace QUT.GplexBuffers
            int b2 = stream.ReadByte();
            if (b0 == 0xef && b1 == 0xbb && b2 == 0xbf)
                return 65001; // UTF8
-
-            int b3 = stream.ReadByte();
-            // Check for UTF-32 BOM (Byte Order Mark)
-            if (b0 == 0x00 && b1 == 0x00 && b2 == 0xFE && b3 == 0xFF)
-                return 12001; // UTF-32BE
-            if (b0 == 0xFF && b1 == 0xFE && b2 == 0x00 && b3 == 0x00)
-                return 12000; // UTF-32LE
-
            //
            // There is no unicode preamble, so we
            // return denoter for the machine default.
@@ -473,150 +495,7 @@ namespace QUT.GplexBuffers
            stream.Seek(0, SeekOrigin.Begin);
            return 0;
        }
-
-        private static Encoding GuessEncoding(Stream stream)
-        {
-            stream.Seek(0, SeekOrigin.Begin);
-            if (IsUtf16(stream, out Encoding encoding))
-            {
-                stream.Seek(0, SeekOrigin.Begin);
-                return encoding;
-            }
-
-            stream.Seek(0, SeekOrigin.Begin);
-            if (IsUtf8(stream))
-            {
-                stream.Seek(0, SeekOrigin.Begin);
-                return Encoding.UTF8;
-            }
-
-            stream.Seek(0, SeekOrigin.Begin);
-            if (IsAscii(stream))
-            {
-                stream.Seek(0, SeekOrigin.Begin);
-                return Encoding.ASCII;
-            }
-
-#if !NET462_OR_GREATER
-            stream.Seek(0, SeekOrigin.Begin);
-            if (IsLatin1(stream))
-            {
-                stream.Seek(0, SeekOrigin.Begin);
-                return Encoding.Latin1;
-            }
-#endif
-            stream.Seek(0, SeekOrigin.Begin);
-            return Encoding.Default;
-        }
-
-        private static bool IsUtf8(Stream stream)
-        {
-            int b;
-            while ((b = stream.ReadByte()) != -1)
-            {
-                if ((b & 0x80) == 0) // 0xxxxxxx
-                {
-                    continue;
-                }
-                else if ((b & 0xE0) == 0xC0) // 110xxxxx 10xxxxxx
-                {
-                    if (!IsValidUtf8ContinuationByte(stream)) return false;
-                }
-                else if ((b & 0xF0) == 0xE0) // 1110xxxx 10xxxxxx 10xxxxxx
-                {
-                    if (!IsValidUtf8ContinuationByte(stream) || !IsValidUtf8ContinuationByte(stream)) return false;
-                }
-                else if ((b & 0xF8) == 0xF0) // 11110xxx 10xxxxxx 10xxxxxx 10xxxxxx
-                {
-                    if (!IsValidUtf8ContinuationByte(stream) || !IsValidUtf8ContinuationByte(stream) || !IsValidUtf8ContinuationByte(stream)) return false;
-                }
-                else
-                {
-                    return false;
-                }
-            }
-            return true;
-        }
-
-        private static bool IsValidUtf8ContinuationByte(Stream stream)
-        {
-            int b = stream.ReadByte();
-            return b != -1 && (b & 0xC0) == 0x80; // 10xxxxxx
-        }
-
-        private static bool IsUtf16(Stream stream, out Encoding encoding)
-        {
-            encoding = Encoding.Default;
-
-            if (stream.Length >= 2)
-            {
-                // Read the first two bytes
-                int b0 = stream.ReadByte();
-                int b1 = stream.ReadByte();
-
-                // Check for UTF-16 BOM (Byte Order Mark)
-                if (b0 == 0xFF && b1 == 0xFE)
-                {
-                    encoding = Encoding.Unicode;
-                    return true; // UTF-16LE
-                }
-                if (b0 == 0xFE && b1 == 0xFF)
-                {
-                    encoding = Encoding.BigEndianUnicode;
-                    return true; // UTF-16BE
-                }
-
-                // Reset the stream position
-                stream.Seek(0, SeekOrigin.Begin);
-
-                // Check for common UTF-16 patterns
-                byte[] buffer = new byte[2];
-                while (stream.Read(buffer, 0, 2) == 2)
-                {
-                    if (buffer[0] == 0x00 && buffer[1] != 0x00)
-                    {
-                        encoding = Encoding.BigEndianUnicode;
-                        return true; // Likely UTF-16BE
-                    }
-                    if (buffer[0] != 0x00 && buffer[1] == 0x00)
-                    {
-                        encoding = Encoding.Unicode;
-                        return true; // Likely UTF-16LE
-                    }
-                }
-            }
-
-            return false;
-        }
-
-        private static bool IsAscii(Stream stream)
-        {
-            int b;
-            while ((b = stream.ReadByte()) != -1)
-            {
-                // Latin1 includes byte values from 0x00 to 0xFF
-                if (b > 0x7F)
-                {
-                    return false;
-                }
-            }
-            return true;
-        }
-
-        private static bool IsLatin1(Stream stream)
-        {
-            int b;
-            while ((b = stream.ReadByte()) != -1)
-            {
-                // Latin1 includes byte values from 0x00 to 0xFF
-                if (b > 0xFF)
-                {
-                    return false;
-                }
-            }
-            return true;
-        }
-
+#endif // !BYTEMODE
    }
 #endif // !NOFILES
    #endregion Buffer classes
@@ -661,6 +540,180 @@ namespace QUT.GplexBuffers
            return 0;
        }
    }
+#region guesser
+#if (!BYTEMODE)
+    // ==============================================================
+    // ============          Encoding Guesser           =============
+    // ==============================================================
+
+    /// <summary>
+    /// This class provides a simple finite state automaton that
+    /// scans the file looking for (1) valid UTF-8 byte patterns,
+    /// (2) bytes >= 0x80 which are not part of a UTF-8 sequence.
+    /// The method then guesses whether it is UTF-8 or maybe some 
+    /// local machine default encoding.  This works well for the
+    /// various Latin encodings.
+    /// </summary>
+    internal class Guesser
+    {
+        ScanBuff buffer;
+
+        public int GuessCodePage() { return Scan(); }
+
+        const int maxAccept = 10;
+        const int initial = 0;
+        const int eofNum = 0;
+        const int goStart = -1;
+        const int INITIAL = 0;
+        const int EndToken = 0;
+
+        #region user code
+        /* 
+         *  Reads the bytes of a file to determine if it is 
+         *  UTF-8 or a single-byte code page file.
+         */
+        public long utfX;
+        public long uppr;
+        #endregion user code
+
+        int state;
+        int currentStart = startState[0];
+        int code;
+
+        #region ScannerTables
+        static int[] startState = new int[] { 11, 0 };
+
+        #region CharacterMap
+        static sbyte[] map = new sbyte[256] {
+/*     '\0' */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 
+/*   '\x10' */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 
+/*   '\x20' */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 
+/*      '0' */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 
+/*      '@' */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 
+/*      'P' */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 
+/*      '`' */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 
+/*      'p' */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 
+/*   '\x80' */ 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 
+/*   '\x90' */ 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 
+/*   '\xA0' */ 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 
+/*   '\xB0' */ 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 
+/*   '\xC0' */ 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 
+/*   '\xD0' */ 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 
+/*   '\xE0' */ 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 
+/*   '\xF0' */ 4, 4, 4, 4, 4, 4, 4, 4, 5, 5, 5, 5, 5, 5, 5, 5 };
+        #endregion
+
+        static sbyte[][] nextState = new sbyte[][] {
+            new sbyte[] {0, 0, 0, 0, 0, 0},
+            new sbyte[] {-1, -1, 10, -1, -1, -1},
+            new sbyte[] {-1, -1, -1, -1, -1, -1},
+            new sbyte[] {-1, -1, 8, -1, -1, -1},
+            new sbyte[] {-1, -1, 5, -1, -1, -1},
+            new sbyte[] {-1, -1, 6, -1, -1, -1},
+            new sbyte[] {-1, -1, 7, -1, -1, -1},
+            null,
+            new sbyte[] {-1, -1, 9, -1, -1, -1},
+            null,
+            null,
+            new sbyte[] {-1, 1, 2, 3, 4, 2}
+        };
+
+
+        [SuppressMessage("Microsoft.Performance", "CA1810:InitializeReferenceTypeStaticFieldsInline")]
+        // Reason for suppression: cannot have self-reference in array initializer.
+        static Guesser()
+        {
+            nextState[7] = nextState[2];
+            nextState[9] = nextState[2];
+            nextState[10] = nextState[2];
+        }
+
+        int NextState()
+        {
+            if (code == ScanBuff.EndOfFile)
+                return eofNum;
+            else
+                return nextState[state][map[code]];
+        }
+        #endregion
+
+        public Guesser(System.IO.Stream file) { SetSource(file); }
+
+        public void SetSource(System.IO.Stream source)
+        {
+            this.buffer = new BuildBuffer(source);
+            code = buffer.Read();
+        }
+
+        int Scan()
+        {
+            for (; ; )
+            {
+                int next;
+                state = currentStart;
+                while ((next = NextState()) == goStart)
+                    code = buffer.Read();
+
+                state = next;
+                code = buffer.Read();
+
+                while ((next = NextState()) > eofNum)
+                {
+                    state = next;
+                    code = buffer.Read();
+                }
+                if (state <= maxAccept)
+                {
+                    #region ActionSwitch
+#pragma warning disable 162
+                    switch (state)
+                    {
+                        case eofNum:
+                            switch (currentStart)
+                            {
+                                case 11:
+                                    if (utfX == 0 && uppr == 0) return -1; /* raw ascii */
+                                    else if (uppr * 10 > utfX) return 0;   /* default code page */
+                                    else return 65001;                     /* UTF-8 encoding */
+                                    break;
+                            }
+                            return EndToken;
+                        case 1: // Recognized '{Upper128}',	Shortest string "\xC0"
+                        case 2: // Recognized '{Upper128}',	Shortest string "\x80"
+                        case 3: // Recognized '{Upper128}',	Shortest string "\xE0"
+                        case 4: // Recognized '{Upper128}',	Shortest string "\xF0"
+                            uppr++;
+                            break;
+                        case 5: // Recognized '{Utf8pfx4}{Utf8cont}',	Shortest string "\xF0\x80"
+                            uppr += 2;
+                            break;
+                        case 6: // Recognized '{Utf8pfx4}{Utf8cont}{2}',	Shortest string "\xF0\x80\x80"
+                            uppr += 3;
+                            break;
+                        case 7: // Recognized '{Utf8pfx4}{Utf8cont}{3}',	Shortest string "\xF0\x80\x80\x80"
+                            utfX += 3;
+                            break;
+                        case 8: // Recognized '{Utf8pfx3}{Utf8cont}',	Shortest string "\xE0\x80"
+                            uppr += 2;
+                            break;
+                        case 9: // Recognized '{Utf8pfx3}{Utf8cont}{2}',	Shortest string "\xE0\x80\x80"
+                            utfX += 2;
+                            break;
+                        case 10: // Recognized '{Utf8pfx2}{Utf8cont}',	Shortest string "\xC0\x80"
+                            utfX++;
+                            break;
+                        default:
+                            break;
+                    }
+#pragma warning restore 162
+                    #endregion
+                }
+            }
+        }
+    } // end class Guesser
+    
+#endif // !BYTEMODE
+#endregion
 #endif // !NOFILES

 // End of code copied from embedded resource
@@ -1,7 +1,7 @@
 <Project Sdk="Microsoft.NET.Sdk">

  <PropertyGroup>
-    <TargetFramework>net8.0</TargetFramework>
+    <TargetFramework>net7.0</TargetFramework>
    <ImplicitUsings>enable</ImplicitUsings>
    <Nullable>enable</Nullable>
  </PropertyGroup>
@@ -1,7 +1,7 @@
 <Project Sdk="Microsoft.NET.Sdk">

  <PropertyGroup>
-    <TargetFramework>net8.0</TargetFramework>
+    <TargetFramework>net7.0</TargetFramework>
    <ImplicitUsings>enable</ImplicitUsings>
    <Nullable>enable</Nullable>
  </PropertyGroup>
@@ -1 +0,0 @@
-VITE_DATASTREAM_WEBSOCKET_URL=wss://localhost:8443/ws
@@ -1 +0,0 @@
-VITE_DATASTREAM_WEBSOCKET_URL=wss://localhost:8443/ws
@@ -19,13 +19,9 @@
    "@mui/material": "^5.15.20",
    "@mui/x-data-grid": "^7.7.1",
    "bulma": "^1.0.1",
-    "classnames": "^2.5.1",
    "react": "^18.3.1",
-    "react-data-grid": "^7.0.0-beta.44",
    "react-data-table-component": "^7.6.2",
    "react-dom": "^18.3.1",
-    "react-json-view-lite": "^1.4.0",
-    "react-pro-sidebar": "^1.1.0",
    "react-use-websocket": "^4.8.1",
    "rpc-websockets": "^7.11.2"
  },
@@ -37,7 +33,6 @@
    "eslint-plugin-react": "^7.34.2",
    "eslint-plugin-react-hooks": "^4.6.2",
    "eslint-plugin-react-refresh": "^0.4.7",
-    "react-mui-sidebar": "^1.3.5",
    "vite": "^5.2.13"
  }
 }
@@ -1,40 +0,0 @@
-import { Sidebar, Menu, MenuItem, Submenu, Logo } from "react-mui-sidebar";
-import './App.css';
-
-const AnimatedSideBar = () => {
-  return (
-    <Sidebar className={"sidebar-container"} width={"270px"}>
-      <Logo img="https://adminmart.com/wp-content/uploads/2024/03/logo-admin-mart-news.png">
-        AdminMart
-      </Logo>
-      <Menu subHeading="HOME">
-        <MenuItem link="/" badge="true">
-          Modern
-        </MenuItem>
-        <MenuItem>eCommerce</MenuItem>
-        <MenuItem>Analytical</MenuItem>
-      </Menu>
-      <Menu subHeading="APPS">
-        <MenuItem>Chat</MenuItem>
-        <MenuItem>Calendar</MenuItem>
-      </Menu>
-      <Menu subHeading="OTHERS">
-        <Submenu title="Menu Level">
-          <MenuItem>Post</MenuItem>
-          <MenuItem>Details</MenuItem>
-          <Submenu title="Level 2">
-            <MenuItem>new</MenuItem>
-            <MenuItem>Hello</MenuItem>
-          </Submenu>
-        </Submenu>
-
-        <MenuItem>Chip</MenuItem>
-        <MenuItem target="_blank" link="google.com">
-          External Link
-        </MenuItem>
-      </Menu>
-    </Sidebar>
-  );
-};
-
-export default AnimatedSideBar;
@@ -2,54 +2,6 @@
@use "@material/icon-button";
@use "@material/data-table/data-table";

-.root-div {
-  display: flex;
-  flex-direction: row;
-  align-items: flex-start;
-}
-
-.main-content-div {
-  display: flex;
-  flex-direction: row;
-}
-
-.title-div {
-  width: 100%;
-}
-
-.container {
-  display: flex;
-}
-
-.container-column {
-  display: flex;
-  flex-direction: column;
-}
-
-.container-row {
-  display: flex;
-  flex-direction: row;
-  align-items: stretch;
-}
-
-box.query-input-form {
-   /* p: 2, display: "flex", flexFlow: "column", alignItems: "stretch"  */
-  display: flex;
-  flex-flow: column;
-  align-items: stretch;
-}
-
-/* li.ps-menuitem-root:hover {
-  background-color: #ff9922;  
-} */
-
-.container-row-centered {
-  display: flex;
-  flex-direction: row;
-  justify-content: center;
-  align-items: center;
-}
-
 h1#main-header {
  padding: 4px 8px 4px 8px;
  text-align: center;
@@ -74,28 +26,3 @@ td#cell-string {
 td#cell-date-time {
  text-align: center;
 }
-
-/* Sidebar.css */
-/* .sidebar-container {
-  display: flex;
-  flex-direction: column;
-}
-
-.sidebar {
-  position: relative;
-  top: 0;
-  left: 0;
-  height: 100vh;
-  background-color: #333;
-  color: white;
-  overflow: hidden;
-  transition: transform 0.3s ease;
-}
-
-.sidebar.closed {
-  width: 0;
-}
-
-.sidebar.open {
-  width: 250px;
-} */
@@ -1,171 +1,22 @@
-import { ThemeProvider, CssBaseline, GlobalStyles } from '@mui/material';
-import QueryStatusView from './QueryStatusView';
+import { createTheme, ThemeProvider, CssBaseline } from '@mui/material';
 import useMediaQuery from '@mui/material/useMediaQuery';
 import QueryControl from './QueryControl';
-import DataTable from './DataTable';
-import { useState, useEffect, useMemo } from 'react';
-import useWebSocket, { ReadyState } from 'react-use-websocket';
-import Sidebar from './SideBar';
 import getTheme from './Theme';
-
 import '@fontsource/noto-sans';
 import './App.css';
-import './SideBar.css';
-
-const webSocketUrl = import.meta.env.VITE_DATASTREAM_WEBSOCKET_URL;
-// Always use the same subscription id (1), because we only show 1 table at a time.
-// The server will automatically dispose the previous subscription and start a new one
-// if we're currently subscribed to another query.
-const subscriptionId = "1";
-
-var rpcReqIdCounter = 1;
-
-function subscribeReq(query) {
-  return {
-    jsonrpc: "2.0",
-    id: rpcReqIdCounter++,
-    method: "subscribe",
-    params: {
-      subscription_id: subscriptionId,
-      query: query
-    }
-  };
-}
-
-function unsubscribeReq() {
-  return {
-    jsonrpc: "2.0",
-    id: rpcReqIdCounter++,
-    method: "unsubscribe",
-    params: {
-      subscription_id: subscriptionId,
-    }
-  };
-}

 const App = () => {
  const prefersDarkMode = useMediaQuery('(prefers-color-scheme: dark)');
  const theme = getTheme({ mode: prefersDarkMode ? 'dark' : 'light' });
-  const globalStyles = {
-    ':root': {
-      '--primary-color': theme.palette.primary.main,
-      '--primary-dark-color': theme.palette.primary.dark,
-      '--secondary-color': theme.palette.secondary.main,
-      '--secondary-dark-color': theme.palette.secondary.dark,
-      '--background-color': theme.palette.background.default,
-    },
-  };
-  const [blueTable, setBlueTable] = useState({ columns: null, rows: null });
-  const [queryError, setQueryError] = useState(null);
-  const [queryState, setQueryState] = useState("Unsubscribed");
-  const queryStateDetails = useMemo(() => {
-    if (queryState == "Unsubscribed") {
-      return "Enter query above, then click the play button to subscribe";
-    }
-    if (queryState == "Subscribed") {
-      return null;
-    }
-  }, [queryState]);
-  const [sidebarIsOpen, setSidebarIsOpen] = useState(false);
-  const [cannedQuery, setCannedQuery] = useState(null);

-  const onCannedQueryClicked = (cannedQuery) => {
-    console.info(`Executing canned query: ${JSON.stringify(cannedQuery)}`);
-    const queryText = cannedQuery.query.join("\n");
-    setCannedQuery({ queryText: queryText, clickTime: Date.now() });
-    subscribe(queryText);
-  };
-
-  const { sendJsonMessage, getWebSocket, readyState } = useWebSocket(
-    webSocketUrl,
-    {
-      onOpen: (e) => console.info(`WebSocket open. ${JSON.stringify(e)}`),
-      onClose: (e) => console.info(`WebSocket closed. ${JSON.stringify(e)}`),
-      onError: (e) => console.error(`WebSocket error: ${JSON.stringify(e)}`),
-      shouldReconnect: () => true,
-    });
-
-  useEffect(() => {
-    console.info(`WebSocket readyState: ${readyState}`)
-    if (readyState != ReadyState.OPEN) {
-      return;
-    }
-    const ws = getWebSocket();
-    ws.onmessage = e => {
-      console.info(`Received message: ${e.data}`);
-      const msg = JSON.parse(e.data);
-      setQueryError(msg.error);
-      if (msg.error != null) {
-        setQueryState("Error");
-      } else if (msg.id != null) {
-        setQueryState(msg.result.status);
-      } else {
-        console.assert(msg.params.subscription_id == subscriptionId);
-        console.info(`Received rows: ${msg.params.table.all}`);
-        setBlueTable({ columns: msg.params.table.columns, rows: msg.params.table.all });
-      }
-    };
-  }, [getWebSocket, readyState]);
-
-  const subscribe = (query) => {
-    const req = subscribeReq(query);
-    console.info(`Subscribing: ${JSON.stringify(subscribeReq(query))}`);
-    sendJsonMessage(req);
-  };
-
-  const unsubscribe = () => {
-    const req = unsubscribeReq();
-    console.info(`Unsubscribing: ${JSON.stringify(req)}`);
-    sendJsonMessage(req);
-  };
-
-  const onOpened = () => {
-    setSidebarIsOpen(true);
-  };
-
-  const onClosed = () => {
-    setSidebarIsOpen(false);
-  };
-
-  const sidebarWidth = sidebarIsOpen ? '250px' : '65px';
  return (
    <ThemeProvider theme={theme}>
      <CssBaseline />
-      <GlobalStyles styles={globalStyles} />
-      <div className='root-div'>
-        <div style={{
-          display: 'flex',
-          flexDirection: 'column',
-          alignItems: 'flex-start',
-          flexWrap: 'nowrap',
-          width: sidebarWidth,
-          transition: 'width 0.3s ease',
-        }}>
-          <Sidebar onSelection={onCannedQueryClicked} onOpened={onOpened} onClosed={onClosed} />
-        </div>
-        <div style={{
-          display: 'flex',
-          flexDirection: 'column',
-          left: sidebarWidth,
-          width: '100%',
-          transition: 'width 0.3s ease',
-        }}>
-          <div style={{ width: '100%', display: 'flex', flexDirection: 'column', alignItems: 'flex-start', margin: theme.spacing(2, 0, 0, 0) }}>
-            <div style={{ width: '90%', rightMargin: '100px' }}>
-              <QueryControl
-                queryState={queryState}
-                queryText={cannedQuery?.queryText}
-                queryTextResetTime={cannedQuery?.clickTime}
-                subscribe={subscribe}
-                unsubscribe={unsubscribe} />
-            </div>
-            <div style={{ width: '90%', rightMargin: '100px' }}>
-              <QueryStatusView queryState={queryState} details={queryStateDetails} error={queryError} />
-              <DataTable columns={blueTable?.columns ?? []} rows={blueTable?.rows ?? []} />
-            </div>
-          </div>
-        </div>
-      </div>
+      <h1 id='main-header' style={{
+        color: theme.palette.text.primary,
+        fontFamily: 'Noto Sans',
+      }}>BTS Cloud Streams</h1>
+      <QueryControl websocketUrl="ws://localhost:5000/ws" theme={theme} />
    </ThemeProvider>
  );
 };
@@ -1,53 +0,0 @@
-@use "@material/checkbox";
-@use "@material/icon-button";
-@use "@material/data-table/data-table";
-
-.data-grid-container {
-  overflow: auto;
-  white-space: nowrap;
-}
-
-
-.data-grid {
-  max-height: 1000px;
-}
-
-div[role=columnheader] {
-  border-top: 0px;
-  border-bottom: 0px;
-  background-color: var(--primary-dark-color);
-}
-
-/* div[role=columnheader] {
-  border-top: 0px;
-  border-bottom: 0px;
-  background-color: var(--primary-dark-color);
-}
-
-.rdg-row {
-  border: 0px;
-  border-top: 0px;
-  border-bottom: 0px;
-  background-color: rgba(var(--primary-dark-color), 0.8);
-  color: var(--primary-color);
-}
-
-.rdg-row-odd {
-  border: 0px;
-  border-top: 0px;
-  border-bottom: 0px;
-  background-color: var(--primary-dark-color);
-  color: var(--primary-color);
-}
-
-.rdg-cell {
-  border-bottom: 0px;
-}
-
-.rdg-cell.even {
-  border-bottom: 5px;
-}
-
-.rdg-cell.odd {
-  border-bottom: 5px;
-} */
@@ -1,40 +1,19 @@
 import PropTypes from 'prop-types';
-import { useTheme } from '@mui/material/styles';
 import { DataGrid } from '@mui/x-data-grid';
-import './DataGrid.css'
-import { useMemo, useState } from 'react';
+
+function getGridColDef(col) {
+  return {
+    field: col.name,
+    headerName: col.name,
+  };
+}

 export default function DataTable({ columns, rows }) {
-  const theme = useTheme();
-  const [columnWidths, setColumnWidths] = useState({});
-
-  const handleColumnResize = (params) => {
-    const newWidth = params.width;
-    setColumnWidths((prevWidths) => ({
-      ...prevWidths,
-      [params.colDef.field]: params.width,
-    }));
-  };
-
-  const getGridColDef = useMemo(() => {
-    return col => {
-      return {
-        field: col.name,
-        headerName: col.name,
-        width: columnWidths[col.name],
-      };
-    }
-  }, [columnWidths]);
-
  return (
    <DataGrid
-      className='data-grid-container'
-      autosizeOnMount={false}
-      columns={columns?.map(getGridColDef) ?? []}
-      onColumnResize={handleColumnResize}
-      rows={rows ?? []}
+      columns={columns.map(getGridColDef)}
+      rows={rows}
      sx={{
-        margin: theme.spacing(2, 0, 0, 0),
        borderColor: 'primary.main',
        backgroundColor: 'primary.dark',
        color: 'text.primary',
@@ -70,5 +49,4 @@ DataTable.propTypes = {
    })
  ).isRequired,
  rows: PropTypes.arrayOf(PropTypes.array).isRequired,
-  apiRef: PropTypes.object,
 };
@@ -1,119 +1,100 @@
 import PropTypes from 'prop-types';
 import DataTable from './DataTable';
-import { useRef, useState, useEffect } from 'react';
-import { useGridApiRef } from '@mui/x-data-grid';
-import Box from '@mui/material/Box';
-import Button from '@mui/material/Button';
-import Grid from '@mui/material/Grid';
-import Paper from '@mui/material/Paper';
-import PlayArrowIcon from '@mui/icons-material/PlayArrow';
-import StopIcon from '@mui/icons-material/Stop';
-import TextField from '@mui/material/TextField';
-import Tooltip from '@mui/material/Tooltip';
+import QueryInputForm from './QueryInputForm';
+import Container from '@mui/material/Container';
+import React from 'react';

-const QueryControl = (props) => {
-  const { queryState, queryText, queryTextResetTime, subscribe, unsubscribe } = props;
-  // const theme = useTheme();
-  let textInput = useRef();
-  const [queryTextChangeTime, setQueryTextChangeTime] = useState(null);
-  useEffect(() => {
-    if (queryText == null || textInput.current == null) {
-      return;
-    }
-    // Init text field (unless a non-empty string is already present, or the user has modified
-    // the query - we don't want to overwrite their changes).
-    if (queryTextChangeTime == null || queryTextResetTime > queryTextChangeTime) {
-      console.info(`Initializing query text: ${queryText}`)
-      textInput.current.value = queryText;
-    }
-  }, [queryText, queryTextChangeTime, queryTextResetTime]);
-  const onClearClick = () => {
-    textInput.current.value = '';
-  };
+var rpcReqIdCounter = 1;

-  const onRunClick = () => {
-    subscribe(textInput.current.value);
-  };
-
-  const onStopClick = () => {
-    unsubscribe();
-  };
-
-  const onKeyDown = (e) => {
-    // Detect keyboard shortcuts.
-    //   Ctrl+Enter : Run
-    if (e.ctrlKey && e.key === 'Enter') {
-      onRunClick();
+function subscribeReq(query) {
+  return {
+    jsonrpc: "2.0",
+    id: rpcReqIdCounter++,
+    method: "subscribe",
+    params: {
+      subscription_id: "1",
+      query: query
    }
  };
+}

-  const onChange = (e) => {
-    // So we know not to overwrite the changes with the canned query on re-render
-    setQueryTextChangeTime(Date.now());
+function unsubscribeReq() {
+  return {
+    jsonrpc: "2.0",
+    id: rpcReqIdCounter++,
+    method: "unsubscribe",
+    params: {
+      subscription_id: "1",
+    }
  };
+}

-  return (
-    <Box sx={{ display: "flex", flexFlow: "column" }}>
-      <Paper elevation={3} sx={{ color: 'primary.main' }}>
-        <Box sx={{ p: 2 }}>
-          <TextField
-            fullWidth
-            multiline
-            rows={10}
-            variant="outlined"
-            onChange={onChange}
-            onKeyDown={onKeyDown}
-            inputRef={textInput}
-            placeholder="Enter Query..."
-          />
-        </Box>
-        <Box sx={{ p: 2, borderTop: 1 }}>
-          <Grid container spacing={2} justifyContent="flex-start">
-            <Grid item xs>
-              <Tooltip title="Clear Query Text">
-                <Button onClick={onClearClick} sx={{ padding: '11px', color: 'text.dark' }}>
-                  Clear
-                </Button>
-              </Tooltip>
-            </Grid>
-            <Grid item>
-              <Tooltip title="Run (Ctrl+Enter)">
-                <Button onClick={onRunClick} sx={{
-                  color: `queryStartButton.${queryState}.color`,
-                  bgcolor: `queryStartButton.${queryState}.backgroundColor`,
-                  padding: '11px'
-                }}>
-                  <PlayArrowIcon />
-                </Button>
-              </Tooltip>
-            </Grid>
-            <Grid item>
-              <Tooltip title="Stop">
-                <Button onClick={onStopClick} sx={{
-                  color: `queryStopButton.${queryState}.color`,
-                  bgcolor: `queryStopButton.${queryState}.backgroundColor`,
-                  padding: '11px'
-                }}>
-                  <StopIcon />
-                </Button>
-              </Tooltip>
-            </Grid>
-          </Grid>
-        </Box>
-      </Paper>
-    </Box>
-  );
-};
+class QueryControl extends React.Component {
+  constructor({ websocketUrl, theme }) {
+    super(websocketUrl);
+    console.log('Connecting to WebSocket:', websocketUrl);
+    this.state = {
+      rows: null,
+      ws: new WebSocket(websocketUrl),
+      columns: null,
+      isRunning: false,
+      theme: theme,
+    };
+    this.state.ws.onopen = () => {
+      console.info('WebSocket open');
+    };
+    this.state.ws.onclose = () => {
+      console.info('WebSocket closed');
+    };
+    this.state.ws.onmessage = e => {
+      console.info(`Received message: ${e.data}`);
+      const msg = JSON.parse(e.data);
+      if (msg.id != null) {
+        let newState = { ...this.state };
+        newState.isRunning = msg.result.status == 'Subscribed';
+        this.setState(newState);
+      } else {
+        let newState = { ...this.state };
+        newState.rows = msg.params.all;
+        console.info(`Received rows: ${newState.rows}`);
+        newState.columns = msg.params.columns;
+        this.setState(newState);
+      }
+    };
+
+    this.start = (query) => {
+      this.state.ws.send(JSON.stringify(subscribeReq(query)));
+    };
+
+    this.stop = () => {
+      this.state.ws.send(JSON.stringify(unsubscribeReq()));
+    };
+  }
+
+  componentWillUnmount() {
+    if (this.state.ws) {
+      console.warn('Closing websocket because component is unmounting');
+      this.state.ws.close();
+    }
+  }
+
+  render() {
+    console.info('Rendering');
+    return (
+      <Container>
+        <QueryInputForm isRunning={this.state.isRunning} onStart={this.start} onStop={this.stop} />
+        {
+          this.state.rows == null ? <p style={{ color: this.state.theme.palette.warning.main }}>No data received</p> :
+            <DataTable columns={this.state.columns} rows={this.state.rows} />
+        }
+      </Container>
+    );
+  }
+}

 QueryControl.propTypes = {
-  columns: PropTypes.array,
-  rows: PropTypes.array,
-  isRunning: PropTypes.bool,
-  queryText: PropTypes.string,
-  queryTextResetTime: PropTypes.number,
-  queryState: PropTypes.string,
-  subscribe: PropTypes.func,
-  unsubscribe: PropTypes.func,
+  websocketUrl: PropTypes.string.isRequired,
+  theme: PropTypes.object.isRequired,
 };

 export default QueryControl;
@@ -0,0 +1,93 @@
+import { useRef } from 'react';
+import Box from '@mui/material/Box';
+import Button from '@mui/material/Button';
+import Grid from '@mui/material/Grid';
+import Paper from '@mui/material/Paper';
+import PlayArrowIcon from '@mui/icons-material/PlayArrow';
+import PropTypes from 'prop-types';
+import StopIcon from '@mui/icons-material/Stop';
+import TextField from '@mui/material/TextField';
+import Tooltip from '@mui/material/Tooltip';
+
+const QueryInputForm = ({ isRunning, onStart, onStop }) => {
+  let textInput = useRef(null);
+
+  const onClearClick = () => {
+    textInput.current.value = '';
+  };
+
+  const onRunClick = () => {
+    onStart(textInput.current.value);
+  };
+
+  const onStopClick = () => {
+    onStop();
+  };
+
+  const onKeyDown = (e) => {
+    // Detect keyboard shortcuts.
+    //   Ctrl+Enter : Run
+    if (e.ctrlKey && e.key === 'Enter') {
+      onRunClick();
+    }
+  };
+
+  return (
+    <Box sx={{ p: 2 }}>
+      <Paper elevation={3} sx={{ color: 'primary.main' }}>
+        <Box sx={{ p: 2 }}>
+          <TextField
+            fullWidth
+            multiline
+            rows={10}
+            variant="outlined"
+            onKeyDown={onKeyDown}
+            inputRef={textInput}
+            placeholder="Enter Query..."
+          />
+        </Box>
+        <Box sx={{ p: 2, borderTop: 1 }}>
+          <Grid container spacing={2} justifyContent="flex-start">
+            <Grid item xs>
+              <Tooltip title="Clear Query Text">
+                <Button onClick={onClearClick} sx={{ padding: '11px', color: 'text.dark' }}>
+                  Clear
+                </Button>
+              </Tooltip>
+            </Grid>
+            <Grid item>
+              <Tooltip title="Run (Ctrl+Enter)">
+                <Button onClick={onRunClick} sx={{
+                  color: `query_start_button.${isRunning ? 'active' : 'inactive'}.color`,
+                  bgcolor: `query_start_button.${isRunning ? 'active' : 'inactive'}.background_color`,
+                  padding: '11px'
+                }}>
+                  <PlayArrowIcon />
+                </Button>
+              </Tooltip>
+            </Grid>
+            <Grid item>
+              <Tooltip title="Stop">
+                <Button onClick={onStopClick} sx={{
+                  color: `query_stop_button.${isRunning ? 'inactive' : 'active'}.color`,
+                  bgcolor: `query_stop_button.${isRunning ? 'inactive' : 'active'}.background_color`,
+                  padding: '11px'
+                }}>
+                  <StopIcon />
+                </Button>
+              </Tooltip>
+            </Grid>
+          </Grid>
+        </Box>
+      </Paper>
+    </Box>
+  );
+};
+
+QueryInputForm.propTypes = {
+  isRunning: PropTypes.bool.isRequired,
+  onStart: PropTypes.func.isRequired,
+  onStop: PropTypes.func.isRequired,
+};
+
+export default QueryInputForm;
@@ -1,73 +0,0 @@
-import { useState } from 'react';
-import PropTypes from 'prop-types';
-import { Box, Typography, Collapse } from '@mui/material';
-import Alert from '@mui/material/Alert';
-import AlertTitle from '@mui/material/AlertTitle';
-import { useTheme } from '@mui/material/styles';
-import { alpha } from '@mui/system';
-
-const QueryStatusView = ({ queryState, details, error }) => {
-  const theme = useTheme();
-  const [open, setOpen] = useState(false);
-
-  let header = queryState;
-  let headerContent = null;
-  let detailElements = [];
-  let severity = queryState == "Subscribed" ? "success" : "info";
-  let cursor = details == null ? "default" : "pointer";
-  if (error != null) {
-    severity = "error";
-    cursor = "pointer";
-    header = `Error Code: ${error.code}`;
-    headerContent = error.message;
-    detailElements.push(<Typography variant="body1">{error.data.error_message}</Typography>);
-    detailElements.push(<Typography variant="subtitle1">Stack Trace:</Typography>);
-    (error.data.stack_trace ?? []).forEach(line => detailElements.push(
-      <Typography variant="body2" component="pre" sx={{ whiteSpace: 'pre-wrap' }}>
-        {line}
-      </Typography>));
-  } else if (details != null) {
-    detailElements.push(<Typography variant="body1">{details}</Typography>);
-  }
-  return (
-    <Box sx={{ margin: theme.spacing(2, 0, 0, 0) }}>
-      <Alert
-        severity={severity}
-        onClick={() => setOpen(!open)}
-        sx={{
-          margin: theme.spacing(2, 0, 0, 0),
-          cursor: cursor,
-        }}>
-        <AlertTitle>{header}</AlertTitle>
-        {headerContent}
-      </Alert>
-      {detailElements.length == 0
-        ? null
-        : <Collapse in={open}>
-          <Box
-            sx={{
-              backgroundColor: alpha(theme.palette[severity].light, 0.75),
-              color: theme.palette[severity].contrastText,
-              padding: theme.spacing(2),
-              marginTop: theme.spacing(0),
-              borderRadius: theme.shape.borderRadius,
-            }}
-          >
-            {detailElements}
-          </Box>
-        </Collapse>}
-    </Box>
-  );
-};
-
-QueryStatusView.propTypes = {
-  queryState: PropTypes.string,
-  details: PropTypes.string,
-  error: PropTypes.shape({
-    code: PropTypes.number,
-    message: PropTypes.string,
-    data: PropTypes.object,
-  })
-};
-
-export default QueryStatusView;
@@ -1,27 +0,0 @@
-import 'react-data-grid/lib/styles.css';
-import PropTypes from 'prop-types';
-import DataGrid from 'react-data-grid';
-import './DataGrid.css';
-
-function getGridColDef(col) {
-  return {
-    key: col.name,
-    name: col.name,
-  };
-}
-
-export default function ReactDataGrid({columns, rows}) {
-  return (
-    <DataGrid className={"data-grid"} rows={rows} columns={columns.map(getGridColDef)} />
-  )
-}
-
-ReactDataGrid.propTypes = {
-  columns: PropTypes.arrayOf(
-    PropTypes.shape({
-      name: PropTypes.string.isRequired,
-      type: PropTypes.string.isRequired,
-    })
-  ).isRequired,
-  rows: PropTypes.arrayOf(PropTypes.array).isRequired,
-};
@@ -1,98 +0,0 @@
-/* Sidebar.css */
-.app-container {
-  display: flex;
-  width: 100%;
-  overflow: hidden;
-  flex-direction: row;
-  justify-content: flex-start;
-  align-items: flex-start;
-}
-
-.sidebar {
-  display: flex;
-  flex-direction: row;
-  justify-content: flex-start;
-  align-items: flex-start;
-  background-color: transparent;
-  color: white;
-  overflow: hidden;
-  transition: width 0.3s ease;
-}
-
-.sidebar.open {
-  width: 100%;
-}
-
-.sidebar.closed {
-  width: 0%;
-}
-
-.sidebar-toggle-button {
-  border: 0px;
-  border-radius: 50%;
-  box-shadow: rgba(9, 30, 66, 0.08) 0px 0px 0px 1px, rgba(9, 30, 66, 0.08) 0px 2px 4px 1px;
-  color: var(--primary-color);
-  cursor: pointer;
-  height: 24px;
-  opacity: 1;
-  outline: 0px;
-  padding: 0px;
-  background-color: transparent;
-  transition: background-color 100ms linear 0s, color 100ms linear 0s, opacity 300ms cubic-bezier(0.2, 0, 0, 1) 0s, transform 300ms cubic-bezier(0.2, 0, 0, 1) 0s;
-  width: 20%;
-}
-
-.sidebar-toggle-button:hover {
-  transition: background-color 100ms linear 0s, color 100ms linear 0s, opacity 300ms cubic-bezier(0.2, 0, 0, 1) 0s, transform 300ms cubic-bezier(0.2, 0, 0, 1) 0s;
-}
-
-.main-content.open {
-  width: 165px;
-  transition: width 0.3s ease;
-  background-color: transparent;
-}
-
-.main-content.closed {
-  width: 0%;
-  /* min-width: 25px; */
-  transition: width 0.3s ease;
-  background-color: transparent;
-}
-
-.sidebar-menu-item {
-  width: 100%;
-  box-sizing: border-box;
-}
-
-.sidebar-menu-item.open {
-  width: 100%;
-  /* justify-content: flex-start; */
-  transition: width 0.3s ease;
-}
-
-.sidebar-menu-item.closed {
-  width: 0%;
-  min-width: 0px;
-  /* justify-content: flex-start; */
-  transition: width 0.3s ease;
-}
-
-.sidebar-menu-item-content.open {
-  width: 100%;
-  /* justify-content: flex-start; */
-  transition: width 0.3s ease;
-}
-
-.sidebar-menu-item-content.closed {
-  width: 0px;
-  /* justify-content: flex-start; */
-  transition: width 0.3s ease;
-}
-
-/* Ensure span can shrink */
-.button-text {
-  display: block;    /* Make span a block element */
-  width: 100%;       /* Ensure it takes the full width of its parent */
-  overflow: hidden;  /* Hide any overflow content */
-  white-space: nowrap; /* Prevent text wrapping */
-}
@@ -1,109 +0,0 @@
-import { useState, useEffect } from 'react';
-import Button from '@mui/material/Button';
-import { useTheme } from '@mui/material/styles';
-import PropTypes from 'prop-types';
-import Tooltip from '@mui/material/Tooltip';
-import ChevronLeftRounded from "@mui/icons-material/ChevronLeftRounded"
-import ChevronRightRounded from '@mui/icons-material/ChevronRightRounded';
-import FileCopyIcon from '@mui/icons-material/FileCopy';
-import './SideBar.css';
-
-const Sidebar = ({ onSelection, onOpened, onClosed }) => {
-  const theme = useTheme();
-  const [isOpen, setIsOpen] = useState(false);
-  const [cannedQueries, setCannedQueries] = useState([]);
-  const Icon = isOpen ? ChevronLeftRounded : ChevronRightRounded;
-
-  const toggleIsOpen = () => {
-    if (isOpen) {
-      setIsOpen(false);
-      if (onClosed != null) {
-        onClosed();
-      }
-    } else {
-      setIsOpen(true);
-      if (onOpened != null) {
-        onOpened();
-      }
-    }
-  };
-
-  const onMenuItemClick = (event, cannedQuery) => {
-    console.info(`menu item clicked: ${cannedQuery.name}`);
-    onSelection(cannedQuery);
-  };
-
-  // const onMenuItemClick = async (event, cannedQuery) => {
-  //   try {
-  //     await navigator.clipboard.writeText(cannedQuery.query.join("\n"));
-  //     alert(`Copied ${cannedQuery.name} to clipboard`);
-  //   } catch (err) {
-  //     console.error('Failed to copy query text: ', err);
-  //   }
-  // };
-
-  useEffect(() => {
-    const fetchCannedQueries = async () => {
-      try {
-        const response = await fetch('api/cannedqueries');
-        const data = await response.json();
-        setCannedQueries(data);
-      } catch (error) {
-        console.error('Error fetching WebSocket config:', error);
-      }
-    }
-    fetchCannedQueries();
-  }, []);
-
-  const mainContentClass = `main-content ${isOpen ? 'open' : 'closed'}`;
-  const menuItemClass = `sidebar-menu-item ${isOpen ? 'open' : 'closed'}`;
-  const menuItemContentClass = `sidebar-menu-item-content ${isOpen ? 'open' : 'closed'}`;
-  const toggleButtonTitle = `${isOpen ? 'Hide' : 'Show'} Common Queries`;
-
-  return (
-    <div
-      className={"sidebar"}
-      style={{
-        display: 'flex',
-        flexDirection: 'row',
-        justifyContent: 'flexStart',
-        alignItems: 'flexStart',
-        margin: theme.spacing(2, 0, 0, 0)
-      }}
-    >
-      <div className={mainContentClass}>
-        {cannedQueries.map(cq =>
-          <Button key={cq.name} className={menuItemClass} onClick={(event) => onMenuItemClick(event, cq)}>
-            <div className={menuItemContentClass}>
-              <span className='button-text'>{cq.name}</span>
-            </div>
-          </Button>)
-        }
-        {/* {cannedQueries.map(cq =>
-          <Button
-            variant="contained"
-            color="primary"
-            startIcon={<FileCopyIcon />}
-            onClick={onMenuItemClick}
-          >
-            <div className={menuItemContentClass}>
-              <span className='button-text'>{cq.name}</span>
-            </div>
-          </Button>)} */}
-      </div>
-      <Tooltip title={toggleButtonTitle}>
-        <Button onClick={toggleIsOpen}>
-          <Icon className="w-5 h-5" />
-        </Button>
-      </Tooltip>
-    </div>
-  );
-};
-
-export default Sidebar;
-
-Sidebar.propTypes = {
-  onSelection: PropTypes.func,
-  onOpened: PropTypes.func,
-  onClosed: PropTypes.func,
-};
@@ -42,56 +42,32 @@ const getDarkPalette = () => {
      dark: '#ffffff12',
      contrastText: '#c8d4f7',
    },
-    action: {
-      hoverOpacity: 0.08
-    },
    text: {
      primary: '#eaf8f8',
      dark: '#eaf8f8a0'
    },
-    queryStartButton: {
-      Subscribed: {
+    query_start_button: {
+      active: {
        color: '#33baf9',
-        backgroundColor: '#ffffff40',
+        background_color: '#ffffff40',
      },
-      Unsubscribed: {
+      inactive: {
        color: '#ffffff40',
-        backgroundColor: '#ffffff00',
+        background_color: '#ffffff00',
      }
    },
-    queryStopButton: {
-      Subscribed: {
-        color: '#ff4444a0',
-        backgroundColor: '#ffffff40',
-      },
-      Unsubscribed: {
-        color: '#ff444450',
-        backgroundColor: '#ffffff00',
-      }
-    },
-    error: {
-      light: '#e57373',
-      main: '#f44336',
-      dark: '#d32f2f',
-      contrastText: '#fff',
-    },
    warning: {
-      light: '#ffb74d',
-      main: '#ff9800',
-      dark: '#f57c00',
-      contrastText: '#000',
+      main: '#c4823f'
    },
-    info: {
-      light: '#64b5f6',
-      main: '#2196f3',
-      dark: '#303030',
-      contrastText: '#fff',
-    },
-    success: {
-      light: '#81c784',
-      main: '#4caf50',
-      dark: '#388e3c',
-      contrastText: '#000',
+    query_stop_button: {
+      active: {
+        color: '#ff4444a0',
+        background_color: '#ffffff40',
+      },
+      inactive: {
+        color: '#ff444450',
+        background_color: '#ffffff00',
+      }
    },
  };
 };
@@ -3,5 +3,7 @@ import ReactDOM from 'react-dom/client';
 import App from './App';

 ReactDOM.createRoot(document.getElementById('root')).render(
-  <App />
+  <React.StrictMode>
+    <App />
+  </React.StrictMode>
 )
@@ -33,7 +33,7 @@ const target = env.ASPNETCORE_HTTPS_PORT
  ? `https://localhost:${env.ASPNETCORE_HTTPS_PORT}`
  : env.ASPNETCORE_URLS
    ? env.ASPNETCORE_URLS.split(';')[0]
-    : 'https://localhost:8443';
+    : 'https://localhost:5001';

 export default defineConfig(({ $$command, mode, $$isSsrBuild, $$isPreview }) => {
  return {
@@ -29,12 +29,7 @@
    <ProjectReference Include="..\..\client\spark_websocket\spark_websocket.csproj" />
  </ItemGroup>

-  <ItemGroup>
-    <ConfigurationProfile Include="appsettings.Development.json" />
-    <ConfigurationProfile Include="appsettings.Production.json" />
-    <ConfigurationProfile Include="appsettings.ProductionAws.json" />
-    <ConfigurationProfile Include="appsettings.Staging.json" />
-    <ConfigurationProfile Include="appsettings.json" />
-  </ItemGroup>
-
+  <!-- <ItemGroup>
+    <Content Update="wwwroot\alt-index.html" CopyToPublishDirectory="Never" />
+  </ItemGroup> -->
 </Project>
@@ -1,5 +0,0 @@
-namespace Bts.DataStreamWebApp.Server;
-
-public class CannedQueriesConfig {
-   public List<CannedQuery>? Items { get; set; }
-}
@@ -1,6 +0,0 @@
-namespace Bts.DataStreamWebApp.Server;
-
-public class CannedQuery {
-   public string? Name { get; set; }
-   public List<string>? Query { get; set; }
-}
@@ -5,10 +5,10 @@
  "ViewStreamDatabase": {
    "Interval": "00:00:00.1"
  },
-  "Spark": {
-    "ConfigServer": "http://172.30.2.181:8000/config"
-  },
-  "SparkApi": {
+  "Sparkd": {
+    "Config": {
+      "ConfigServer": "http://localhost:8000/config"
+    },
    "Credentials": {
      "User": "bts",
      "Password": ""
@@ -18,25 +18,38 @@
    "EnablePosition": true
  },
  "PubSub": {
+    "MillisecondsBetweenSubInfoUpdates": 1000,
+    "MillisecondsBetweenTimeSliceUpdates": 5000,
+    "MillisecondsBetweenNoBufferSpaceRetries": 100,
+    "RetriesOnNoBufferSpace": 1,
+    "DoItAllSecondsBetweenLogs": 0,
    "HeartbeatInterval": 15000,
    "MissingHeartbeatsBeforeGc": 50,
-    "ServerPort": 22019,
+    "UseManagedSockets": true,
+    "StateUpdatePeriod": 5000,
+    "StatisticsUpdatePeriod": 5000,
+    "DisposedConnectionCheckPeriod": 1000,
+    "MessageProcessingWarningThreshold": 1000,
+    "ServerPort": 12345,
    "Routing": {
      "PublisherRoutes": [
        {
          "Name": "default",
-          "ServerAddress": "bts-cloud://172.30.2.181:22019",
+          "ServerAddress": "bts-cloud://127.0.0.1:12345",
          "TagRegExp": ".*"
        }
      ],
      "SubscriberRoutes": [
        {
          "Name": "default",
-          "ServerAddress": "bts-cloud://172.30.2.181:22019",
+          "ServerAddress": "bts-cloud://127.0.0.1:12345",
          "TagRegExp": ".*"
        }
      ]
    },
+    "MaxPacketSize": 500000000,
+    "BlueCommMustDie": true,
+    "CrushableTags": [],
    "TagTypeToAssemblyMap": {
      "Entries": [
        {
@@ -45,6 +58,26 @@
        }
      ]
    },
+    "ServerQueue": "PubSubQueue",
+    "PortRange": {
+      "First": 30400,
+      "Last": 31700
+    },
+    "BindToRandomPort": false,
+    "DebugTags": "^NOTAGS$",
+    "LatencyReportThresholdMs": 1500,
+    "UseSubstations": false,
+    "UseDynamicClients": false,
+    "IsSubstation": false,
+    "InternallyPublishSubInfo": true,
+    "UseManagedClientsInNativeCode": false,
+    "CompressedTags": "^NOTAGS$",
+    "TagsToReportCompressionRatio": "^NOTAGS$",
+    "DisableMsmq": false,
+    "UsePrepublishedServerState": false,
+    "IpTranslation": null,
+    "GcActivationThreshold": 0,
+    "WaitPeriodAfterGc": 1000,
    "TagHandlerFactory": {
      "Handlers": [
        {
@@ -65,51 +98,9 @@
        {
          "TagType": ".*",
          "Assembly": "DataStreamCore",
-          "Class": "bts.PubsubServerGenericDataHandler"
+         "Class": "bts.PubsubServerGenericDataHandler"
        }
      ]
    }
-  },
-  "Kestrel": {
-    "Endpoints": {
-      "Http": {
-        "Url": "http://172.30.2.181:8080"
-      },
-      "Https": {
-        "Url": "https://172.30.2.181:8443"
-      }
-    }
-  },
-  "CannedQueries": [
-    {
-      "Name": "BBO",
-      "Query": [
-        "bbo"
-      ]
-    },
-    {
-      "Name": "Trades",
-      "Query": [
-        "timesales"
-      ]
-    },
-    {
-      "Name": "Instruments",
-      "Query": [
-        "instruments"
-      ]
-    },
-    {
-      "Name": "Books",
-      "Query": [
-        "books"
-      ]
-    },
-    {
-      "Name": "Benzinga News",
-      "Query": [
-        "benzinga_news"
-      ]
-    }
-  ]
+  }
 }
@@ -0,0 +1,4 @@
+{
+  "content_root_path": "build/Bts.DataStreamWebApp.Client/production",
+  "web_root_path": "dist"
+}
@@ -1,18 +0,0 @@
-namespace Bts.DataStreamWebApp.Server;
-
-using Microsoft.AspNetCore.Mvc;
-
-[ApiController]
-[Route("api/[controller]")]
-public class CannedQueriesController : ControllerBase {
-   readonly IConfiguration config;
-
-   public CannedQueriesController(IConfiguration config) =>
-      this.config = config;
-
-   [HttpGet(Name = "GetCannedQueries")]
-   public IEnumerable<CannedQuery>? Get() {
-      var cannedQueries = config.GetSection("CannedQueries").Get<List<CannedQuery>>()?.ToArray() ?? Array.Empty<CannedQuery>();
-      return cannedQueries;
-   }
-}
@@ -1,22 +0,0 @@
-// using Bts.DataStreamWebApp.Server;
-
-// using Microsoft.AspNetCore.Mvc;
-
-// [ApiController]
-// [Route("api/[controller]")]
-// public class ConfigController : ControllerBase {
-//    private readonly IConfiguration _configuration;
-
-//    public ConfigController(IConfiguration configuration) {
-//       _configuration = configuration;
-//    }
-
-//    [HttpGet("websocket")]
-//    public ActionResult<DataStreamWebSocketConfig> GetWebSocketConfig() {
-//       var endpoint = $"{_configuration["Kestrel:Endpoints:Http:Url"]}/ws";
-//       var webSocketConfig = new DataStreamWebSocketConfig {
-//          Endpoint = endpoint // or Https based on your setup
-//       };
-//       return Ok(webSocketConfig);
-//    }
-// }
@@ -7,7 +7,6 @@ using bts.websocket_core.json_rpc;
 using Bts.DataStreamCore;
 using Bts.DataStreamWebApp.Server.Messages;

-using System;
 using System.Net.WebSockets;
 using System.Text;
 using System.Text.Json;
@@ -97,10 +96,7 @@ class DataStreamSubscription : IDisposable, IHostedService {
         IncludeCurrent = true,
         UseObjectRows = true,
      });
-      var res = JsonRpcRequest.CreateNotification(method, new DataStreamSubscriptionUpdate() {
-         SubscriptionId = Id,
-         Table = dp
-      });
+      var res = JsonRpcRequest.CreateNotification(method, dp);
      var str = serialize(res, serializerOpts);
      if (str is null) {
         logger.LogError("Unsubscribing from stream early because we failed to serialize an outgoing DataPacket");
@@ -113,12 +109,8 @@ class DataStreamSubscription : IDisposable, IHostedService {
         dp.schema.columnNames.Length,
         str.Length);
      var bytes = Encoding.UTF8.GetBytes(str);
-      try {
-         await webSocket.SendAsync(bytes, WebSocketMessageType.Text, true, CancellationToken.None);
-         LastFired = dp;
-      } catch (WebSocketException ex) {
-         logger.LogError($"sendDataPacket failed on webSocket.SendAsync: {ex}");
-      }
+      await webSocket.SendAsync(bytes, WebSocketMessageType.Text, true, CancellationToken.None);
+      LastFired = dp;
   }

   string? serialize<T>(T message, JsonSerializerOptions? opts) {
@@ -1,13 +0,0 @@
-namespace Bts.DataStreamWebApp.Server;
-
-using BCG;
-
-using System.Text.Json.Serialization;
-
-public class DataStreamSubscriptionUpdate {
-   [JsonPropertyName("subscription_id")]
-   public string? SubscriptionId { get; set; }
-
-   [JsonPropertyName("table")]
-   public DataPacket? Table { get; set; }
-}
@@ -1,5 +0,0 @@
-namespace Bts.DataStreamWebApp.Server;
-
-public class DataStreamWebSocketConfig {
-   public string? Endpoint { get; set; }
-}
@@ -58,67 +58,40 @@ public class DataStreamWebSocketServer : IDisposable, IAsyncDisposable {
   async Task onSubscribe(JsonRpcRequest<JsonNode> req, CancellationToken cancellationToken) {
      ArgumentNullException.ThrowIfNull(req.Id);
      ArgumentException.ThrowIfNullOrEmpty(req.Method);
-      if (!SubscribeRequest.Parse(req, out var subReq, out var parseErr, out var errCode)) {
-         logger.LogError("Invalid 'subscribe' request: {Reason}", parseErr);
-         var rpcError = JsonRpcResponse.CreateError<SubscriptionError>(
-            req.Id, new() { Code = (long)errCode, Message = "Failed to parse subscription request", Data = parseErr });
-         await send(rpcError, cancellationToken);
-         return;
-      }
-      var subId = subReq.SubscriptionId ?? throw new InvalidOperationException("SubscriptionId is null (parse false)");
-      disposeSubscription(subId);
-      if (!subReq.ParseQuery(out var vs, out parseErr, out errCode)) {
-         logger.LogError("Invalid 'subscribe' request: {Reason}", parseErr);
-         var rpcError = JsonRpcResponse.CreateError<SubscriptionError>(
-            req.Id, new() { Code = (long)errCode, Message = "Failed to parse subscription request", Data = parseErr });
+      if (!SubscribeRequest.ParseQuery(req, out var vs, out var subId, out var errMsg, out var errCode)) {
+         logger.LogError("Invalid 'subscribe' request: {Reason}", errMsg);
+         var rpcError = new JsonRpcError<JsonNode>() {
+            Code = (long)errCode,
+            Message = $"{errCode}: {errMsg}",
+         };
         await send(rpcError, cancellationToken);
         return;
      }
      if (subscriptions.TryGetValue(subId, out var oldSub)) {
-         logger.LogInformation("Disposing existing subscription with id {SubscriptionId}", subId);
         oldSub.Dispose();
      }
      var sub = new DataStreamSubscription(subId, req.Id.Value, vs, webSocket, logger);
      subscriptions[subId] = sub;
      await sendSubscriptionStatus(req.Id.Value, subId, req.Method, SubscriptionStatus.Subscribed, cancellationToken);
-      vs.ErrorChanged += OnStreamError;
      await sub.StartAsync(cancellationToken);
-
-      async void OnStreamError(Exception exception) {
-         logger.LogError("Error in ViewStream (disposing subscription): {Exception}", exception);
-         subscriptions.Remove(sub.Id);
-         sub.Dispose();
-         var error = new SubscriptionError() {
-            ErrorMessage = "Error in ViewStream (disposing subscription)"
-         };
-         if (exception is not null) {
-            error.ErrorMessage += ". " + exception.Message;
-            error.StackTrace = exception.StackTrace?.Split("\n")?.ToList();
-         }
-         var rpcError = JsonRpcResponse.CreateError<SubscriptionError>(
-            req.Id, new() { Code = (long)ErrorCode.ViewStreamError, Message = "Error in ViewStream (disposing subscription)", Data = error });
-         await send(rpcError, cancellationToken);
-      }
   }

   async Task onUnsubscribe(JsonRpcRequest<UnsubscribeRequest> req, CancellationToken cancellationToken) {
      ArgumentNullException.ThrowIfNull(req.Id);
      ArgumentException.ThrowIfNullOrEmpty(req.Method);
      var subId = req.Params?.SubscriptionId ?? string.Empty;
-      if (!disposeSubscription(subId)) {
-         logger.LogInformation("Unable to dispose subscription. Subscription with id {Id} not found.", subId);
-      }
-      await sendSubscriptionStatus(req.Id.Value, subId, "unsubscribe", SubscriptionStatus.Unsubscribed, cancellationToken);
-   }
-
-   bool disposeSubscription(string subId) {
      if (!subscriptions.TryGetValue(subId, out var sub)) {
-         return false;
+         logger.LogInformation("Ignoring '{Method}' request because no subscription was found with id {Id}.", req.Method, subId);
+         return;
      }
-      logger.LogInformation("Disposing subscription with id {Id}", subId);
+      logger.LogInformation("Disposing subscription");
      sub.Dispose();
      subscriptions.Remove(subId);
-      return true;
+      await sendSubscriptionStatus(req.Id.Value, subId, req.Method, SubscriptionStatus.Unsubscribed, cancellationToken);
+   }
+
+   async Task unsubscribe(JsonRpcId requestId, string subscriptionId, CancellationToken cancellationToken) {
+      await sendSubscriptionStatus(requestId, subscriptionId, "unsubscribe", SubscriptionStatus.Unsubscribed, cancellationToken);
   }

   Task sendSubscriptionStatus(JsonRpcId reqId, string subId, string method, SubscriptionStatus status, CancellationToken cancellationToken) {
@@ -1,11 +0,0 @@
-FROM bluetradingsystems/linux_dotnet7
-
-ARG SOURCE_DIR="build/Release/bin/net7.0/linux-x64/publish"
-ARG WEB_ROOT="build/Bts.DataStreamWebApp.Client/production/dist"
-ARG CONF_FILE="src/DataStreamSuite/Bts.DataStreamWebApp.Server/spark.conf"
-
-RUN dotnet dev-certs https --trust
-COPY --chown=bts:bts "${SOURCE_DIR}" "bin"
-COPY --chown=bts:bts "${WEB_ROOT}" "bin/wwwroot"
-COPY --chown=bts:bts "${CONF_FILE}" ".local/share/bts/Bts.DataStreamWebApp.Server/conf/spark.conf"
-WORKDIR "/home/bts/bin"
@@ -1,5 +0,0 @@
-namespace Bts.DataStreamWebApp.Server;
-
-public class CannedQueriesResponse {
-   public List<CannedQuery>? Items { get; set; }
-}
@@ -3,5 +3,4 @@ namespace Bts.DataStreamWebApp.Server.Messages;
 enum ErrorCode {
   None = 0,
   InvalidSubscriptionRequest,
-   ViewStreamError,
 }
@@ -18,52 +18,47 @@ public class SubscribeRequest {
   [JsonPropertyName("subscription_id")]
   public string? SubscriptionId { get; set; }

-   internal static bool Parse(JsonRpcRequest<JsonNode> req, [NotNullWhen(true)]out SubscribeRequest? subscription, [NotNullWhen(false)]out SubscriptionError? error, out ErrorCode errorCode) {
-      subscription = null;
+   internal static bool ParseQuery(JsonRpcRequest<JsonNode> req, [NotNullWhen(true)] out ViewStream? stream, [NotNullWhen(true)] out string? subscriptionId, [NotNullWhen(false)] out string? errorMessage, out ErrorCode errorCode) {
+      stream = null;
+      subscriptionId = null;
      errorCode = ErrorCode.None;
      var paramsJsonObj = req.Params?.AsObject();
-      error = null;
      if (paramsJsonObj is null) {
         errorCode = ErrorCode.InvalidSubscriptionRequest;
-         error = new() { ErrorMessage = "'params' object not found" };
+         errorMessage = "'params' object not found";
         return false;
      }
-      subscription = paramsJsonObj.Deserialize<SubscribeRequest>();
-      if (subscription is null) {
+      var subReq = paramsJsonObj.Deserialize<SubscribeRequest>();
+      if (subReq is null) {
         errorCode = ErrorCode.InvalidSubscriptionRequest;
-         error = new() { ErrorMessage = "'params' object invalid" };
+         errorMessage = "'params' object invalid";
         return false;
      }
-      return true;
-   }
-
-   internal bool ParseQuery([NotNullWhen(true)] out ViewStream? stream, [NotNullWhen(false)] out SubscriptionError? error, out ErrorCode errorCode) {
-      stream = null;
-      errorCode = ErrorCode.None;
-      error = null;
-      if (Query is null) {
+      subscriptionId = subReq.SubscriptionId ?? string.Empty;
+      var query = subReq.Query;
+      if (query is null) {
         errorCode = ErrorCode.InvalidSubscriptionRequest;
-         error = new() { ErrorMessage = "'params.query' field not found (expected a string)" };
+         errorMessage = "'params.query' field not found (expected a string)";
         return false;
      }
-      var exp = DataStreamExpressionsInterpreter.Evaluate(Query);
+      errorMessage = null;
+      var exp = DataStreamExpressionsInterpreter.Evaluate(query);
      if (exp is null) {
         errorCode = ErrorCode.InvalidSubscriptionRequest;
-         error = new() { ErrorMessage = "'params.query' expression has invalid syntax" };
+         errorMessage = "'params.query' expression has invalid syntax";
         return false;
      }
-      Exception? exception = null;
+      string? exceptionMsg = null;
      try {
         stream = exp.getView(new Dictionary<string, StatementExpression>(0));
      } catch (Exception ex) {
-         exception = ex;
+         exceptionMsg = ex.ToString();
      }
      if (stream is null) {
         errorCode = ErrorCode.InvalidSubscriptionRequest;
-         error = new() { ErrorMessage = "Failed to create ViewStream from 'params.query' expression (check for typos in arguments/typenames)" };
-         if (exception is not null) {
-            error.ErrorMessage += ". " + exception.Message;
-            error.StackTrace = exception.StackTrace?.Split("\n")?.ToList();
+         errorMessage = "Failed to create ViewStream from 'params.query' expression (check for typos in arguments/typenames)";
+         if (exceptionMsg is not null) {
+            errorMessage += ". Error" + exceptionMsg;
         }
         return false;
      }
@@ -1,11 +0,0 @@
-namespace Bts.DataStreamWebApp.Server.Messages;
-
-using System.Text.Json.Serialization;
-
-public class SubscriptionError {
-   [JsonPropertyName("error_message")]
-   public string? ErrorMessage { get; set; }
-
-   [JsonPropertyName("stack_trace")]
-   public List<string>? StackTrace { get; set; }
-}
@@ -1,82 +1,59 @@
-using Bts.DataStreamCore.Services;
-using Bts.PubSub.Services;
-
-using Spark.Api.Services;
-using Spark.Core.Services;
-
-using System.Reactive.Linq;
-using System.Reflection;
-
-var builder = WebApplication.CreateBuilder(new WebApplicationOptions {
-   Args = args,
-   ApplicationName = "Bts.DataStreamWebApp.Server",
-});
-ConfigureApp(builder.Configuration, builder.Environment.EnvironmentName);
-ConfigureServices(builder.Services, builder.Configuration);
-
-var app = builder.Build();
-var wsOpts = new WebSocketOptions {
-   KeepAliveInterval = TimeSpan.FromMinutes(2)
-};
-if (app.Environment.IsDevelopment()) {
-   app.UseDirectoryBrowser("/browse");
-   app.UseSwagger();
-   app.UseSwaggerUI();
-}
-app.UseHttpsRedirection();
-app.UseWebSockets(wsOpts);
-app.UseDefaultFiles();
-app.UseStaticFiles();
-app.UseAuthorization();
-app.MapControllers();
-try {
-   Console.Write($"Starting Application");
-   app.Run();
-   Console.Write($"Application exited gracefully.");
-} catch (Exception ex) {
-   Console.Error.Write($"Application crashed: {ex}");
-   return 1;
-}
-return 0;
-
-static void ConfigureApp(ConfigurationManager configBuilder, string environment) {
-   // We prefix appsettings with the exe name because multiple exes will usually be in the build dir
-   // (this way, each exe can still load its own appsettings files from the current directory).
-   var exeName = Assembly.GetCallingAssembly().GetName().Name;
-   AddJsonFile($"{exeName}.appsettings.json");
-   AddJsonFile($"{exeName}.appsettings.{environment}.json");
-
-   var cfg = configBuilder.GetSection("config")?.Value;
-   if (!SetBasePath(cfg)) {
-      AddJsonFile(cfg);
-   }
-
-   bool SetBasePath(string? path) {
-      if (!Directory.Exists(path)) {
-         return false;
-      }
-      configBuilder.SetBasePath(path);
-      return true;
-   }
-
-   bool AddJsonFile(string? path) {
-      if (!File.Exists(path)) {
-         return false;
-      }
-      configBuilder.AddJsonFile(path);
-      return true;
-   }
-}
-
-static void ConfigureServices(IServiceCollection services, IConfiguration config) {
-   services.AddDirectoryBrowser()
-      .AddControllers().Services
-      .AddEndpointsApiExplorer()
-      .AddSwaggerGen();
-   services.AddSpaStaticFiles();
-   services.AddSpark(config.GetSection("Spark"))
-      .AddSparkApi(config.GetSection("SparkApi"))
-      .AddMasterWorker(config.GetSection("MasterWorker"))
-      .AddViewStreamDatabase()
-      .AddPubSubClient(config.GetSection("PubSub"));
-}
+using Bts.DataStreamCore.Services;
+using Bts.PubSub.Services;
+
+using Spark.Api.Services;
+
+using System.Reactive.Linq;
+
+var builder = WebApplication.CreateBuilder(new WebApplicationOptions {
+   Args = args,
+   ApplicationName = "Bts.DataStreamWebApp.Server",
+});
+ConfigureApp(builder.Configuration);
+ConfigureServices(builder.Services, builder.Configuration);
+
+var app = builder.Build();
+if (app.Environment.IsDevelopment()) {
+   app.UseDirectoryBrowser();
+}
+var wsOpts = new WebSocketOptions {
+   KeepAliveInterval = TimeSpan.FromMinutes(2)
+};
+app.UseWebSockets(wsOpts);
+app.UseDefaultFiles();
+app.UseStaticFiles();
+if (app.Environment.IsDevelopment()) {
+   app.UseSwagger();
+   app.UseSwaggerUI();
+}
+app.UseHttpsRedirection();
+app.UseAuthorization();
+app.MapControllers();
+try {
+   Console.Write($"Starting Application");
+   app.Run();
+   Console.Write($"Application exited gracefully.");
+} catch (Exception ex) {
+   Console.Error.Write($"Application crashed: {ex}");
+   return 1;
+}
+return 0;
+
+static void ConfigureApp(ConfigurationManager configBuilder) {
+   configBuilder.AddCommandLine(Environment.GetCommandLineArgs());
+   var cfgRoot = configBuilder.GetSection("config");
+   if (cfgRoot?.Value is not null) {
+      configBuilder.AddJsonFile(cfgRoot.Value);
+   }
+}
+
+static void ConfigureServices(IServiceCollection services, IConfiguration config) {
+   services.AddDirectoryBrowser();
+   services.AddControllers();
+   services.AddEndpointsApiExplorer();
+   services.AddSwaggerGen();
+   services.AddMasterWorker(config.GetSection("MasterWorker"));
+   services.AddViewStreamDatabase(config.GetSection("ViewStreamDatabase"));
+   services.AddPubSubClient(config.GetSection("PubSub"));
+   services.AddSparkd(config.GetSection("Sparkd"));
+}
@@ -6,9 +6,8 @@
      "dotnetRunMessages": true,
      "launchBrowser": true,
      "launchUrl": "swagger",
-      "applicationUrl": "http://localhost:8080",
+      "applicationUrl": "http://localhost:5000",
      "environmentVariables": {
-        "ASPNETCORE_HTTP_PORT": "8080",
        "ASPNETCORE_ENVIRONMENT": "Development",
        "ASPNETCORE_HOSTINGSTARTUPASSEMBLIES": "Microsoft.AspNetCore.SpaProxy",
      }
@@ -17,10 +16,8 @@
      "commandName": "Project",
      "dotnetRunMessages": true,
      "launchBrowser": true,
-      "applicationUrl": "http://localhost:8080;https://localhost:8443",
+      "applicationUrl": "http://localhost:5000;https://localhost:5001",
      "environmentVariables": {
-        "ASPNETCORE_HTTP_PORT": "8080",
-        "ASPNETCORE_HTTPS_PORT": "8443",
        "ASPNETCORE_ENVIRONMENT": "Development",
        "ASPNETCORE_HOSTINGSTARTUPASSEMBLIES": "Microsoft.AspNetCore.SpaProxy",
      }
@@ -1,52 +0,0 @@
-{
-  "PubSub": {
-    "Routing": {
-      "PublisherRoutes": [
-        {
-          "Name": "default",
-          "ServerAddress": "bts-cloud://localhost:22019",
-          "TagRegExp": ".*"
-        }
-      ],
-      "SubscriberRoutes": [
-        {
-          "Name": "default",
-          "ServerAddress": "bts-cloud://localhost:22019",
-          "TagRegExp": ".*"
-        }
-      ]
-    }
-  },
-  "CannedQueries": [
-    {
-      "Name": "BBO",
-      "Query": [
-        "bbo"
-      ]
-    },
-    {
-      "Name": "Trades",
-      "Query": [
-        "timesales"
-      ]
-    },
-    {
-      "Name": "Instruments",
-      "Query": [
-        "instruments"
-      ]
-    },
-    {
-      "Name": "Books",
-      "Query": [
-        "books"
-      ]
-    },
-    {
-      "Name": "Benzinga News",
-      "Query": [
-        "benzinga_news"
-      ]
-    }
-  ]
-}
@@ -1,8 +0,0 @@
-{
-  "Spark": {
-    "ConfigServer": "http://sparkd:8000/config"
-  },
-  "SparkApi": {
-    "IsEnabled": true
-  }
-}
@@ -1,44 +0,0 @@
-{
-  "CannedQueries": [
-    {
-      "Name": "BBO",
-      "Query": [
-        "bbo"
-      ]
-    },
-    {
-      "Name": "Trades",
-      "Query": [
-        "timesales"
-      ]
-    },
-    {
-      "Name": "Instruments",
-      "Query": [
-        "instruments"
-      ]
-    },
-    {
-      "Name": "Books",
-      "Query": [
-        "books"
-      ]
-    },
-    {
-      "Name": "Benzinga News",
-      "Query": [
-        "benzinga_news"
-      ]
-    }
-  ],
-  "Spark": {
-    "ConfigServer": "http://sparkd:8000/config"
-  },
-  "SparkApi": {
-    "IsEnabled": true,
-    "Credentials": {
-      "User": "bts",
-      "Password": ""
-    }
-  }
-}
@@ -5,81 +5,5 @@
      "Microsoft.AspNetCore": "Warning"
    }
  },
-  "AllowedHosts": "*",
-  "Kestrel": {
-    "Endpoints": {
-      "Http": {
-        "Url": "http://0.0.0.0:8080"
-      },
-      "Https": {
-        "Url": "https://0.0.0.0:8443"
-      }
-    }
-  },
-  "MasterWorker": {
-    "Interval": "00:00:00.1"
-  },
-  "ViewStreamDatabase": {
-    "Interval": "00:00:00.1"
-  },
-  "SparkApi": {
-    "IsEnabled": false,
-    "Credentials": {
-      "User": "bts",
-      "Password": ""
-    }
-  },
-  "PubSub": {
-    "HeartbeatInterval": 15000,
-    "MissingHeartbeatsBeforeGc": 50,
-    "ServerPort": 22019,
-    "Routing": {
-      "PublisherRoutes": [
-        {
-          "Name": "default",
-          "ServerAddress": "bts-cloud://pubsub:22019",
-          "TagRegExp": ".*"
-        }
-      ],
-      "SubscriberRoutes": [
-        {
-          "Name": "default",
-          "ServerAddress": "bts-cloud://pubsub:22019",
-          "TagRegExp": ".*"
-        }
-      ]
-    },
-    "TagTypeToAssemblyMap": {
-      "Entries": [
-        {
-          "HandlerClass": "BCG.PubSubServerTagHandler",
-          "Assembly": "DataStreamCore.dll"
-        }
-      ]
-    },
-    "TagHandlerFactory": {
-      "Handlers": [
-        {
-          "TagType": "DataStream",
-          "Assembly": "DataStreamCore",
-          "Class": "bts.PubsubServerStreamTagHandler"
-        },
-        {
-          "TagType": "System_Sub",
-          "Assembly": "PubSub",
-          "Class": "BCG.SystemSubscriptionMapPublisher"
-        },
-        {
-          "TagType": "System_Stats",
-          "Assembly": "PubSub",
-          "Class": "BCG.SystemStatsPublisher"
-        },
-        {
-          "TagType": ".*",
-          "Assembly": "DataStreamCore",
-         "Class": "bts.PubsubServerGenericDataHandler"
-        }
-      ]
-    }
-  }
+  "AllowedHosts": "*"
 }
@@ -1,15 +0,0 @@
-Logging {
-  value {
-    console {
-      default_level: "Info"
-    }
-    syslog {
-      default_level: "Info"
-    }
-    file {
-      default_level: "Info"
-    }
-    initial_queue_size: 120001
-    max_queue_growth: 4
-  }
-}
@@ -1,13 +0,0 @@
-<Project Sdk="Microsoft.NET.Sdk">
-
-  <PropertyGroup>
-    <TargetFramework>net7.0</TargetFramework>
-    <ImplicitUsings>enable</ImplicitUsings>
-    <Nullable>enable</Nullable>
-  </PropertyGroup>
-
-  <ItemGroup>
-    <PackageReference Include="Microsoft.Extensions.Hosting" Version="7.0.1" />
-  </ItemGroup>
-
-</Project>
@@ -1,59 +0,0 @@
-namespace Bts.Extensions.Hosting;
-
-using Microsoft.Extensions.Configuration;
-using Microsoft.Extensions.Hosting;
-
-using System.Diagnostics;
-
-public static class HostBuilderExtensions {
-   public static IHostBuilder CreateBtsBuilder(string[] args) {
-      // Obtain calling assembly name here, rather than in the lambda where it is actually needed,
-      // because want the name of the assembly that called this function, not the assembly that will eventually invoke
-      // the lambda...that will be the Microsoft.Extensions.Hosting assembly.
-      // Get the name of the assembly that called this function. That will act as the "appsettings"
-      // json file prefix
-      var exeName = GetCallingAssemblyName();
-      return Host.CreateDefaultBuilder(args)
-         .ConfigureAppConfiguration((context, configBuilder) =>
-            ConfigureApp(configBuilder, context.HostingEnvironment.EnvironmentName, exeName));
-   }
-
-   public static void ConfigureApp(IConfigurationBuilder configBuilder, string? environment = null, string? prefix = null) {
-      var baseFileName = prefix is null ? "appsettings" : $"{prefix}.appsettings";
-      AddJsonFile($"{baseFileName}.json");
-      if (environment is not null) {
-         AddJsonFile($"{baseFileName}.{environment}.json");
-      }
-
-      var cfg = configBuilder.Build().GetSection("config")?.Value;
-      if (!SetBasePath(cfg)) {
-         AddJsonFile(cfg);
-      }
-
-      bool SetBasePath(string? path) {
-         if (!Directory.Exists(path)) {
-            return false;
-         }
-         configBuilder.SetBasePath(path);
-         return true;
-      }
-
-      bool AddJsonFile(string? path) {
-         if (!File.Exists(path)) {
-            return false;
-         }
-         configBuilder.AddJsonFile(path);
-         return true;
-      }
-   }
-
-   public static string GetCallingAssemblyName() =>
-      new StackTrace()
-         .GetFrame(2)
-         ?.GetMethod()
-         ?.Module
-         ?.Assembly
-         ?.GetName()
-         ?.Name
-         ?? throw new InvalidOperationException("Failed to determine name of calling assembly (calling method is null)");
-}
@@ -1,9 +0,0 @@
-<Project Sdk="Microsoft.NET.Sdk">
-
-  <PropertyGroup>
-    <TargetFramework>net7.0</TargetFramework>
-    <ImplicitUsings>enable</ImplicitUsings>
-    <Nullable>enable</Nullable>
-  </PropertyGroup>
-
-</Project>
@@ -1,28 +0,0 @@
-namespace Bts.Net;
-
-using System.Linq;
-using System.Net;
-
-/// <summary>
-/// Provides facilities to compare hosts for equality.
-/// <para>
-/// This class doesn't implement IEqualityComparer because a consistent hash code cannot be
-/// guarenteed for equivalent host strings. The equals impl relies on Dns.GetHostAddresses, which
-/// is non-determinstic (implementations that depends on mutable properties face a similar problem).
-/// </para>
-/// </summary>
-public class HostComparer {
-   public static bool Equals(string? x, string? y) {
-      if (x is null) {
-         return y is null;
-      }
-      if (y is null) {
-         return false;
-      }
-      return Dns.GetHostAddresses(x).Intersect(
-         Dns.GetHostAddresses(y), IpAddressComparer.Instance).Any();
-   }
-
-   public static bool Equals(IPHostEntry x, IPHostEntry y) =>
-      x.AddressList.Intersect(y.AddressList, IpAddressComparer.Instance).Any();
-}
@@ -1,30 +0,0 @@
-namespace Bts.Net;
-
-using System.Diagnostics.CodeAnalysis;
-using System.Net;
-
-/// <summary>
-/// Compares ip addresses. Unlike the default comparer, v4 and equivalent v6 addresses are
-/// considered equal.
-/// </summary>
-public class IpAddressComparer : IEqualityComparer<IPAddress> {
-   public static IEqualityComparer<IPAddress> Instance { get; } = new IpAddressComparer();
-
-   public bool Equals(IPAddress? x, IPAddress? y) {
-      if (x is null) {
-         return y is null;
-      }
-      if (y is null) {
-         return false;
-      }
-      return x.Equals(y) ||
-         (x.IsIPv4MappedToIPv6 && x.MapToIPv4().Equals(y)) ||
-         (y.IsIPv4MappedToIPv6 && y.MapToIPv4().Equals(x));
-   }
-
-   public int GetHashCode([DisallowNull] IPAddress address) =>
-      address.IsIPv4MappedToIPv6
-         // Return consistent hash code for v4 and equivalent v6
-         ? address.MapToIPv4().GetHashCode()
-         : address.GetHashCode();
-}
@@ -1,33 +0,0 @@
-namespace Bts.Net;
-
-using System;
-using System.Net;
-using System.Text.Json;
-using System.Text.Json.Serialization;
-
-public class IpHostEntryJsonConverter : JsonConverter<IPHostEntry> {
-   public override IPHostEntry? Read(ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options) {
-      throw new NotImplementedException();
-   }
-
-   public override void Write(Utf8JsonWriter writer, IPHostEntry value, JsonSerializerOptions options) {
-      writer.WriteStartObject();
-      // HostName
-      writer.WriteString(nameof(value.HostName), value.HostName);
-      // Aliases
-      writer.WritePropertyName("Aliases");
-      writer.WriteStartArray();
-      foreach (var alias in value.Aliases) {
-         writer.WriteStringValue(alias);
-      }
-      writer.WriteEndArray();
-      // Addresses
-      writer.WritePropertyName("Addresses");
-      writer.WriteStartArray();
-      foreach (var address in value.AddressList) {
-         writer.WriteStringValue(address.ToString());
-      }
-      writer.WriteEndArray();
-      writer.WriteEndObject();
-   }
-}
@@ -1,39 +0,0 @@
-namespace Bts.Net;
-
-using System;
-using System.IO;
-using System.Net;
-using System.Text;
-using System.Text.Json;
-
-/// <summary>
-/// This class seriailzes IpHostEntry to json (string). A common buffer is used to reduce memory
-/// allocations on repeated calls to <see cref="Serialize"/>. For that reason, this class is NOT
-/// thread safe.
-/// </summary>
-public class IpHostEntryJsonSerializer {
-   readonly MemoryStream buffer = new();
-   readonly IpHostEntryJsonConverter converter;
-
-   public IpHostEntryJsonSerializer(IpHostEntryJsonConverter? converter = null) =>
-      this.converter = converter ?? new() { };
-
-   public string? Serialize(IPHostEntry dp, JsonSerializerOptions? opts = null) {
-      opts ??= JsonSerializerOptions.Default;
-      buffer.Position = 0;
-      buffer.SetLength(0);
-      using (var jsonWriter = new Utf8JsonWriter(buffer)) {
-         converter.Write(jsonWriter, dp, opts);
-      }
-      var bytes = buffer.GetBuffer();
-      var span = new ArraySegment<byte>(bytes, 0, (int)buffer.Length);
-      return Encoding.UTF8.GetString(span);
-   }
-
-   public void Serialize(IPHostEntry dp, Utf8JsonWriter jsonWriter, JsonSerializerOptions? opts = null) {
-      opts ??= JsonSerializerOptions.Default;
-      buffer.Position = 0;
-      buffer.SetLength(0);
-      converter.Write(jsonWriter, dp, opts);
-   }
-}
@@ -1,94 +0,0 @@
-namespace Bts.Net;
-
-using System.Diagnostics.CodeAnalysis;
-using System.Net;
-using System.Text.RegularExpressions;
-
-public static partial class IpUtils {
-   static readonly Regex ipv4Regex = new(string.Join('.', @"(((0|1)\d)|(2\d)|\d)?\d"));
-
-   // public static void ParseUri(string str, out string? scheme, out string? host, out string? port, out string? path, out string? queryParams) {
-   //    if (Uri.TryCreate(str, UriKind.Absolute, out var uri)) {
-   //    }
-   // }
-
-   // public static bool AreEndpointsEqual(Uri? x, Uri? y) {
-   //    if (x is null) {
-   //       return y is null;
-   //    }
-   //    if (y is null) {
-   //       return false;
-   //    }
-   //    if (x.IsAbsoluteUri != y.IsAbsoluteUri) {
-   //       return false;
-   //    }
-   //    if (x.IsAbsoluteUri) {
-   //       if (x.Scheme != y.Scheme) {
-   //          return false;
-   //       }
-   //    }
-   //    string? xHost = null;
-   // }
-
-   // static string ResolveHostIpv4(Uri uri) {
-   //    // switch (uri.HostNameType) {
-   //    //    case UriHostNameType.IPv4:
-   //    //       return uri.Host;
-   //    //    case UriHostNameType.Dns:
-   //    //       break;
-   //    //    case UriHostNameType.IPv6:
-   //    //       var addresses = Dns.GetHostAddresses(uri.Host);
-   //    //       foreach (var address in addresses) {
-   //    //          if (address.IsIPv4MappedToIPv6) {
-   //    //             return address.MapToIPv4().Address;
-   //    //          }
-   //    //       }
-   //    //       break;
-   //    // }
-   //    var hostAddresses = Dns.GetHostAddresses(uri.Host);
-   //    foreach (IPAddress address in hostAddresses) {
-   //       if (address.Equals(remoteEndPoint.Address) ||
-   //           (address.IsIPv4MappedToIPv6 && address.MapToIPv4().Equals(remoteEndPoint.Address)) ||
-   //           (remoteEndPoint.Address.IsIPv4MappedToIPv6 && remoteEndPoint.Address.MapToIPv4().Equals(address))) {
-   //          isMatch = true;
-   //          break;
-   //       }
-   //    }
-   // }
-
-   // public static bool AreEndpointsEqual(string? x, string? y) {
-   //    if (x == y) {
-   //       return true;
-   //    }
-   //    if (x is null) {
-   //       return y is null;
-   //    }
-   //    if (y is null) {
-   //       return false;
-   //    }
-   //    var xUri = new Uri(x);
-   //    var yUri = new Uri(y);
-   //    // return xUri.Equals(yUri);
-   // }
-
-   public static bool IsIpv4(string host) {
-      host = removeSchemaFromUrl(host);
-      host = removePortFromUrl(host);
-      return ipv4Regex.IsMatch(host);
-   }
-
-   static string removeSchemaFromUrl(string url) {
-      var i = url.IndexOf("://");
-      return i < 0 ? url : url.Substring(i + 3);
-   }
-
-   static string removePortFromUrl(string url) {
-      var match = MyRegex().Match(url);
-      return match.Success
-         ? string.Concat(url.AsSpan(0, match.Index), url.AsSpan(match.Index + match.Length))
-         : url;
-   }
-
-   [GeneratedRegex(":\\d{1,4}")]
-   private static partial Regex MyRegex();
-}
@@ -1,17 +1,20 @@
 <Project Sdk="Microsoft.NET.Sdk">

  <PropertyGroup>
-    <TargetFramework>net8.0</TargetFramework>
+    <TargetFramework>net7.0</TargetFramework>
    <ImplicitUsings>enable</ImplicitUsings>
    <Nullable>enable</Nullable>
  </PropertyGroup>

  <ItemGroup>
+    <PackageReference Include="Google.Protobuf" Version="3.24.3" />
    <PackageReference Include="YaccLexTools" Version="1.2.2" />
  </ItemGroup>

  <ItemGroup>
+    <ProjectReference Include="..\DataStreamCore\DataStreamCore.csproj" />
    <ProjectReference Include="..\ExpressionCore\ExpressionCore.csproj" />
+    <ProjectReference Include="..\..\client\spark_core\spark_core.csproj" />
  </ItemGroup>

  <ItemGroup Label="ColumnMapExpression">
@@ -1,11 +0,0 @@
-namespace bts.BlueOrb {
-   using BCG;
-
-   [DataStreamUserRespresentation("AddColumnsStream")]
-   public class AddColumnsStream: ViewStream {
-      public AddColumnsStream(ViewStream parent_str, string add_col_expr) {
-         ViewStream rs = new BCG.AddColumnsStream(parent_str, add_col_expr);
-         mySetRealSource(rs);
-      }
-   }
-}
Author	SHA1	Message	Date
John Colvin	ba664b95b8	fix use dep injection in bts.datastreamwebapp.server	2024-07-11 11:06:29 -04:00
John Colvin	d6f370f862	fix Add spark service modules designed for DependencyInjection	2024-07-11 11:05:23 -04:00
John Colvin	079826f4de	Fix mismatched types in PubsubHeartbeatStream schema	2024-07-11 10:59:41 -04:00
John Colvin	2eee3cdaab	Change DataStreamPublisher json fields to match cs class property names verbatim	2024-07-11 10:43:24 -04:00
John Colvin	5e3215ee52	fix: Always call 'handle_internal_pubs' in PubSubMessageReactor main loop	2024-07-11 10:38:30 -04:00
John Colvin	7739cc2f3d	Use Dependency Injection in Bts.DataStreamWebApp.Server	2024-07-11 10:37:29 -04:00
John Colvin	8347a7c126	refactor: Change Bts.DataStreamWebApp.Server framework from net8 -> net7 Although it would build on net8, all other cs code targets net7. Furthermore, VS Code intellisense seems less reliable/crashes frequently when the sln file targets both frameworks).	2024-07-11 10:37:28 -04:00
John Colvin	fbfc960df6	improvement: Make DataStreamSubscription an IHostedService To properly manage its life cycle	2024-07-11 10:37:28 -04:00
John Colvin	451f721030	improvement: Accept CancellationToken, catch exceptions when running ds websocket server	2024-07-11 10:37:28 -04:00
John Colvin	bb992c0410	Fix mismatched types in PubSubServerTagStatsStream schema	2024-07-11 10:37:28 -04:00
John Colvin	db18aebf92	Add DataStreamPublisher service modules designed for Dependency Injection	2024-07-11 10:37:28 -04:00
John Colvin	8127c9c9fe	refactor: Decouple ViewPublisher from IPubTableBroker impl	2024-07-11 10:37:28 -04:00
John Colvin	1825b339b1	Change PubSubServer json fields to match cs class property names verbatim	2024-07-11 10:37:28 -04:00
John Colvin	03ab3fbe90	Get rid of JsonPropertyName attributes in PubSubConfig The Dependency Injection framework (based around dotnet ApplicationBuilder) does not support this attribute, and maps property names verbatim to json fields.	2024-07-11 10:37:28 -04:00
John Colvin	b61bf16a42	Add PubSub service modules designed for Dependency Injection	2024-07-11 10:37:28 -04:00
John Colvin	d969489bf4	refactor: Decouple RemoteQueryManager from IManagedSubscriber, IPubTableBroker impl	2024-07-11 10:37:28 -04:00
John Colvin	9cd4974f49	refactor: Decouple ManagedSubscriber from ISubscriberCore impl	2024-07-11 10:37:28 -04:00
John Colvin	793634c8cb	refactor: Only support 1 tag type in IPubSubServerTagHandler	2024-07-11 10:37:28 -04:00
John Colvin	eae20b6c94	refactor: Decouple PubTableBroker from IPublisherCore impl Reference interface rather than the PublisherCore impl.	2024-07-11 10:37:28 -04:00
John Colvin	2848bb6a5d	improvement: Accept CancellationToken in JsonRpcRequestObservable	2024-07-11 10:37:16 -04:00
John Colvin	687324998b	Add spark service modules designed for Dependency Injection	2024-07-11 10:36:04 -04:00
John Colvin	97cebd62d4	fix: Dispose market data sub in various md streams	2024-07-11 10:36:04 -04:00
John Colvin	492ca0c28f	cleanup: Remove unused pubsub tag handler code * Removed 'IPubSubServerTagHandler.register' - All implementations of this function were empty * Removed 'IPubSubServerTagHandler.supportedTagTypes' - This property wasn't referenced anywhere * Remove IWorker inheritance from IPubSubServerTagHandler - No reason these interfaces have to be tied together	2024-07-10 13:35:18 -04:00
John Colvin	baa5019039	refactor: Rename CloudStreamManager.cs -> RemoteQueryManager.cs This matches the file name	2024-07-10 01:18:21 -04:00
John Colvin	ac2d31e1fc	fix build: Update vcpkg On some vs versions, vcpkg cmake scripts would fail w/ an error saying "Could not locate a complete Visual Studio instance"	2024-07-09 15:16:36 -04:00
John Colvin	9821134bc3	Change pub sub schema prefix 'bluto://' -> 'bts-cloud://' In config json files (not actual code change)	2024-07-09 15:14:30 -04:00
John Colvin	9cf892322d	fix build: Fix prop names after refactoring PubSubRoute * Change 'server_address' -> 'ServerAddress' * Change 'tag_reg_exp' -> 'TagRegExp'	2024-07-09 15:14:30 -04:00
John Colvin	77bf91be1e	build: Add scripts to publish data stream apps	2024-07-09 15:14:30 -04:00
John Colvin	63b064ac79	Fix typo in log msg	2024-07-09 15:14:30 -04:00
John Colvin	b617775c29	Fix net7 -> net8 typo in publish_data_stream_web_app.py The DataStream Web App is on net8	2024-07-09 15:14:30 -04:00
John Colvin	4f7d77420c	Fix build scripts so they propagate the exit code back to the shell I noticed this was broken because launch.json configurations would still start the debugger even when the preLaunchTask (build) failed!	2024-07-09 15:14:30 -04:00
John Colvin	3ba6a91ca7	vscode: Add DataStreamPublisher debug cfg to skip build	2024-07-09 15:14:30 -04:00
John Colvin	0abde9eb7c	build: Use separate cmd to build DataStreamSuite/QueryAnalyzer.sln The DataStreamSuite.sln builds the new webapp now, which will (eventually) replace DSQA. We'll keep it around for now, but with a clean separation from the main build.	2024-07-09 15:14:30 -04:00
John Colvin	1531a57bc5	feat: Add DataStream WebApp Included in DataStreamSuite.sln, but not DataStreamQueryAnalyzer.sln	2024-07-09 15:14:29 -04:00
John Colvin	a62ea64c5e	build: Copy libspark to all dotnet dirs in single cmake target This is cleaner than the previous impl, which used separate build targets to copy libspark to "net7.0", and "net7.0-windows" (dirs). Note: I decided to clean this up now because I'm about to upgrade the new DataStream WebApp to net8.0, and did not want to add yet another build target to copy libspark to "net8.0".	2024-07-09 15:14:29 -04:00
John Colvin	450773f588	fix: Parse ServerAddress as normal string The "JsonValueOrArrayConverter<T>" doesn't work the way I thought it would. When it is associated with a property via the JsonConverter attribute, dotnet json parsing code throws an exception if "T" isn't assignable to the property type.	2024-07-09 15:14:29 -04:00
John Colvin	f09d6badaf	fix build (api server): Change namespace to bts.websocket_core	2024-07-09 15:14:29 -04:00
John Colvin	6b054e0718	Remove docker submodules again	2024-07-09 15:14:29 -04:00
John Colvin	5e2789d16e	Change externals url from ssh to https style So we don't need ssh keys to clone it (the builtin credential manager makes https authentication safe/easy)	2024-07-09 15:14:29 -04:00
John Colvin	9ebdb2e460	cleanup: Fix benign typos in build scripts Change BUILD_SPARK_PY var name to BUILD_PY (because these scripts to actually build 'spark')	2024-07-09 15:14:29 -04:00
John Colvin	d7bf82722b	fix build (warn): Only use Registry on windows	2024-07-09 15:14:29 -04:00
John Colvin	e1526e25a7	cleanup: Fix numerous build warnings in DataStreamSuite Nearly all warnings were due to nullability issues, since the concept of nullable reference types did not exist when the DataStream code was originally written. * PubSubClientMessage - Normalize indentation/whitespace - Fix some typos - Address nullability issues to fix build warnings - Modernize code * Compression related files - ZipEntry - ZipFile - ZipInputStream - TarHeader * RemoteMessage - make 'message' nullable to resolve warnings elsewhere (in calling code) * PubSubSingleTagStateEvent.cs * PubSubMessageReactor * PubSubState * SystemHeartbeatPublisher * SystemStatsPublisher * SingleTagData * ManagedTCPMessageHub * PubsubDeserializationController * PublisherCore * InflaterHuffmanTree * AsyncTCPConnection_APM * ThreadSafeQueue * ManagedSubscriber * SubscriberCore * PubSubServerManagedTCPConnection * AsyncTCPConnection_APM * SuperMapStream * MapStream * TradeBarsStream	2024-07-09 15:14:26 -04:00
John Colvin	089ba9b369	cleanup: Normalize indentation in TradeBarsStream	2024-07-09 15:09:02 -04:00
John Colvin	567976043f	cleanup: Disable nullability warnings * BZip2InputStream * TarInputStream * Inflater * InflaterDynHeader * BZip2OutputStream * DeflaterHuffman * ParallelJoinTest	2024-07-09 15:07:24 -04:00
John Colvin	2fd3382939	cleanup: Modernize PubSubRoute code, json parsing	2024-07-09 15:06:34 -04:00
John Colvin	a96f1c8f17	cleanup: Remove unused DateTime grammar from exp parser	2024-07-09 15:03:30 -04:00
John Colvin	6f623f9a37	Add JsonRpcRequestObservable, AsyncEnumerable	2024-07-09 15:03:30 -04:00
John Colvin	827189cdb4	Add json facilities for DataStream websocket api	2024-07-09 15:03:30 -04:00
John Colvin	123f12ab88	refactor: Add handy WebsocketExtensions * IsSocketStateOpen * IsClosedProperly	2024-07-09 14:59:06 -04:00
John Colvin	f51ba8fb56	feat: Add JsonRpc types for websocket paradigm	2024-07-09 14:59:06 -04:00
John Colvin	2506fb4f2b	cleanup: Remove empty file (clearly added by accident)	2024-07-09 14:59:06 -04:00
John Colvin	f92964114e	fix: Fix improper 'OfType' linq cast Previously, animations were expressed through polymorphism (derived from the base class CellAnimation), but the new impl uses composition. This 'OfType' call was leftover from the polymorphism implementation, an attempt to linq cast elements of base class (CellAnimation) to BackgroundAnimation.	2024-07-09 14:59:06 -04:00
John Colvin	baa0f1430b	editorconfig: Use standard c# capitalization rules * Use 'PascalCase' for public members (instead of 'snake_case') * Use 'camelCase' for private members (instead of '_snake_case')	2024-07-09 14:59:06 -04:00
John Colvin	7104bd787a	refactor: Add WebsocketClient class Basically an ISubject (I/O) wrapper for IWebsocketConnectionGateway	2024-07-09 14:59:06 -04:00
John Colvin	5db92d1f7d	refactor: Change WebsocketServerConfig.port to uint32 It should always be a number between 1-65535	2024-07-09 14:59:06 -04:00
John Colvin	eb0fcf0b84	refactor: Rename WebsocketServerConfig proto pkg 'bts.websocket_core' This matches the namespace used in spark_websocket csproj. Eventually, I'd like to use conventional capitalization (TitleCase namespaces/types, etc.), but let's shelf that until we have enough time/energy to do a more comprehensive refactoring.	2024-07-09 14:59:06 -04:00
John Colvin	a52b187313	refactor: Remove nullability constraint in GetValueOrDefault	2024-07-09 14:59:06 -04:00
John Colvin	669b059eb2	refactor: Use events to avoid circular deps for ws sync/gateway	2024-07-09 14:59:06 -04:00
John Colvin	36ab8492d3	refactor: Let caller specify nullability of T in json parse fn	2024-07-09 14:48:59 -04:00
John Colvin	54f37b2401	Merge remote-tracking branch 'origin/development' into development.ps_linux	2024-07-09 14:46:18 -04:00
				`@@ -1 +0,0 @@`
				`PYTHONPATH=/workspaces/spark/build/sparkd-debug-clang/src/lib_python_interop;/workspaces/spark/src/lib_python_interop/python_interop_plugins`
				`@@ -1 +0,0 @@`
				`VITE_DATASTREAM_WEBSOCKET_URL=wss://localhost:8443/ws`