check point (bc i don't trust googledrive sync yet)

dev: Add gitpod.yml for cloud workspaces
externals: Change repo url to use read-only access token
2024-02-09 23:06:26 -06:00 · 2024-02-09 23:06:26 -06:00 · 2024-02-09 23:06:26 -06:00 · 2024-02-09 23:06:26 -06:00 · 2024-02-09 23:06:26 -06:00 · 2024-02-09 23:06:26 -06:00
3019 changed files with 67398 additions and 83461 deletions
@@ -25,12 +25,6 @@
      "commands": [
        "fake"
      ]
-    },
-    "dotnet-t4": {
-      "version": "3.0.0",
-      "commands": [
-        "t4"
-      ]
    }
  }
-}
+}
@@ -1,133 +0,0 @@
-FROM ubuntu:22.04
-
-ARG DEBIAN_FRONTEND=noninteractive
-
-# Note installations that take a while (more than 10s)
-
-RUN apt-get update
-RUN apt-get install -y ca-certificates
-RUN apt-get install -y ccache
-RUN apt-get install -y curl
-RUN apt-get install -y fish
-RUN apt-get install -y gdb
-# git-all: ~250s
-RUN apt-get install -y git-all
-RUN apt-get install -y gnupg
-# liblttng-ust-dev: ~30s
-RUN apt-get install -y liblttng-ust-dev
-RUN apt-get install -y libssh2-1-dev
-RUN apt-get install -y locales
-RUN apt-get install -y ninja-build
-# openssh-server: ~40s
-RUN apt-get install -y openssh-server
-RUN apt-get install -y pkg-config
-RUN apt-get install -y software-properties-common
-RUN apt-get install -y sudo
-RUN apt-get install -y tzdata
-RUN apt-get install -y vim
-RUN apt-get install -y zip
-
-# Python: ~30s
-RUN apt-get install -y \
-   python3-pip \
-   python3-venv \
-   && python3 -m pip install --upgrade pip
-
-# GCC
-ARG gcc_major=13
-RUN \
-   add-apt-repository ppa:ubuntu-toolchain-r/test \
-   && apt-get update \
-   && apt-get install -y g++-$gcc_major
-
-# Clang
-ARG clang_major=17
-RUN \
-   curl -fsSL https://apt.llvm.org/llvm-snapshot.gpg.key 2> /dev/null \
-   | gpg --dearmor - \
-   | tee /usr/share/keyrings/llvm-snapshot.gpg > /dev/null \
-   && echo "deb [signed-by=/usr/share/keyrings/llvm-snapshot.gpg] http://apt.llvm.org/$(lsb_release -cs)/ llvm-toolchain-$(lsb_release -cs)-$clang_major main" \
-   | tee -a "/etc/apt/sources.list.d/llvm.list" \
-   && apt-get update \
-   && apt-get install -y \
-   clang-$clang_major \
-   clang-format-$clang_major \
-   clangd-$clang_major
-
-# CMake
-RUN \
-   curl -fsSL https://apt.kitware.com/keys/kitware-archive-latest.asc 2> /dev/null \
-   | gpg --dearmor - \
-   | tee /usr/share/keyrings/kitware-archive-keyring.gpg > /dev/null \
-   && echo "deb [signed-by=/usr/share/keyrings/kitware-archive-keyring.gpg] https://apt.kitware.com/ubuntu/ $(lsb_release -cs) main" \
-   | tee /etc/apt/sources.list.d/kitware.list > /dev/null \
-   && apt-get update \
-   && apt-get install -y cmake
-
-# Dotnet
-RUN apt-get update && apt-get install -y dotnet7
-
-# Node
-ARG node_major=18
-RUN mkdir -p /etc/apt/keyrings \
-   && curl -fsSL https://deb.nodesource.com/gpgkey/nodesource-repo.gpg.key \
-   | gpg --dearmor -o /etc/apt/keyrings/nodesource.gpg \
-   && echo "deb [signed-by=/etc/apt/keyrings/nodesource.gpg] https://deb.nodesource.com/node_$node_major.x nodistro main" \
-   | tee /etc/apt/sources.list.d/nodesource.list \
-   && apt-get update \
-   && apt-get install -y nodejs
-
-# Emumba (deps)
-RUN apt-get update && apt-get -y install rapidjson-dev
-# Emumba
-COPY emumba /emumba
-RUN dpkg -i /emumba/emumba-deribit-connector-3.3.1-Linux.LICENSE_DISABLED.deb \
-    && rm -r /emumba
-
-# Pcap
-RUN apt-get update && apt-get install -y libpcap0.8-dev
-
-# Databento (deps)
-RUN apt-get update && apt-get -y install libzstd-dev
-
-# Docker in Docker
-RUN apt-get update && apt-get install -y \
-    docker.io \
-    docker-compose \
-    docker-compose-v2
-
-# Set up default executables (update-alternatives)
-RUN \
-   update-alternatives --install /usr/bin/clang clang /usr/bin/clang-$clang_major 100; \
-   update-alternatives --install /usr/bin/clang++ clang++ /usr/bin/clang++-$clang_major 100; \
-   update-alternatives --install /usr/bin/clang-format clang-format /usr/bin/clang-format-$clang_major 100; \
-   update-alternatives --install /usr/bin/clangd clangd /usr/bin/clangd-$clang_major 100; \
-   update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-$gcc_major 100; \
-   update-alternatives --install /usr/bin/g++ g++ /usr/bin/g++-$gcc_major 100
-
-# Add compiler symlinks for ccache 
-RUN \
-   ln -s /usr/bin/ccache /usr/local/bin/clang; \
-   ln -s /usr/bin/ccache /usr/local/bin/clang++; \
-   ln -s /usr/bin/ccache /usr/local/bin/clang-$clang_major; \
-   ln -s /usr/bin/ccache /usr/local/bin/clang++-$clang_major; \
-   ln -s /usr/bin/ccache /usr/local/bin/gcc; \
-   ln -s /usr/bin/ccache /usr/local/bin/g++
-
-RUN locale-gen en_US.UTF-8
-
-ARG docker_user="bts"
-ARG docker_uid="1000"
-ARG docker_pass="bts"
-
-RUN \
-   groupadd -g "$docker_uid" "$docker_user" \
-   && useradd \
-   -m \
-   -u "$docker_uid" \
-   -g "$docker_uid" \
-   -G sudo \
-   --password "$(openssl passwd -1 "$docker_pass")" \
-   "$docker_user"
-
-RUN usermod -aG docker "$docker_user"
@@ -1,43 +0,0 @@
-{
-  "dockerFile": "Dockerfile",
-  // Creates local dir to mount to HOME in docker 
-  "initializeCommand": "mkdir -p ${localEnv:HOME}/.bts/devcontainer-mounts/${devcontainerId}",
-  "hostRequirements": {
-    "memory": "4gb",
-    "storage": "6gb"
-  },
-  "postCreateCommand": "/home/bts/spark/docker/sparkd/dev/devcontainer_post_create.sh",
-  "containerUser": "bts",
-  "workspaceMount": "source=${localWorkspaceFolder},target=/home/bts/spark,type=bind",
-  "workspaceFolder": "/home/bts/spark",
-  "updateRemoteUserUID": true,
-  "shutdownAction": "none",
-  // Note: If running multiple containers on the same host, each must use a
-  // unique set of ports. Usually we just add 1000 to each port as needed. For
-  // example, if 8000-8003 is taken, go with 9000-9003, then 10000-10003, etc.
-  "runArgs": [
-    "--shm-size",
-    "2gb",
-    "--network",
-    "host"
-  ],
-  "mounts": [
-    "source=${localEnv:HOME}/.bts/devcontainer-mounts/${devcontainerId},target=/home/bts,type=bind",
-    "source=/var/run/docker.sock,target=/var/run/docker.sock,type=bind"
-  ],
-  "customizations": {
-    "vscode": {
-      "settings": {
-        "clangd.arguments": [
-          "--compile-commands-dir=${workspaceFolder}/build/clang/debug"
-        ]
-      },
-      "extensions": [
-        "eamodio.gitlens",
-        "llvm-vs-code-extensions.vscode-clangd",
-        "ms-vscode.cmake-tools",
-        "ms-vscode.cpptools"
-      ]
-    }
-  }
-}
@@ -1,4 +0,0 @@
-#!/usr/bin/env bash
-
-this_dir="$(realpath "$(dirname "$(readlink -f "${BASH_SOURCE[0]}")")")"
-sudo dpkg -i "$this_dir/emumba-deribit-connector-3.3.1-Linux.LICENSE_DISABLED.deb"
@@ -1,81 +0,0 @@
-# This file defines the global default configuration for docker deployments.
-# It is tracked in source control. For temporary modifications, please create
-# a separate file called ".env.local" (it's in .gitignore, so won't be tracked).
-# If a variable is defined in both files, the value in .env.local will be used.
-#
-# About the deployment procedure:
-#   We use Docker Compose to build images and launch containers for various services
-#   that make up our software suite:
-#     - sparkd
-#     - pubsub
-#     - md-publisher
-#     - webapp (DSQA on the web)
-#     - api-server (websocket based)
-#   From inside a devcontainer, you can launch all services in separate containers
-#   on your machine by running:
-#
-#     tools/deploy.sh
-#
-#   This script builds release versions for each service, packaged (generally) in
-#   self-contained directories. Next, it runs a few "docker compose" commands to
-#   build docker images loaded with the self-contained binaries from the previous
-#   step, and start containers with those images to run each service. To view the
-#   commands that deploy.sh runs internally (without actually running them), run:
-#
-#     tools/deploy.sh --dry-run
-#
-#   For additional information/help, run:
-#
-#     tools/deploy.sh --help
-#
-
-### Sparkd ###
-# The internal image for sparkd to use. For more info, view:
-#   src/app_sparkd/Dockerfile
-SPARKD__IMAGE=sparkd
-# The 'spark.conf' for this service to use
-SPARKD__CONF_FILE=src/app_sparkd/spark.conf
-# To run replay md, put recorded md files in SPARKD__MD_RECORDING_DIR, and use:
-#   SPARKD__IMAGE=sparkd_md_replay
-#   SPARKD__CONF_FILE=src/app_sparkd/spark.local.conf
-# (configure which md to replay in the new SPARKD__CONF_FILE)
-SPARKD__MD_RECORDING_DIR=.md_recording
-SPARKD__SOURCE_DIR=install/sparkd-release-gcc
-SPARKD__CONFIG_PORT=8000
-SPARKD__QUICKBUS_PORT=8001
-SPARKD__SIM_FIX_PORT=8002
-
-### Pubsub ###
-# The 'spark.conf' for this service to use
-PUBSUB__CONF_FILE=src/DataStreamSuite/PubSubServer/spark.conf
-PUBSUB__CS_SOURCE_DIR=build/Release/bin/net7.0/linux-x64/publish
-PUBSUB__CPP_SOURCE_DIR=install/data-stream-suite-linux
-# The appsettings to load at runtime. Expects a file named:
-#   src/DataStreamSuite/PubSub/appsettings.${PUBSUB__DOTNET_ENVIRONMENT}.json
-PUBSUB__DOTNET_ENVIRONMENT=Staging
-
-### Md Publisher ###
-# The 'spark.conf' for this service to use
-MD_PUBLISHER__CONF_FILE=src/DataStreamSuite/DataStreamPublisher/spark.conf
-MD_PUBLISHER__SOURCE_DIR=build/Release/bin/net7.0/linux-x64/publish
-# Determines which appsettings to load at runtime. Expects a file named:
-#   src/DataStreamSuite/DataStreamPublisher/appsettings.${MD_PUBLISHER__DOTNET_ENVIRONMENT}.json
-MD_PUBLISHER__DOTNET_ENVIRONMENT=MarketData
-
-### Webapp (DSQA on the web) ###
-# The 'spark.conf' for this service to use
-WEBAPP__CONF_FILE=src/client/spark_api_server/spark.conf
-WEBAPP__SOURCE_DIR=build/Release/bin/net7.0/linux-x64/publish
-WEBAPP__WEB_ROOT=build/Bts.DataStreamWebApp.Client/production/dist
-# Determines which appsettings to load at runtime. Expects a file named:
-#   src/DataStreamSuite/Bts.DataStreamWebApp.Server/appsettings.${WEBAPP__ASPNETCORE_ENVIRONMENT}.json
-WEBAPP__ASPNETCORE_ENVIRONMENT=Staging
-
-### Api Server ###
-# The 'spark.conf' for this service to use
-API_SERVER__CONF_FILE=src/client/spark_api_server/spark.conf
-API_SERVER__CS_SOURCE_DIR=build/Release/bin/net7.0/linux-x64/publish
-API_SERVER__CPP_SOURCE_DIR=install/spark-api-server-linux
-# Determines which appsettings to load at runtime. Expects a file named:
-#   src/client/spark_api_server/appsettings.${API_SERVER__ASPNETCORE_ENVIRONMENT}.json
-API_SERVER__ASPNETCORE_ENVIRONMENT=Staging
@@ -1,4 +0,0 @@
-*.cmd text=auto eol=crlf
-*.cs text=auto eol=crlf
-*.sh text=auto eol=lf
-*.py text=auto eol=lf
@@ -8,9 +8,7 @@ cmake-build-debug/
 .idea/
 .dir-locals.el
 *.pyc
-**.vscode/*
-!.vscode/launch.json
-!.vscode/tasks.json
+**.vscode
 .vs
 obj/
 *.csproj.user
@@ -56,9 +54,6 @@ trace_data/
 /src/Spark.Admin.WebApp/src/bin/
 **/CMakeDeps.cmake
 **/CMakeDeps_win.cmake
-.devcontainer/
+**/.devcontainer
 *.Generated.cs
 *.lst
-src/app_sparkd/spark.local.conf
-.env.local
-.md_recording
@@ -1,6 +1,6 @@
 [submodule "externals"]
 	path = externals
-	url = https://bitbucket.org/bluetradesys/externals.git
+	url = https://x-token-auth:ATCTT3xFfGN0h_ENnUSfEAIzbfVzkQPLG26cIli50MtcTwZ7VEc8lin5tJYqibjlBJ9PM9R642Ra5GjiL52TBfbO8KeUBT0UoMT7gLQHL2Y1TkW9yZr6NkvsoDkSl49DeD4Ijiu4_0u4qEm4oMVDDg2_fB059-uOWOo7ZL94tGZET8CSVpFIGvs=B0E03660@bitbucket.org/bluetradesys/externals.git
 [submodule "vcpkg"]
 	path = vcpkg
 	url = https://github.com/microsoft/vcpkg.git
@@ -0,0 +1,49 @@
+# This configuration file was automatically generated by Gitpod.
+# Please adjust to your needs (see https://www.gitpod.io/docs/introduction/learn-gitpod/gitpod-yaml)
+# and commit this file to your remote git repository to share the goodness with others.
+
+# Learn more from ready-to-use templates: https://www.gitpod.io/docs/introduction/getting-started/quickstart
+
+image: gitpod/workspace-dotnet-8
+tasks:
+  - name: Install Deps
+    init: |
+      (
+        set -e
+        sudo install-packages rapidjson-dev
+        sudo install-packages libssh2-1-dev
+        sudo install-packages openssh-server
+        sudo install-packages liblttng-ust-dev
+        sudo install-packages libpcap0.8-dev
+        sudo install-packages ccache
+        sudo ln -s $(which ccache) /usr/local/bin/clang
+        sudo ln -s $(which ccache) /usr/local/bin/clang++
+        sudo ln -s $(which ccache) /usr/local/bin/clang-17
+        sudo ln -s $(which ccache) /usr/local/bin/clang++-17
+        sudo ln -s $(which ccache) /usr/local/bin/gcc
+        sudo ln -s $(which ccache) /usr/local/bin/g++
+        sudo ln -s $(which ccache) /usr/local/bin/gcc-12
+        sudo ln -s $(which ccache) /usr/local/bin/g++-12
+        if [ -e externals/src/emumba/install_emumba-deribit-connector.sh ]; then
+          externals/src/emumba/install_emumba-deribit-connector.sh
+        fi
+      )
+  - name: Setup Python venv
+    init: |
+      python -m pip install --upgrade pip
+      python -m venv .venv
+      . .venv/bin/activate
+      python -m pip install --upgrade pip
+      pip install -r requirements.txt
+
+ports:
+  - name: sparkd config
+    port: 8000
+    onOpen: notify
+  - name: sparkd bus
+    port: 8001
+
+gitConfig:
+  alias.st: "status"
+  status.short: "true"
+  core.pager: "less -FXRS"
@@ -1,369 +0,0 @@
-{
-  "version": "0.2.0",
-  "configurations": [
-    {
-      "name": ".NET Core Attach",
-      "type": "coreclr",
-      "request": "attach"
-    },
-    {
-      "name": "Api Server",
-      "type": "coreclr",
-      "request": "launch",
-      "program": "",
-      "windows": {
-        "preLaunchTask": "Build Api Server (Debug)",
-        "program": "spark_api_server.exe",
-        "args": [
-          "run",
-        ],
-        "cwd": "build/Debug/bin/net7.0",
-      },
-      "linux": {
-        "preLaunchTask": "Build Api Server (Debug)",
-        "program": "spark_api_server",
-        "args": [
-          "run",
-        ],
-        "cwd": "build/Debug/bin/net7.0",
-      },
-      "env": {
-        "ASPNETCORE_ENVIRONMENT": "Development",
-      },
-      "stopAtEntry": false,
-      "console": "integratedTerminal"
-    },
-    {
-      "name": "Api Server (Skip Build)",
-      "type": "coreclr",
-      "request": "launch",
-      "program": "",
-      "windows": {
-        "program": "spark_api_server.exe",
-        "args": [
-          "run",
-        ],
-        "cwd": "build/Debug/bin/net7.0",
-      },
-      "linux": {
-        "program": "spark_api_server",
-        "args": [
-          "run",
-        ],
-        "cwd": "build/Debug/bin/net7.0",
-      },
-      "env": {
-        "ASPNETCORE_ENVIRONMENT": "Development",
-      },
-      "stopAtEntry": false,
-      "console": "integratedTerminal"
-    },
-    {
-      "name": "Bts.DataStreamWebApp.Server",
-      "type": "coreclr",
-      "request": "launch",
-      "preLaunchTask": "Build DataStream (Debug)",
-      "program": "${workspaceFolder}/build/Debug/bin/net7.0/Bts.DataStreamWebApp.Server.dll",
-      "cwd": "${workspaceFolder}/build/Debug/bin/net7.0",
-      "env": {
-        "ASPNETCORE_HTTP_PORT": "8080",
-        "ASPNETCORE_HTTPS_PORT": "8443",
-        "ASPNETCORE_ENVIRONMENT": "${input:dataStreamWebAppServerEnvironment}",
-        "ASPNETCORE_HOSTINGSTARTUPASSEMBLIES": "Microsoft.AspNetCore.SpaProxy",
-      }
-    },
-    {
-      "name": "Bts.DataStreamWebApp.Server (Skip Build)",
-      "type": "coreclr",
-      "request": "launch",
-      "program": "${workspaceFolder}/build/Debug/bin/net7.0/Bts.DataStreamWebApp.Server.dll",
-      "cwd": "${workspaceFolder}/build/Debug/bin/net7.0",
-      "env": {
-        "ASPNETCORE_HTTP_PORT": "8080",
-        "ASPNETCORE_HTTPS_PORT": "8443",
-        "ASPNETCORE_ENVIRONMENT": "${input:dataStreamWebAppServerEnvironment}",
-        "ASPNETCORE_HOSTINGSTARTUPASSEMBLIES": "Microsoft.AspNetCore.SpaProxy",
-      }
-    },
-    {
-      "name": "DataStreamPublisher",
-      "type": "coreclr",
-      "request": "launch",
-      "preLaunchTask": "Build DataStream (Debug)",
-      "program": "",
-      "linux": {
-        "program": "DataStreamPublisher",
-      },
-      "windows": {
-        "program": "DataStreamPublisher.exe",
-      },
-      "env": {
-        "DOTNET_ENVIRONMENT": "${input:dataStreamPublisherEnvironment}",
-      },
-      "cwd": "${workspaceFolder}/build/Debug/bin/net7.0",
-      "stopAtEntry": false,
-      "console": "integratedTerminal"
-    },
-    {
-      "name": "DataStreamPublisher (Skip Build)",
-      "type": "coreclr",
-      "request": "launch",
-      "program": "",
-      "linux": {
-        "program": "DataStreamPublisher",
-      },
-      "windows": {
-        "program": "DataStreamPublisher.exe",
-      },
-      "env": {
-        "DOTNET_ENVIRONMENT": "${input:dataStreamPublisherEnvironment}",
-      },
-      "cwd": "${workspaceFolder}/build/Debug/bin/net7.0",
-      "stopAtEntry": false,
-      "console": "integratedTerminal"
-    },
-    {
-      "name": "(gdb) Launch",
-      "type": "cppdbg",
-      "request": "launch",
-      "program": "/home/bts/spark/build/sparkd-debug-clang/server/sparkd",
-      "args": ["run"],
-      "stopAtEntry": false,
-      "cwd": "/home/bts/spark/build/sparkd-debug-clang/server",
-      "environment": [],
-      "externalConsole": false,
-      "MIMode": "gdb",
-    },
-    {
-      "name": "DataStreamQueryAnalyzer",
-      "type": "coreclr",
-      "request": "launch",
-      "preLaunchTask": "Build DataStreamQueryAnalyzer (Debug)",
-      "program": "",
-      "windows": {
-        "program": "DataStreamQueryAnalyzer.exe",
-        "args": [
-          "--config",
-          "${workspaceFolder}/src/DataStreamSuite/DataStreamQueryAnalyzer/Configurations/DataStreamQueryAnalyzer.json"
-        ],
-        "cwd": "build/Debug/bin/net7.0-windows",
-        "env": {
-          "PYTHON_EXE": "${workspaceFolder}/.venv/Scripts/python.exe"
-        },
-      },
-      "stopAtEntry": false,
-      "console": "integratedTerminal",
-    },
-    {
-      "name": "PubSubServer",
-      "type": "coreclr",
-      "request": "launch",
-      "preLaunchTask": "Build DataStream (Debug)",
-      "program": "",
-      "linux": {
-        "program": "PubSubServer",
-      },
-      "windows": {
-        "program": "PubSubServer.exe",
-      },
-      "cwd": "build/Debug/bin/net7.0",
-      "env": {
-        "DOTNET_ENVIRONMENT": "${input:pubSubServerEnvironment}",
-      },
-      "stopAtEntry": false,
-      "console": "integratedTerminal"
-    },
-    {
-      "name": "PubSubServer (Skip Build)",
-      "type": "coreclr",
-      "request": "launch",
-      "program": "",
-      "linux": {
-        "program": "PubSubServer",
-      },
-      "windows": {
-        "program": "PubSubServer.exe",
-      },
-      "cwd": "build/Debug/bin/net7.0",
-      "env": {
-        "DOTNET_ENVIRONMENT": "${input:pubSubServerEnvironment}",
-      },
-      "stopAtEntry": false,
-      "console": "integratedTerminal"
-    },
-    {
-      "name": "Spark",
-      "type": "coreclr",
-      "request": "launch",
-      "program": "",
-      "windows": {
-        "preLaunchTask": "Build Spark (Debug)",
-        "program": "Spark.exe",
-        "args": [
-          "--sparkd",
-          "http://127.0.0.1:8000/config"
-        ],
-        "cwd": "build/Debug/bin/net7.0-windows",
-      },
-      "stopAtEntry": false,
-      "console": "integratedTerminal"
-    },
-    {
-      "name": "Spark (Parameterized)",
-      "type": "coreclr",
-      "request": "launch",
-      "preLaunchTask": "Build Spark (Debug)",
-      "program": "",
-      "windows": {
-        "program": "Spark.exe",
-        "args": [
-          "--sparkd",
-          "${input:sparkConfigServer}"
-        ],
-        "cwd": "build/Debug/bin/net7.0-windows",
-      },
-      "stopAtEntry": false,
-      "console": "integratedTerminal"
-    },
-    {
-      "name": "Sparkd",
-      "type": "cppdbg",
-      "request": "launch",
-      "preLaunchTask": "Build Sparkd (Debug | Clang)",
-      "program": "",
-      "linux": {
-        "program": "${workspaceFolder}/build/sparkd-debug-clang/src/app_sparkd/sparkd",
-      },
-      "args": [
-        "run"
-      ],
-      "cwd": "${workspaceFolder}/build/sparkd-debug-clang/src/app_sparkd",
-      "stopAtEntry": false,
-      "externalConsole": false,
-      "MIMode": "gdb",
-      "setupCommands": [
-        {
-          "description": "Enable pretty-printing for gdb",
-          "text": "-enable-pretty-printing",
-          "ignoreFailures": true
-        },
-        {
-          "description": "Set Disassembly Flavor to Intel",
-          "text": "-gdb-set disassembly-flavor intel",
-          "ignoreFailures": true
-        }
-      ]
-    },
-    {
-      "name": "Sparkd (Skip Build)",
-      "type": "cppdbg",
-      "request": "launch",
-      "program": "",
-      "linux": {
-        "program": "${workspaceFolder}/build/sparkd-debug-clang/src/app_sparkd/sparkd",
-      },
-      "args": [
-        "run"
-      ],
-      "cwd": "${workspaceFolder}/build/sparkd-debug-clang/src/app_sparkd",
-      "stopAtEntry": false,
-      "externalConsole": false,
-      "MIMode": "gdb",
-      "setupCommands": [
-        {
-          "description": "Enable pretty-printing for gdb",
-          "text": "-enable-pretty-printing",
-          "ignoreFailures": true
-        },
-        {
-          "description": "Set Disassembly Flavor to Intel",
-          "text": "-gdb-set disassembly-flavor intel",
-          "ignoreFailures": true
-        }
-      ]
-    },
-    {
-      "name": "Sparkd Web (Skip Build)",
-      "type": "cppdbg",
-      "request": "launch",
-      "program": "",
-      "linux": {
-        "program": "${workspaceFolder}/build/sparkd-debug-clang/src/app_sparkd/sparkd",
-      },
-      "args": [
-        "web"
-      ],
-      "cwd": "${workspaceFolder}/build/sparkd-debug-clang/src/app_sparkd",
-      "stopAtEntry": false,
-      "externalConsole": false,
-      "MIMode": "gdb",
-      "setupCommands": [
-        {
-          "description": "Enable pretty-printing for gdb",
-          "text": "-enable-pretty-printing",
-          "ignoreFailures": true
-        },
-        {
-          "description": "Set Disassembly Flavor to Intel",
-          "text": "-gdb-set disassembly-flavor intel",
-          "ignoreFailures": true
-        }
-      ]
-    },
-  ],
-  "inputs": [
-    {
-      "id": "dataStreamPublisherEnvironment",
-      "type": "pickString",
-      "description": "Publisher Configuration",
-      "options": [
-        "Development",
-        "MarketData",
-        "Production",
-        "ProductionAws",
-        "Staging",
-      ],
-      "default": "MarketDataPublisher",
-    },
-    {
-      "id": "pubSubServerEnvironment",
-      "type": "pickString",
-      "description": "PubSub Configuration",
-      "options": [
-        "Development",
-        "EchoServer",
-        "Production",
-        "ProductionAws",
-        "Staging",
-      ],
-      "default": "Development",
-    },
-    {
-      "id": "dataStreamWebAppServerEnvironment",
-      "type": "pickString",
-      "description": "DataStream WebApp Environment",
-      "options": [
-        "Development",
-        "Production",
-        "ProductionAws",
-        "Staging",
-      ]
-    },
-    {
-      "id": "sparkConfigServer",
-      "type": "promptString",
-      "description": "Config Server URL",
-      "default": "http://localhost:8000/config",
-    },
-    {
-      "id": "isSparkApiEnabled",
-      "type": "pickString",
-      "description": "Enable Spark Api (connect to sparkd)",
-      "options": [
-        "True",
-        "False",
-      ],
-      "default": "True",
-    }
-  ]
-}
@@ -14,17 +14,12 @@ set(
  CACHE
  STRING
  "External dependencies root folder (should contain folders: lib, bin, include")
-
-# SHARED_MEMORY_PREFIX serves as a kind of namespace so multiple applications
-# can run simultaneously without fighting over the same shm files (e.g., sparkd,
-# api server, etc.).
-set(SHARED_MEMORY_PREFIX "\${BTS_SHARED_MEMORY_PREFIX}" CACHE STRING "Prefix to use when creating shared memory files")
-set(EXTERNALS_BIN "${EXTERNALS_ROOT}/bin")
-set(EXTERNALS_INCLUDE "${EXTERNALS_ROOT}/include")
-set(EXTERNALS_LIB "${EXTERNALS_ROOT}/lib/$<CONFIG>")
+set(SPARK_BIN "${EXTERNALS_ROOT}/bin")
+set(SPARK_INCLUDE "${EXTERNALS_ROOT}/include")
+set(SPARK_LIB "${EXTERNALS_ROOT}/lib/$<CONFIG>")
+set(BUILD_API_SERVER OFF CACHE BOOL "Build the api server")
 set(BUILD_DOCS OFF CACHE BOOL "Build documentation")
 set(BUILD_SDK OFF CACHE BOOL "Build the sdk")
-set(EXPORT_MD_CONSUMER OFF CACHE BOOL "Export (extern) C functions for MD consumer interop")
 if (WIN32)
  set(BUILD_SPARKD OFF CACHE BOOL "Build sparkd")
  set(BUILD_ADMIN_WEB_APP OFF CACHE BOOL "Build the Admin WebApp")
@@ -33,85 +28,129 @@ else()
  set(BUILD_ADMIN_WEB_APP ON CACHE BOOL "Build the Admin WebApp")
 endif()

-add_compile_definitions(SHARED_MEMORY_PREFIX="${SHARED_MEMORY_PREFIX}")
-if(DEFINED APP_DATA_DIR)
-  add_compile_definitions(APP_DATA_DIR="${APP_DATA_DIR}")
-endif()
-add_compile_definitions(EXPORT_MD_CONSUMER="${EXPORT_MD_CONSUMER}")
+include_directories(BEFORE SYSTEM ${SPARK_INCLUDE} ${EXTERNALS_ROOT}/include/zlib)
+link_directories(${SPARK_LIB})

-if(BUILD_SPARKD)
-  # Note: All macros that begin w/ 'CCAPI' are ccapi's macros (defined and/or
-  # read by their code). We override some below.
-  add_compile_definitions(SPARK_ENABLE_CCAPI)
-
-  # This allows us to override hardcoded exchange-urls with ones read from
-  # config (i.e., resolved at runtime rather than compile-time)
-  add_compile_definitions(CCAPI_EXPOSE_INTERNAL)
-
-  # Enable ccapi services
-  add_compile_definitions(CCAPI_ENABLE_SERVICE_MARKET_DATA)
-  add_compile_definitions(CCAPI_ENABLE_SERVICE_EXECUTION_MANAGEMENT)
-  # Enable ccapi exchanges
-  add_compile_definitions(CCAPI_ENABLE_EXCHANGE_BINANCE)
-  add_compile_definitions(CCAPI_ENABLE_EXCHANGE_BINANCE_COIN_FUTURES)
-  add_compile_definitions(CCAPI_ENABLE_EXCHANGE_BINANCE_US)
-  add_compile_definitions(CCAPI_ENABLE_EXCHANGE_BINANCE_USDS_FUTURES)
-  add_compile_definitions(CCAPI_ENABLE_EXCHANGE_COINBASE)
-  add_compile_definitions(CCAPI_ENABLE_EXCHANGE_DERIBIT)
-  add_compile_definitions(CCAPI_ENABLE_EXCHANGE_OKX)
-
-  set(CCAPI_LOG_LEVEL
-    "Warn"
-    CACHE STRING "<Fatal|Error|Warn|Info|Debug|Trace> Log level for ccapi")
-  string(TOUPPER "${CCAPI_LOG_LEVEL}" _ccapi_log_level_upper)
-  add_compile_definitions("CCAPI_ENABLE_LOG_${_ccapi_log_level_upper}")
-
-  # ** Binance **
-  add_compile_definitions(CCAPI_BINANCE_URL_REST_BASE="https://api.binance.com")
-  add_compile_definitions(BINANCE_INSTRUMENTS_PATH="/api/v3/exchangeInfo")
-  add_compile_definitions(BINANCE_PRICE_STATS_PATH="/api/v3/ticker/24hr")
-
-  # ** Binance US **
-  add_compile_definitions(BINANCE_US_INSTRUMENTS_PATH="/api/v3/exchangeInfo")
-  add_compile_definitions(BINANCE_US_PRICE_STATS_PATH="/api/v3/ticker/24hr")
-
-  # ** Binance USDS Futures **
-  add_compile_definitions(CCAPI_BINANCE_USDS_FUTURES_URL_REST_BASE="https://dapi.binance.com")
-  add_compile_definitions(BINANCE_USDS_FUTURES_INSTRUMENTS_PATH="/dapi/v1/exchangeInfo")
-  add_compile_definitions(BINANCE_USDS_FUTURES_PRICE_STATS_PATH="/dapi/v1/ticker/24hr")
-
-  # ** Deribit **
-  add_compile_definitions(DERIBIT_INSTRUMENT_PATH="/api/v2/public/get_instrument")
-  add_compile_definitions(DERIBIT_INSTRUMENTS_PATH="/api/v2/public/get_instruments")
-  add_compile_definitions(DERIBIT_COMBOS_PATH="/api/v2/public/get_combos")
-  add_compile_definitions(DERIBIT_TICKER_PATH="/api/v2/public/ticker")
-  add_compile_definitions(DERIBIT_CURRENCIES_METHOD="public/get_currencies")
-  add_compile_definitions(DERIBIT_INSTRUMENT_METHOD="public/get_instrument")
-  add_compile_definitions(DERIBIT_INSTRUMENTS_METHOD="public/get_instruments")
-  add_compile_definitions(DERIBIT_COMBO_DETAILS_METHOD="public/get_combo_details")
-  add_compile_definitions(DERIBIT_COMBOS_METHOD="public/get_combos")
-
-  # ** Okx **
-  add_compile_definitions(OKX_INSTRUMENTS_PATH="/api/v5/public/instruments")
-  add_compile_definitions(OKX_UNDERLYING_PATH="/api/v5/public/underlying")
+if(BUILD_API_SERVER)
+  add_compile_definitions(SPARK_ENABLE_API_SERVER)
 endif()

-# subdirs
-add_subdirectory(src/lib_system)
-add_subdirectory(src/lib_domain)
-add_subdirectory(src/lib_http)
-if (BUILD_SPARKD)
-  add_subdirectory(src/lib_python_md)
-  add_subdirectory(src/app_sparkd)
-  add_subdirectory(src/data_files)
-endif()
-add_subdirectory(src/lib_spark)
-if (BUILD_ADMIN_WEB_APP)
-  add_subdirectory(src/Spark.Admin.WebApp)
-endif()
-if(BUILD_DOCS)
-  add_subdirectory(doc/user)
-endif()
-if(BUILD_SDK)
-  add_subdirectory(src/sdk)
+if(WIN32)
+  add_compile_definitions(SHARED_MEMORY_PREFIX="spark")
+  add_compile_definitions(APP_DATA_DIR="\$\{APPDATA\}/bts/")
+  # subdirs
+  add_subdirectory(src/lib)
+else()
+  # make sparkd look for libspark.so in its directory
+  set(CMAKE_BUILD_WITH_INSTALL_RPATH TRUE)
+  set(CMAKE_INSTALL_RPATH ".")
+  set(CMAKE_INSTALL_PREFIX "${CMAKE_SOURCE_DIR}/install")
+  set(SPARK_INSTALL_PREFIX "${CMAKE_INSTALL_PREFIX}/$<CONFIG>")
+
+  if(BUILD_API_SERVER)
+    add_compile_definitions(APP_DATA_DIR="\$\{HOME\}/spark_api/")
+    add_compile_definitions(SHARED_MEMORY_PREFIX="spark_api")
+  elseif(BUILD_SPARKD)
+    add_compile_definitions(SHARED_MEMORY_PREFIX="sparkd")
+    add_compile_definitions(APP_DATA_DIR="\$\{HOME\}/sparkdata/")
+
+    # Note: All macros that begin w/ 'CCAPI' are ccapi's macros (defined and/or
+    # read by their code). We override some below.
+    add_compile_definitions(SPARK_ENABLE_CCAPI)
+
+    # This allows us to override hardcoded exchange-urls with ones read from
+    # config (i.e., resolved at runtime rather than compile-time)
+    add_compile_definitions(CCAPI_EXPOSE_INTERNAL)
+
+    # Enable ccapi services
+    add_compile_definitions(CCAPI_ENABLE_SERVICE_MARKET_DATA)
+    add_compile_definitions(CCAPI_ENABLE_SERVICE_EXECUTION_MANAGEMENT)
+    # Enable ccapi exchanges
+    add_compile_definitions(CCAPI_ENABLE_EXCHANGE_BINANCE)
+    add_compile_definitions(CCAPI_ENABLE_EXCHANGE_BINANCE_COIN_FUTURES)
+    add_compile_definitions(CCAPI_ENABLE_EXCHANGE_BINANCE_US)
+    add_compile_definitions(CCAPI_ENABLE_EXCHANGE_BINANCE_USDS_FUTURES)
+    add_compile_definitions(CCAPI_ENABLE_EXCHANGE_COINBASE)
+    add_compile_definitions(CCAPI_ENABLE_EXCHANGE_DERIBIT)
+    add_compile_definitions(CCAPI_ENABLE_EXCHANGE_OKX)
+
+    set(CCAPI_LOG_LEVEL
+      "Warn"
+      CACHE STRING "<Fatal|Error|Warn|Info|Debug|Trace> Log level for ccapi")
+    string(TOUPPER "${CCAPI_LOG_LEVEL}" _ccapi_log_level_upper)
+    add_compile_definitions("CCAPI_ENABLE_LOG_${_ccapi_log_level_upper}")
+
+    # ** Binance **
+    add_compile_definitions(CCAPI_BINANCE_URL_REST_BASE="https://api.binance.com")
+    add_compile_definitions(BINANCE_INSTRUMENTS_PATH="/api/v3/exchangeInfo")
+    add_compile_definitions(BINANCE_PRICE_STATS_PATH="/api/v3/ticker/24hr")
+
+    # ** Binance US **
+    add_compile_definitions(BINANCE_US_INSTRUMENTS_PATH="/api/v3/exchangeInfo")
+    add_compile_definitions(BINANCE_US_PRICE_STATS_PATH="/api/v3/ticker/24hr")
+
+    # ** Binance USDS Futures **
+    add_compile_definitions(CCAPI_BINANCE_USDS_FUTURES_URL_REST_BASE="https://dapi.binance.com")
+    add_compile_definitions(BINANCE_USDS_FUTURES_INSTRUMENTS_PATH="/dapi/v1/exchangeInfo")
+    add_compile_definitions(BINANCE_USDS_FUTURES_PRICE_STATS_PATH="/dapi/v1/ticker/24hr")
+
+    # ** Deribit **
+    add_compile_definitions(DERIBIT_INSTRUMENT_PATH="/api/v2/public/get_instrument")
+    add_compile_definitions(DERIBIT_INSTRUMENTS_PATH="/api/v2/public/get_instruments")
+    add_compile_definitions(DERIBIT_COMBOS_PATH="/api/v2/public/get_combos")
+    add_compile_definitions(DERIBIT_TICKER_PATH="/api/v2/public/ticker")
+    add_compile_definitions(DERIBIT_CURRENCIES_METHOD="public/get_currencies")
+    add_compile_definitions(DERIBIT_INSTRUMENT_METHOD="public/get_instrument")
+    add_compile_definitions(DERIBIT_INSTRUMENTS_METHOD="public/get_instruments")
+    add_compile_definitions(DERIBIT_COMBO_DETAILS_METHOD="public/get_combo_details")
+    add_compile_definitions(DERIBIT_COMBOS_METHOD="public/get_combos")
+
+    # ** Okx **
+    add_compile_definitions(OKX_INSTRUMENTS_PATH="/api/v5/public/instruments")
+    add_compile_definitions(OKX_UNDERLYING_PATH="/api/v5/public/underlying")
+  endif()
+
+  # subdirs
+  if (BUILD_SPARKD)
+    add_subdirectory(src/server/sparkd)
+    add_subdirectory(src/data_files)
+  endif()
+  add_subdirectory(src/lib)
+  if (BUILD_ADMIN_WEB_APP)
+    add_subdirectory(src/Spark.Admin.WebApp)
+  endif()
+  if(BUILD_DOCS)
+    add_subdirectory(doc/user)
+  endif()
+  if(BUILD_SDK)
+    add_subdirectory(src/sdk)
+  endif()
+
+  if(BUILD_SPARKD)
+    add_custom_command(
+      OUTPUT ${CMAKE_BINARY_DIR}/server
+      COMMAND mkdir -p ${CMAKE_BINARY_DIR}/server
+      COMMAND ln ARGS -sf ${CMAKE_BINARY_DIR}/src/server/sparkd/sparkd ${CMAKE_BINARY_DIR}/server
+      COMMAND ln ARGS -sf ${CMAKE_BINARY_DIR}/data_files ${CMAKE_BINARY_DIR}/server
+      COMMAND ln ARGS -sf ${CMAKE_BINARY_DIR}/src/lib/libspark.so ${CMAKE_BINARY_DIR}/server
+      COMMAND
+        ln ARGS -sf
+        ${EXTERNALS_ROOT}/lib/Release/libOnixS.IceImpactMulticastPriceFeedHandlerCpp.so.3.16.0.0
+        ${CMAKE_BINARY_DIR}/server
+      COMMAND ln ARGS -sf ${EXTERNALS_ROOT}/lib/Release/libACE-5.7.5.so ${CMAKE_BINARY_DIR}/server
+      COMMAND mkdir ARGS -p ${CMAKE_BINARY_DIR}/server/plugins
+      COMMENT "Creating server runtime layout"
+      DEPENDS sparkd data_files_dir)
+
+    add_custom_target(server_layout ALL DEPENDS ${CMAKE_BINARY_DIR}/server)
+
+    install(
+      DIRECTORY "${CMAKE_BINARY_DIR}/server/plugins"
+      DESTINATION "${SPARK_INSTALL_PREFIX}/bin")
+    install(
+      FILES
+      "${SPARK_LIB}/libACE-5.7.5.so"
+      "${SPARK_LIB}/libOnixS.IceImpactMulticastPriceFeedHandlerCpp.so.3.16.0.0"
+      DESTINATION "${SPARK_INSTALL_PREFIX}/bin")
+  endif()
 endif()
@@ -15,7 +15,7 @@
        "rhs": "Windows"
      },
      "generator": "Visual Studio 17 2022",
-      "binaryDir": "${sourceDir}/build/${presetName}",
+      "binaryDir": "${sourceDir}/build",
      "environment": {
        "SPARK_CMAKE_CXX_FLAGS": "/MP /WX /wd4250 /wd4251 /wd4244 /wd4267 /wd4355 /wd4275 /wd4345 /bigobj /EHsc /DCURL_STATICLIB /D_WINSOCK_DEPRECATED_NO_WARNINGS /D__TBB_NO_IMPLICIT_LINKAGE /DBOOST_ALL_NO_LIB /std:c++20 /D_SCL_SECURE_NO_WARNINGS /D_CRT_SECURE_NO_WARNINGS /D_WIN32_WINNT=0x0600"
      },
@@ -28,12 +28,9 @@
        "CMAKE_SHARED_LINKER_FLAGS_RELEASE": "/OPT:REF /OPT:ICF /LTCG",
        "CMAKE_TOOLCHAIN_FILE": "${sourceDir}/vcpkg/scripts/buildsystems/vcpkg.cmake",
        "EXTERNALS_ROOT": "${sourceDir}/externals/windows",
-        "CONFIG_DEFAULTS_PATH": "spark/config/profiles/default/*.conf",
-        "DOTNET_BINARY_DIR": "${sourceDir}/build/$<CONFIG>/bin/net7.0;${sourceDir}/build/$<CONFIG>/bin/net7.0-windows",
-        "EXPORT_MD_CONSUMER": {
-          "type": "BOOL",
-          "value": "ON"
-        }
+        "CONFIG_DEFAULTS_PATH": "config/profiles/default/*.conf",
+        "DOTNET_BINARY_DIR": "${sourceDir}/build/$<CONFIG>/bin/net8.0",
+        "DOTNET_WINDOWS_BINARY_DIR": "${sourceDir}/build/$<CONFIG>/bin/net8.0-windows"
      }
    },

@@ -49,11 +46,14 @@
      "binaryDir": "${sourceDir}/build/${presetName}",
      "cacheVariables": {
        "platform": "",
+        "CMAKE_BUILD_WITH_INSTALL_RPATH": {
+          "type": "BOOL",
+          "value": "ON"
+        },
        "CMAKE_CXX_STANDARD": "20",
-        "CMAKE_INSTALL_PREFIX": "${sourceDir}/install/${presetName}",
-        "CMAKE_PREFIX_PATH": "${sourceDir}/cmake/;${sourceDir}/.venv/lib/python3.10/site-packages/pybind11/share/cmake/pybind11",
+        "CMAKE_INSTALL_RPATH": ".",
+        "CMAKE_INSTALL_PREFIX": "${sourceDir}/install/${presetName}/sparkd",
        "CMAKE_EXE_LINKER_FLAGS": "-Wno-undef",
-        "CMAKE_TOOLCHAIN_FILE": "${sourceDir}/vcpkg/scripts/buildsystems/vcpkg.cmake",
        "EXTERNALS_ROOT": "${sourceDir}/externals/linux",
        "BUILD_SDK": {
          "type": "BOOL",
@@ -77,7 +77,7 @@
          "value": "ON"
        },
        "CCAPI_LOG_LEVEL": "Info",
-        "CONFIG_DEFAULTS_PATH": "spark/config/profiles/md_dev/*.conf;spark/config/profiles/default/*.conf"
+        "CONFIG_DEFAULTS_PATH": "config/profiles/md_dev/*.conf;config/profiles/default/*.conf"
      }
    },

@@ -88,74 +88,105 @@
      "cacheVariables": {
        "CMAKE_BUILD_TYPE": "Release",
        "CCAPI_LOG_LEVEL": "Warn",
-        "CONFIG_DEFAULTS_PATH": "spark/config/profiles/md_prod/*.conf;spark/config/profiles/default/*.conf"
+        "CONFIG_DEFAULTS_PATH": "config/profiles/md_prod/*.conf;config/profiles/default/*.conf"
      }
    },

    {
      "name": "sparkd-debug-clang",
      "inherits": "sparkd-debug",
-      "displayName": "Sparkd (Debug | Clang)",
+      "displayName": "Debug (clang)",
      "description": "Debug build compiled w/ clang",
+      "binaryDir": "${sourceDir}/build/clang/debug",
      "cacheVariables": {
        "CMAKE_C_COMPILER": "clang",
        "CMAKE_CXX_COMPILER": "clang++",
-        "CMAKE_CXX_FLAGS_DEBUG": "-g -fPIC -Wall -pedantic -Wextra -Werror -Wno-missing-braces -Wno-ignored-qualifiers --std=c++20 -fdiagnostics-color -DDEBUG"
+        "CMAKE_CXX_FLAGS_DEBUG": "-g -fPIC -Wall -pedantic -Wextra -Werror -Wno-missing-braces -Wno-ignored-qualifiers --std=c++20 -fdiagnostics-color -DDEBUG",
+        "CMAKE_INSTALL_PREFIX": "${sourceDir}/install/clang/debug/sparkd"
      }
    },

    {
      "name": "sparkd-debug-gcc",
      "inherits": "sparkd-debug",
-      "displayName": "Sparkd (Debug | Gcc)",
+      "displayName": "Debug (gcc)",
      "description": "Debug build compiled w/ gcc",
+      "binaryDir": "${sourceDir}/build/gcc/debug",
      "cacheVariables": {
        "CMAKE_C_COMPILER": "gcc",
        "CMAKE_CXX_COMPILER": "g++",
-        "CMAKE_CXX_FLAGS_DEBUG": "-g -fPIC -Wall -pedantic -Wextra -Werror -Wno-missing-braces -Wno-ignored-qualifiers --std=c++20 -fdiagnostics-color -Wno-stringop-overread -DDEBUG"
+        "CMAKE_CXX_FLAGS_DEBUG": "-g -fPIC -Wall -pedantic -Wextra -Werror -Wno-missing-braces -Wno-ignored-qualifiers --std=c++20 -fdiagnostics-color -Wno-stringop-overread -DDEBUG",
+        "CMAKE_INSTALL_PREFIX": "${sourceDir}/install/gcc/debug/sparkd"
      }
    },

    {
      "name": "sparkd-release-clang",
      "inherits": "sparkd-release",
-      "displayName": "Sparkd (Release | Clang)",
+      "displayName": "Release (clang)",
      "description": "Release build compiled w/ clang",
+      "binaryDir": "${sourceDir}/build/clang/release",
      "cacheVariables": {
        "CMAKE_C_COMPILER": "clang",
        "CMAKE_CXX_COMPILER": "clang++",
-        "CMAKE_CXX_FLAGS_RELEASE": "-fPIC -Wall -pedantic -Wextra -Werror -Wno-missing-braces -Wno-ignored-qualifiers --std=c++20 -fdiagnostics-color -fno-omit-frame-pointer -DNDEBUG -O3"
+        "CMAKE_CXX_FLAGS_RELEASE": "-fPIC -Wall -pedantic -Wextra -Werror -Wno-missing-braces -Wno-ignored-qualifiers --std=c++20 -fdiagnostics-color -fno-omit-frame-pointer -DNDEBUG -O3",
+        "CMAKE_INSTALL_PREFIX": "${sourceDir}/install/clang/release/sparkd"
      }
    },

    {
      "name": "sparkd-release-gcc",
      "inherits": "sparkd-release",
-      "displayName": "Sparkd (Release | Gcc)",
+      "displayName": "Release (gcc)",
      "description": "Release build compiled w/ gcc",
+      "binaryDir": "${sourceDir}/build/gcc/release",
      "cacheVariables": {
        "CMAKE_C_COMPILER": "gcc",
        "CMAKE_CXX_COMPILER": "g++",
-        "CMAKE_CXX_FLAGS_RELEASE": "-fPIC -Wall -pedantic -Wextra -Werror -Wno-missing-braces -Wno-ignored-qualifiers -Wno-stringop-overflow --std=c++20 -fdiagnostics-color -Wno-stringop-overread -Wno-maybe-uninitialized -fno-omit-frame-pointer -DNDEBUG -O3"
+        "CMAKE_CXX_FLAGS_RELEASE": "-fPIC -Wall -pedantic -Wextra -Werror -Wno-missing-braces -Wno-ignored-qualifiers --std=c++20 -fdiagnostics-color -Wno-stringop-overread -Wno-maybe-uninitialized -fno-omit-frame-pointer -DNDEBUG -O3",
+        "CMAKE_INSTALL_PREFIX": "${sourceDir}/install/gcc/release/sparkd"
+      }
+    },
+
+    {
+      "name": "spark-api-server-base",
+      "hidden": true,
+      "binaryDir": "${sourceDir}/build/${presetName}",
+      "cacheVariables": {
+        "BUILD_API_SERVER": {
+          "type": "bool",
+          "value": "ON"
+        },
+        "CONFIG_DEFAULTS_PATH": "config/profiles/md_dev/*.conf;config/profiles/default/*.conf"
      }
    },

    {
      "name": "spark-api-server-windows",
      "inherits": "spark",
-      "displayName": "Spark Api Server (Windows)"
+      "displayName": "Spark Api Server",
+      "cacheVariables": {
+        "BUILD_API_SERVER": {
+          "type": "bool",
+          "value": "ON"
+        }
+      }
    },

    {
      "name": "spark-api-server-linux",
      "inherits": "sparkd",
-      "displayName": "Spark Api Server (Linux)",
+      "displayName": "Spark Api Server",
      "generator": "Ninja Multi-Config",
      "cacheVariables": {
        "BUILD_ADMIN_WEB_APP": {
          "type": "BOOL",
          "value": "OFF"
        },
+        "BUILD_API_SERVER": {
+          "type": "bool",
+          "value": "ON"
+        },
        "BUILD_SPARKD": {
          "type": "BOOL",
          "value": "OFF"
@@ -164,16 +195,9 @@
        "CMAKE_CXX_COMPILER": "clang++",
        "CMAKE_CXX_FLAGS_DEBUG": "-g -fPIC -Wall -pedantic -Wextra -Werror -Wno-missing-braces -Wno-ignored-qualifiers --std=c++20 -fdiagnostics-color -DDEBUG",
        "CMAKE_CXX_FLAGS_RELEASE": "-fPIC -Wall -pedantic -Wextra -Werror -Wno-missing-braces -Wno-ignored-qualifiers --std=c++20 -fdiagnostics-color -fno-omit-frame-pointer -DNDEBUG -O3",
-        "CMAKE_EXPORT_COMPILE_COMMANDS": {
-          "type": "BOOL",
-          "value": "ON"
-        },
-        "CONFIG_DEFAULTS_PATH": "spark/config/profiles/default/*.conf",
-        "DOTNET_BINARY_DIR": "${sourceDir}/build/$<CONFIG>/bin/net7.0",
-        "EXPORT_MD_CONSUMER": {
-          "type": "BOOL",
-          "value": "ON"
-        }
+        "CMAKE_INSTALL_PREFIX": "${sourceDir}/install/${presetName}",
+        "CONFIG_DEFAULTS_PATH": "config/profiles/default/*.conf",
+        "DOTNET_BINARY_DIR": "${sourceDir}/build/$<CONFIG>/bin/net8.0"
      }
    },

@@ -196,56 +220,13 @@
    },

    {
-      "name": "data-stream-suite-windows",
-      "condition": {
-        "type": "equals",
-        "lhs": "${hostSystemName}",
-        "rhs": "Windows"
-      },
+      "name": "data-stream-suite",
      "inherits": "spark",
-      "displayName": "Data Stream Suite (Windows)",
+      "displayName": "Data Stream Suite",
      "description": "Data Stream utils ported over from Edge",
      "binaryDir": "${sourceDir}/build/${presetName}",
      "cacheVariables": {
-        "CONFIG_DEFAULTS_PATH": "spark/config/profiles/default/*.conf;spark/config/profiles/data_stream/*.conf",
-        "DOTNET_BINARY_DIR": "${sourceDir}/build/$<CONFIG>/bin/net7.0;${sourceDir}/build/$<CONFIG>/bin/net7.0-windows"
-      }
-    },
-
-    {
-      "name": "data-stream-suite-linux",
-      "displayName": "Data Stream Suite (Linux)",
-      "description": "Data Stream utils ported over from Edge",
-      "condition": {
-        "type": "equals",
-        "lhs": "${hostSystemName}",
-        "rhs": "Linux"
-      },
-      "inherits": "sparkd",
-      "generator": "Ninja Multi-Config",
-      "cacheVariables": {
-        "BUILD_ADMIN_WEB_APP": {
-          "type": "BOOL",
-          "value": "OFF"
-        },
-        "BUILD_SDK": {
-          "type": "BOOL",
-          "value": "OFF"
-        },
-        "BUILD_SPARKD": {
-          "type": "BOOL",
-          "value": "OFF"
-        },
-        "CMAKE_C_COMPILER": "clang",
-        "CMAKE_CXX_COMPILER": "clang++",
-        "CMAKE_CXX_FLAGS_DEBUG": "-g -fPIC -Wall -pedantic -Wextra -Werror -Wno-missing-braces -Wno-ignored-qualifiers --std=c++20 -fdiagnostics-color -DDEBUG",
-        "CMAKE_CXX_FLAGS_RELEASE": "-fPIC -Wall -pedantic -Wextra -Werror -Wno-missing-braces -Wno-ignored-qualifiers --std=c++20 -fdiagnostics-color -fno-omit-frame-pointer -DNDEBUG -O3",
-        "CONFIG_DEFAULTS_PATH": "spark/config/profiles/default/*.conf;spark/config/profiles/data_stream/*.conf",
-        "DOTNET_BINARY_DIR": "${sourceDir}/build/$<CONFIG>/bin/net7.0;${sourceDir}/build/$<CONFIG>/bin/net8.0",
-        "EXPORT_MD_CONSUMER": {
-          "type": "BOOL",
-          "value": "ON"
-        }
+        "CONFIG_DEFAULTS_PATH": "config/profiles/default/*.conf;config/profiles/data_stream/*.conf"
      }
    }
  ],
@@ -263,7 +244,7 @@

    {
      "name": "spark-debug",
-      "displayName": "Spark (Debug)",
+      "displayName": "Debug",
      "inherits": "spark",
      "configurePreset": "spark",
      "configuration": "Debug"
@@ -271,7 +252,7 @@

    {
      "name": "spark-release",
-      "displayName": "Spark (Release)",
+      "displayName": "Release",
      "inherits": "spark",
      "configurePreset": "spark",
      "configuration": "Release"
@@ -284,42 +265,43 @@
        "lhs": "${hostSystemName}",
        "rhs": "Linux"
      },
-      "hidden": true
+      "hidden": true,
+      "targets": [
+        "data_files_dir",
+        "admin_web_app_bundle",
+        "install",
+        "libspark",
+        "rapidjson_project",
+        "server_layout",
+        "sparkd"
+      ]
    },

    {
      "name": "sparkd-debug-clang",
      "configurePreset": "sparkd-debug-clang",
-      "displayName": "Sparkd (Debug | Clang)",
+      "displayName": "Debug (clang)",
      "inherits": "sparkd"
    },

-    {
-      "name": "sparkd-debug-clang-j6",
-      "configurePreset": "sparkd-debug-clang",
-      "displayName": "Sparkd (Debug | Clang | jobs=6)",
-      "inherits": "sparkd",
-      "jobs": 6
-    },
-
    {
      "name": "sparkd-debug-gcc",
      "configurePreset": "sparkd-debug-gcc",
-      "displayName": "Sparkd (Debug | Gcc)",
+      "displayName": "Debug (gcc)",
      "inherits": "sparkd"
    },

    {
      "name": "sparkd-release-clang",
      "configurePreset": "sparkd-release-clang",
-      "displayName": "Sparkd (Release | Clang)",
+      "displayName": "Release (clang)",
      "inherits": "sparkd"
    },

    {
      "name": "sparkd-release-gcc",
      "configurePreset": "sparkd-release-gcc",
-      "displayName": "Sparkd (Release | Gcc)",
+      "displayName": "Release (gcc)",
      "inherits": "sparkd"
    },

@@ -336,14 +318,14 @@

    {
      "name": "spark-api-server-windows-debug",
-      "displayName": "Spark Api Server (Windows | Debug)",
+      "displayName": "Spark Api Server (Debug)",
      "inherits": "spark-api-server-windows",
      "configuration": "Debug"
    },

    {
      "name": "spark-api-server-windows-release",
-      "displayName": "Spark Api Server (Windows | Release)",
+      "displayName": "Spark Api Server (Debug)",
      "inherits": "spark-api-server-windows",
      "configuration": "Release"
    },
@@ -362,14 +344,14 @@
    {
      "name": "spark-api-server-linux-debug",
      "inherits": "spark-api-server-linux",
-      "displayName": "Spark Api Server (Linux | Debug)",
+      "displayName": "Spark Api Server (Debug)",
      "configuration": "Debug"
    },

    {
      "name": "spark-api-server-linux-release",
      "inherits": "spark-api-server-linux",
-      "displayName": "Spark Api Server (Linux | Release)",
+      "displayName": "Spark Api Server (Release)",
      "configuration": "Release"
    },

@@ -389,30 +371,16 @@
    },

    {
-      "name": "data-stream-suite-windows-debug",
-      "displayName": "Data Stream Suite (Windows | Debug)",
-      "configurePreset": "data-stream-suite-windows",
+      "name": "data-stream-suite-debug",
+      "displayName": "DataStreamSuite (Debug)",
+      "configurePreset": "data-stream-suite",
      "configuration": "Debug"
    },

    {
-      "name": "data-stream-suite-windows-release",
-      "displayName": "Data Stream Suite (Windows | Release)",
-      "configurePreset": "data-stream-suite-windows",
-      "configuration": "Release"
-    },
-
-    {
-      "name": "data-stream-suite-linux-debug",
-      "displayName": "Data Stream Suite (Linux | Debug)",
-      "configurePreset": "data-stream-suite-linux",
-      "configuration": "Debug"
-    },
-
-    {
-      "name": "data-stream-suite-linux-release",
-      "displayName": "Data Stream Suite (Linux | Release)",
-      "configurePreset": "data-stream-suite-linux",
+      "name": "data-stream-suite-release",
+      "displayName": "DataStreamSuite (Release)",
+      "configurePreset": "data-stream-suite",
      "configuration": "Release"
    }
  ]
@@ -14,36 +14,6 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "spark_test", "src\client\sp
 EndProject
 Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Spark.Console", "src\client\Spark.Console\Spark.Console.csproj", "{89D666E6-B500-498F-B026-4D167F68ADFC}"
 EndProject
-Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "src", "src", "{267F81EF-A8AB-49FD-9F56-D7EC3C1F7C7C}"
-EndProject
-Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "DataStreamSuite", "DataStreamSuite", "{C09ED521-D2DF-40BC-A76F-EB3CB870B2F6}"
-EndProject
-Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "DataStreamCore", "src\DataStreamSuite\DataStreamCore\DataStreamCore.csproj", "{EF26A475-6101-4C9C-A717-D60BB92A2302}"
-EndProject
-Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "SharedStreams", "src\DataStreamSuite\SharedStreams\SharedStreams.csproj", "{911B08A3-515A-441D-84C8-8A43740B5FC1}"
-EndProject
-Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "PubSub", "src\DataStreamSuite\PubSub\PubSub.csproj", "{14140310-2233-430A-821B-F3058A25234B}"
-EndProject
-Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "BaseUtilities", "src\DataStreamSuite\BaseUtilities\BaseUtilities.csproj", "{43146EA8-6FCC-46B8-88B0-42F9BB5E6E0A}"
-EndProject
-Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "ExpressionCore", "src\DataStreamSuite\ExpressionCore\ExpressionCore.csproj", "{6F22DB96-6EAB-4034-99D9-287021C36F01}"
-EndProject
-Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "ArithmeticExpressions", "src\DataStreamSuite\ArithmeticExpressions\ArithmeticExpressions.csproj", "{A5C33956-A41F-49B3-A41D-1082CBA59D02}"
-EndProject
-Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "DataStreamExpressions", "src\DataStreamSuite\DataStreamExpressions\DataStreamExpressions.csproj", "{2FE745C3-1EDF-4421-8CD7-A63EE0C1AE43}"
-EndProject
-Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Bts.ColumnSortExpressions", "src\DataStreamSuite\Bts.ColumnSortExpressions\Bts.ColumnSortExpressions.csproj", "{FDA791E0-2B91-4AD5-B91D-CF920A4C4E38}"
-EndProject
-Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "ColumnMapExpressions", "src\DataStreamSuite\ColumnMapExpressions\ColumnMapExpressions.csproj", "{B945A183-2174-4ACB-B289-8A5C546656FE}"
-EndProject
-Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "BooleanExpressions", "src\DataStreamSuite\BooleanExpressions\BooleanExpressions.csproj", "{9F6D7DE8-185B-4C1D-B809-3314D092DF28}"
-EndProject
-Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Bts.ColumnExpressions", "src\DataStreamSuite\Bts.ColumnExpressions\Bts.ColumnExpressions.csproj", "{6BD3F6D8-B6FC-44E8-B8FE-06352F638B9B}"
-EndProject
-Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "DataStream", "src\DataStreamSuite\DataStream\DataStream.csproj", "{39AEA075-6457-46C0-8821-1158BE6EFA57}"
-EndProject
-Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Bts.Net", "src\DataStreamSuite\Bts.Net\Bts.Net.csproj", "{CCDCB00A-CABF-4D82-8A5F-33EEC8ACE51D}"
-EndProject
 Global
 	GlobalSection(SolutionConfigurationPlatforms) = preSolution
 		Debug|x64 = Debug|x64
@@ -100,110 +70,6 @@ Global
 		{89D666E6-B500-498F-B026-4D167F68ADFC}.Release|x64.Build.0 = Release|x64
 		{89D666E6-B500-498F-B026-4D167F68ADFC}.RelWithDebInfo|x64.ActiveCfg = Debug|x64
 		{89D666E6-B500-498F-B026-4D167F68ADFC}.RelWithDebInfo|x64.Build.0 = Debug|x64
-		{EF26A475-6101-4C9C-A717-D60BB92A2302}.Debug|x64.ActiveCfg = Debug|Any CPU
-		{EF26A475-6101-4C9C-A717-D60BB92A2302}.Debug|x64.Build.0 = Debug|Any CPU
-		{EF26A475-6101-4C9C-A717-D60BB92A2302}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
-		{EF26A475-6101-4C9C-A717-D60BB92A2302}.MinSizeRel|x64.Build.0 = Debug|Any CPU
-		{EF26A475-6101-4C9C-A717-D60BB92A2302}.Release|x64.ActiveCfg = Release|Any CPU
-		{EF26A475-6101-4C9C-A717-D60BB92A2302}.Release|x64.Build.0 = Release|Any CPU
-		{EF26A475-6101-4C9C-A717-D60BB92A2302}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
-		{EF26A475-6101-4C9C-A717-D60BB92A2302}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
-		{911B08A3-515A-441D-84C8-8A43740B5FC1}.Debug|x64.ActiveCfg = Debug|Any CPU
-		{911B08A3-515A-441D-84C8-8A43740B5FC1}.Debug|x64.Build.0 = Debug|Any CPU
-		{911B08A3-515A-441D-84C8-8A43740B5FC1}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
-		{911B08A3-515A-441D-84C8-8A43740B5FC1}.MinSizeRel|x64.Build.0 = Debug|Any CPU
-		{911B08A3-515A-441D-84C8-8A43740B5FC1}.Release|x64.ActiveCfg = Release|Any CPU
-		{911B08A3-515A-441D-84C8-8A43740B5FC1}.Release|x64.Build.0 = Release|Any CPU
-		{911B08A3-515A-441D-84C8-8A43740B5FC1}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
-		{911B08A3-515A-441D-84C8-8A43740B5FC1}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
-		{14140310-2233-430A-821B-F3058A25234B}.Debug|x64.ActiveCfg = Debug|Any CPU
-		{14140310-2233-430A-821B-F3058A25234B}.Debug|x64.Build.0 = Debug|Any CPU
-		{14140310-2233-430A-821B-F3058A25234B}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
-		{14140310-2233-430A-821B-F3058A25234B}.MinSizeRel|x64.Build.0 = Debug|Any CPU
-		{14140310-2233-430A-821B-F3058A25234B}.Release|x64.ActiveCfg = Release|Any CPU
-		{14140310-2233-430A-821B-F3058A25234B}.Release|x64.Build.0 = Release|Any CPU
-		{14140310-2233-430A-821B-F3058A25234B}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
-		{14140310-2233-430A-821B-F3058A25234B}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
-		{43146EA8-6FCC-46B8-88B0-42F9BB5E6E0A}.Debug|x64.ActiveCfg = Debug|Any CPU
-		{43146EA8-6FCC-46B8-88B0-42F9BB5E6E0A}.Debug|x64.Build.0 = Debug|Any CPU
-		{43146EA8-6FCC-46B8-88B0-42F9BB5E6E0A}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
-		{43146EA8-6FCC-46B8-88B0-42F9BB5E6E0A}.MinSizeRel|x64.Build.0 = Debug|Any CPU
-		{43146EA8-6FCC-46B8-88B0-42F9BB5E6E0A}.Release|x64.ActiveCfg = Release|Any CPU
-		{43146EA8-6FCC-46B8-88B0-42F9BB5E6E0A}.Release|x64.Build.0 = Release|Any CPU
-		{43146EA8-6FCC-46B8-88B0-42F9BB5E6E0A}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
-		{43146EA8-6FCC-46B8-88B0-42F9BB5E6E0A}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
-		{6F22DB96-6EAB-4034-99D9-287021C36F01}.Debug|x64.ActiveCfg = Debug|Any CPU
-		{6F22DB96-6EAB-4034-99D9-287021C36F01}.Debug|x64.Build.0 = Debug|Any CPU
-		{6F22DB96-6EAB-4034-99D9-287021C36F01}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
-		{6F22DB96-6EAB-4034-99D9-287021C36F01}.MinSizeRel|x64.Build.0 = Debug|Any CPU
-		{6F22DB96-6EAB-4034-99D9-287021C36F01}.Release|x64.ActiveCfg = Release|Any CPU
-		{6F22DB96-6EAB-4034-99D9-287021C36F01}.Release|x64.Build.0 = Release|Any CPU
-		{6F22DB96-6EAB-4034-99D9-287021C36F01}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
-		{6F22DB96-6EAB-4034-99D9-287021C36F01}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
-		{A5C33956-A41F-49B3-A41D-1082CBA59D02}.Debug|x64.ActiveCfg = Debug|Any CPU
-		{A5C33956-A41F-49B3-A41D-1082CBA59D02}.Debug|x64.Build.0 = Debug|Any CPU
-		{A5C33956-A41F-49B3-A41D-1082CBA59D02}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
-		{A5C33956-A41F-49B3-A41D-1082CBA59D02}.MinSizeRel|x64.Build.0 = Debug|Any CPU
-		{A5C33956-A41F-49B3-A41D-1082CBA59D02}.Release|x64.ActiveCfg = Release|Any CPU
-		{A5C33956-A41F-49B3-A41D-1082CBA59D02}.Release|x64.Build.0 = Release|Any CPU
-		{A5C33956-A41F-49B3-A41D-1082CBA59D02}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
-		{A5C33956-A41F-49B3-A41D-1082CBA59D02}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
-		{2FE745C3-1EDF-4421-8CD7-A63EE0C1AE43}.Debug|x64.ActiveCfg = Debug|Any CPU
-		{2FE745C3-1EDF-4421-8CD7-A63EE0C1AE43}.Debug|x64.Build.0 = Debug|Any CPU
-		{2FE745C3-1EDF-4421-8CD7-A63EE0C1AE43}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
-		{2FE745C3-1EDF-4421-8CD7-A63EE0C1AE43}.MinSizeRel|x64.Build.0 = Debug|Any CPU
-		{2FE745C3-1EDF-4421-8CD7-A63EE0C1AE43}.Release|x64.ActiveCfg = Release|Any CPU
-		{2FE745C3-1EDF-4421-8CD7-A63EE0C1AE43}.Release|x64.Build.0 = Release|Any CPU
-		{2FE745C3-1EDF-4421-8CD7-A63EE0C1AE43}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
-		{2FE745C3-1EDF-4421-8CD7-A63EE0C1AE43}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
-		{FDA791E0-2B91-4AD5-B91D-CF920A4C4E38}.Debug|x64.ActiveCfg = Debug|Any CPU
-		{FDA791E0-2B91-4AD5-B91D-CF920A4C4E38}.Debug|x64.Build.0 = Debug|Any CPU
-		{FDA791E0-2B91-4AD5-B91D-CF920A4C4E38}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
-		{FDA791E0-2B91-4AD5-B91D-CF920A4C4E38}.MinSizeRel|x64.Build.0 = Debug|Any CPU
-		{FDA791E0-2B91-4AD5-B91D-CF920A4C4E38}.Release|x64.ActiveCfg = Release|Any CPU
-		{FDA791E0-2B91-4AD5-B91D-CF920A4C4E38}.Release|x64.Build.0 = Release|Any CPU
-		{FDA791E0-2B91-4AD5-B91D-CF920A4C4E38}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
-		{FDA791E0-2B91-4AD5-B91D-CF920A4C4E38}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
-		{B945A183-2174-4ACB-B289-8A5C546656FE}.Debug|x64.ActiveCfg = Debug|Any CPU
-		{B945A183-2174-4ACB-B289-8A5C546656FE}.Debug|x64.Build.0 = Debug|Any CPU
-		{B945A183-2174-4ACB-B289-8A5C546656FE}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
-		{B945A183-2174-4ACB-B289-8A5C546656FE}.MinSizeRel|x64.Build.0 = Debug|Any CPU
-		{B945A183-2174-4ACB-B289-8A5C546656FE}.Release|x64.ActiveCfg = Release|Any CPU
-		{B945A183-2174-4ACB-B289-8A5C546656FE}.Release|x64.Build.0 = Release|Any CPU
-		{B945A183-2174-4ACB-B289-8A5C546656FE}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
-		{B945A183-2174-4ACB-B289-8A5C546656FE}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
-		{39AEA075-6457-46C0-8821-1158BE6EFA57}.Debug|x64.ActiveCfg = Debug|Any CPU
-		{39AEA075-6457-46C0-8821-1158BE6EFA57}.Debug|x64.Build.0 = Debug|Any CPU
-		{39AEA075-6457-46C0-8821-1158BE6EFA57}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
-		{39AEA075-6457-46C0-8821-1158BE6EFA57}.MinSizeRel|x64.Build.0 = Debug|Any CPU
-		{39AEA075-6457-46C0-8821-1158BE6EFA57}.Release|x64.ActiveCfg = Release|Any CPU
-		{39AEA075-6457-46C0-8821-1158BE6EFA57}.Release|x64.Build.0 = Release|Any CPU
-		{39AEA075-6457-46C0-8821-1158BE6EFA57}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
-		{39AEA075-6457-46C0-8821-1158BE6EFA57}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
-		{9F6D7DE8-185B-4C1D-B809-3314D092DF28}.Debug|x64.ActiveCfg = Debug|Any CPU
-		{9F6D7DE8-185B-4C1D-B809-3314D092DF28}.Debug|x64.Build.0 = Debug|Any CPU
-		{9F6D7DE8-185B-4C1D-B809-3314D092DF28}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
-		{9F6D7DE8-185B-4C1D-B809-3314D092DF28}.MinSizeRel|x64.Build.0 = Debug|Any CPU
-		{9F6D7DE8-185B-4C1D-B809-3314D092DF28}.Release|x64.ActiveCfg = Release|Any CPU
-		{9F6D7DE8-185B-4C1D-B809-3314D092DF28}.Release|x64.Build.0 = Release|Any CPU
-		{9F6D7DE8-185B-4C1D-B809-3314D092DF28}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
-		{9F6D7DE8-185B-4C1D-B809-3314D092DF28}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
-		{6BD3F6D8-B6FC-44E8-B8FE-06352F638B9B}.Debug|x64.ActiveCfg = Debug|Any CPU
-		{6BD3F6D8-B6FC-44E8-B8FE-06352F638B9B}.Debug|x64.Build.0 = Debug|Any CPU
-		{6BD3F6D8-B6FC-44E8-B8FE-06352F638B9B}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
-		{6BD3F6D8-B6FC-44E8-B8FE-06352F638B9B}.MinSizeRel|x64.Build.0 = Debug|Any CPU
-		{6BD3F6D8-B6FC-44E8-B8FE-06352F638B9B}.Release|x64.ActiveCfg = Release|Any CPU
-		{6BD3F6D8-B6FC-44E8-B8FE-06352F638B9B}.Release|x64.Build.0 = Release|Any CPU
-		{6BD3F6D8-B6FC-44E8-B8FE-06352F638B9B}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
-		{6BD3F6D8-B6FC-44E8-B8FE-06352F638B9B}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
-		{CCDCB00A-CABF-4D82-8A5F-33EEC8ACE51D}.Debug|x64.ActiveCfg = Debug|Any CPU
-		{CCDCB00A-CABF-4D82-8A5F-33EEC8ACE51D}.Debug|x64.Build.0 = Debug|Any CPU
-		{CCDCB00A-CABF-4D82-8A5F-33EEC8ACE51D}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
-		{CCDCB00A-CABF-4D82-8A5F-33EEC8ACE51D}.MinSizeRel|x64.Build.0 = Debug|Any CPU
-		{CCDCB00A-CABF-4D82-8A5F-33EEC8ACE51D}.Release|x64.ActiveCfg = Release|Any CPU
-		{CCDCB00A-CABF-4D82-8A5F-33EEC8ACE51D}.Release|x64.Build.0 = Release|Any CPU
-		{CCDCB00A-CABF-4D82-8A5F-33EEC8ACE51D}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
-		{CCDCB00A-CABF-4D82-8A5F-33EEC8ACE51D}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
 	EndGlobalSection
 	GlobalSection(SolutionProperties) = preSolution
 		HideSolutionNode = FALSE
@@ -214,19 +80,4 @@ Global
 	GlobalSection(Performance) = preSolution
 		HasPerformanceSessions = true
 	EndGlobalSection
-	GlobalSection(NestedProjects) = preSolution
-		{C09ED521-D2DF-40BC-A76F-EB3CB870B2F6} = {267F81EF-A8AB-49FD-9F56-D7EC3C1F7C7C}
-		{EF26A475-6101-4C9C-A717-D60BB92A2302} = {C09ED521-D2DF-40BC-A76F-EB3CB870B2F6}
-		{911B08A3-515A-441D-84C8-8A43740B5FC1} = {C09ED521-D2DF-40BC-A76F-EB3CB870B2F6}
-		{14140310-2233-430A-821B-F3058A25234B} = {C09ED521-D2DF-40BC-A76F-EB3CB870B2F6}
-		{43146EA8-6FCC-46B8-88B0-42F9BB5E6E0A} = {C09ED521-D2DF-40BC-A76F-EB3CB870B2F6}
-		{6F22DB96-6EAB-4034-99D9-287021C36F01} = {C09ED521-D2DF-40BC-A76F-EB3CB870B2F6}
-		{A5C33956-A41F-49B3-A41D-1082CBA59D02} = {C09ED521-D2DF-40BC-A76F-EB3CB870B2F6}
-		{2FE745C3-1EDF-4421-8CD7-A63EE0C1AE43} = {C09ED521-D2DF-40BC-A76F-EB3CB870B2F6}
-		{FDA791E0-2B91-4AD5-B91D-CF920A4C4E38} = {C09ED521-D2DF-40BC-A76F-EB3CB870B2F6}
-		{B945A183-2174-4ACB-B289-8A5C546656FE} = {C09ED521-D2DF-40BC-A76F-EB3CB870B2F6}
-		{39AEA075-6457-46C0-8821-1158BE6EFA57} = {C09ED521-D2DF-40BC-A76F-EB3CB870B2F6}
-		{9F6D7DE8-185B-4C1D-B809-3314D092DF28} = {C09ED521-D2DF-40BC-A76F-EB3CB870B2F6}
-		{6BD3F6D8-B6FC-44E8-B8FE-06352F638B9B} = {C09ED521-D2DF-40BC-A76F-EB3CB870B2F6}
-	EndGlobalSection
 EndGlobal
@@ -1,7 +0,0 @@
-include_guard(GLOBAL)
-
-macro(export_to_parent)
-  foreach(var IN ITEMS ${ARGN})
-    set(${var} ${${var}} PARENT_SCOPE)
-  endforeach()
-endmacro()
@@ -1,142 +0,0 @@
-include_guard(DIRECTORY)
-include(${CMAKE_SOURCE_DIR}/cmake/ExportToParent.cmake)
-include(${CMAKE_SOURCE_DIR}/cmake/InstallBasic.cmake)
-include(${CMAKE_SOURCE_DIR}/cmake/ProtobufGenerateCpp.cmake)
-include(${CMAKE_SOURCE_DIR}/cmake/SetFirstDefined.cmake)
-include(${CMAKE_SOURCE_DIR}/cmake/safe_dep_glob.cmake)
-
-function(init_library)
-  set(_options SHARED STATIC)
-  set(_one_value_args
-    NAME
-    SOURCE_DIR
-    INCLUDE_DIR
-  )
-  set(_multi_value_args)
-  cmake_parse_arguments(_ARGS "${_options}" "${_one_value_args}" "${_multi_value_args}" ${ARGN})
-  if (_ARGS_SHARED AND _ARGS_STATIC)
-    message(FATAL_ERROR "Invalid args passed to init_library: cannot specify STATIC and SHARED")
-  endif()
-  set_first_defined(_name _ARGS_NAME PROJECT_NAME)
-  set_first_defined(_source_dir _ARGS_SOURCE_DIR _name)
-  set_first_defined(_include_dir _ARGS_INCLUDE_DIR _source_dir)
-
-  # Cpp sources
-  file(
-    GLOB_RECURSE ${_name}_SOURCES
-    RELATIVE ${CMAKE_CURRENT_SOURCE_DIR}
-    ${_source_dir}/*.cpp
-  )
-  # Note: cpp files inside 'tests' dir(s) are considered tests.
-  # Remove tests from main sources, and add them to test sources
-  foreach(item ${${_name}_SOURCES})
-    if(${item} MATCHES "tests/.*\.cpp$")
-      list(APPEND ${_name}_TEST_SOURCES ${item})
-    endif()
-  endforeach()
-  foreach(item ${${_name}_TEST_SOURCES})
-    list(REMOVE_ITEM ${_name}_SOURCES ${item})
-  endforeach()
-
-  # Proto sources 
-  file(
-    GLOB_RECURSE ${_name}_PROTO_SOURCES
-    RELATIVE ${CMAKE_CURRENT_SOURCE_DIR}
-    ${_source_dir}/protobufs/*.proto
-  )
-  update_deps_file("${${_name}_PROTO_SOURCES};")
-  protobuf_generate_cpp(
-    CXX_SOURCES ${_name}_CXX_PROTO_SOURCES
-    CXX_HEADERS ${_name}_CXX_PROTO_HEADERS
-    CXX_OUT_DIR ${CMAKE_CURRENT_BINARY_DIR}
-    PROTO_INCLUDE_DIRS
-      ${EXTERNALS_INCLUDE}
-      ${CMAKE_CURRENT_SOURCE_DIR}
-    PROTO_SOURCES
-      ${${_name}_PROTO_SOURCES}
-  )
-
-  export_to_parent(
-    ${_name}_SOURCES
-    ${_name}_TEST_SOURCES
-    ${_name}_PROTO_SOURCES
-    ${_name}_CXX_PROTO_SOURCES
-    ${_name}_CXX_PROTO_HEADERS
-  )
-
-  # Add main lib
-  if (_ARGS_STATIC)
-    add_library(${_name} STATIC ${${_name}_SOURCES} ${${_name}_CXX_PROTO_SOURCES})
-  elseif(_ARGS_SHARED)
-    add_library(${_name} SHARED ${${_name}_SOURCES} ${${_name}_CXX_PROTO_SOURCES})
-  else()
-    add_library(${_name} ${${_name}_SOURCES} ${${_name}_CXX_PROTO_SOURCES})
-  endif()
-  target_include_directories(
-    ${_name}
-    SYSTEM
-    BEFORE
-    PUBLIC
-      $<BUILD_INTERFACE:${EXTERNALS_ROOT}/include>
-      $<BUILD_INTERFACE:${EXTERNALS_ROOT}/include/zlib>
-  )
-  target_include_directories(
-    ${_name}
-    PUBLIC
-      $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}>
-      $<INSTALL_INTERFACE:${CMAKE_INSTALL_INCLUDEDIR}>
-  )
-  # Include protos (if any)
-  if (${_name}_PROTO_SOURCES)
-    target_include_directories(
-      ${_name}
-      PUBLIC
-        $<BUILD_INTERFACE:${CMAKE_CURRENT_BINARY_DIR}>
-    )
-  endif()
-  target_link_directories(
-    ${_name}
-    PUBLIC
-      $<BUILD_INTERFACE:${CMAKE_CURRENT_BINARY_DIR}>
-      $<BUILD_INTERFACE:${EXTERNALS_ROOT}/lib/$<CONFIG>>
-      $<INSTALL_INTERFACE:${CMAKE_INSTALL_LIBDIR}>
-  )
-  # Install main lib
-  foreach(_hdr ${${_name}_CXX_PROTO_HEADERS})
-    file(RELATIVE_PATH _relative_path ${CMAKE_CURRENT_BINARY_DIR} ${_hdr})
-    install(FILES ${_hdr} DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}/${_relative_path})
-  endforeach()
-  install_basic(
-    NAME ${_name}
-    INCLUDE_DIR ${_include_dir}
-  )
-
-  # Add tests executable (if there are any)
-  if (${_name}_TEST_SOURCES AND NOT WIN32)
-    add_executable(${_name}-tests ${${_name}_TEST_SOURCES})
-
-    get_target_property(_lib_type ${_name} TYPE)
-    if (_lib_type STREQUAL "SHARED_LIBRARY")
-      # Set RPATH so the tests exe knows where to find the shared lib.
-      set_target_properties(
-        ${_name}-tests
-        PROPERTIES
-          BUILD_RPATH "$ORIGIN"
-          INSTALL_RPATH "$ORIGIN/../${CMAKE_INSTALL_LIBDIR}"
-      )
-    endif()
-
-    # Link main lib
-    target_link_directories(
-      ${_name}-tests
-      PRIVATE
-        $<BUILD_INTERFACE:${CMAKE_CURRENT_BINARY_DIR}>
-        $<INSTALL_INTERFACE:${CMAKE_INSTALL_LIBDIR}>
-    )
-    target_link_libraries(${_name}-tests PRIVATE ${_name})
-
-    # Catch2
-    find_package(Catch2 CONFIG REQUIRED COMPONENTS Catch2WithMain)
-    target_link_libraries(${_name}-tests PRIVATE Catch2::Catch2WithMain)
-  endif()
-endfunction()
@@ -1,39 +0,0 @@
-include_guard(DIRECTORY)
-include(GNUInstallDirs)
-include(${CMAKE_SOURCE_DIR}/cmake/PackBasic.cmake)
-include(${CMAKE_SOURCE_DIR}/cmake/SetFirstDefined.cmake)
-
-function(install_basic)
-  set(_options)
-  set(_one_value_args NAME INCLUDE_DIR)
-  set(_multi_value_args)
-  cmake_parse_arguments(_ARGS "${_options}" "${_one_value_args}" "${_multi_value_args}" ${ARGN})
-  set_first_defined(_name _ARGS_NAME PROJECT_NAME)
-  set_first_defined(_include_dir _ARGS_INCLUDE_DIR _name)
-
-  file(
-    GLOB_RECURSE ${_name}_HEADERS
-    RELATIVE ${CMAKE_CURRENT_SOURCE_DIR}
-    ${_include_dir}/*.h)
-
-  target_sources(${_name}
-    PUBLIC
-      FILE_SET HEADERS FILES ${${_name}_HEADERS})
-
-  install(
-    TARGETS ${_name}
-    EXPORT ${_name}Targets
-    RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR}
-    LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR}
-    ARCHIVE DESTINATION ${CMAKE_INSTALL_LIBDIR}
-    FILE_SET HEADERS DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}
-  )
-
-  install(
-    EXPORT ${_name}Targets
-    FILE ${_name}Targets.cmake
-    NAMESPACE ${_name}::
-    DESTINATION ${CMAKE_INSTALL_LIBDIR}/cmake/${_name})
-
-  pack_basic()
-endfunction()
@@ -1,44 +0,0 @@
-include_guard(DIRECTORY)
-include(CMakePackageConfigHelpers)
-include(${CMAKE_SOURCE_DIR}/cmake/SetFirstDefined.cmake)
-
-function(pack_basic)
-  set(_options)
-  set(_one_value_args NAME)
-  set(_multi_value_args)
-  cmake_parse_arguments(_ARGS "${_options}" "${_one_value_args}" "${_multi_value_args}" ${ARGN})
-  set_first_defined(_name _ARGS_NAME PROJECT_NAME)
-
-  find_package(Git QUIET)
-
-  if(GIT_FOUND)
-    execute_process(
-      COMMAND ${GIT_EXECUTABLE} describe --tags --always --dirty
-      WORKING_DIRECTORY ${CMAKE_SOURCE_DIR}
-      OUTPUT_VARIABLE PROJECT_VERSION
-      OUTPUT_STRIP_TRAILING_WHITESPACE
-    )
-  endif()
-
-  if(NOT GIT_FOUND OR PROJECT_VERSION STREQUAL "")
-    set(PROJECT_VERSION "0.0.0")
-  endif()
-
-  write_basic_package_version_file(
-    "${CMAKE_CURRENT_BINARY_DIR}/${_name}ConfigVersion.cmake"
-    VERSION ${PROJECT_VERSION}
-    COMPATIBILITY AnyNewerVersion
-  )
-
-  configure_package_config_file(
-    "${CMAKE_CURRENT_SOURCE_DIR}/Config.cmake.in"
-    "${CMAKE_CURRENT_BINARY_DIR}/${_name}Config.cmake"
-    INSTALL_DESTINATION ${CMAKE_INSTALL_LIBDIR}/cmake/${_name}
-  )
-
-  install(
-    FILES
-      "${CMAKE_CURRENT_BINARY_DIR}/${_name}Config.cmake"
-      "${CMAKE_CURRENT_BINARY_DIR}/${_name}ConfigVersion.cmake"
-    DESTINATION ${CMAKE_INSTALL_LIBDIR}/cmake/${_name})
-endfunction()
@@ -0,0 +1,145 @@
+set(Protobuf_PROTOC_EXECUTABLE "${SPARK_BIN}/protoc")
+set(PROTOBUF_INCLUDE_DIRS "${SPARK_INCLUDE}")
+set(PROTOBUF_STATIC_LIB "libprotobuf")
+
+function(Protobuf_Generate_Cpp SRCS HDRS)
+  set(${SRCS})
+  set(${HDRS})
+
+  foreach(FIL ${ARGN})
+    get_filename_component(ABS_FIL ${FIL} ABSOLUTE)
+    get_filename_component(FIL_WE ${FIL} NAME_WE)
+    get_filename_component(FIL_DIR ${FIL} DIRECTORY)
+    if(FIL_DIR)
+      set(FIL_WE "${FIL_DIR}/${FIL_WE}")
+    endif()
+
+    list(APPEND ${SRCS} "${CMAKE_CURRENT_BINARY_DIR}/${FIL_WE}.pb.cc")
+    list(APPEND ${HDRS} "${CMAKE_CURRENT_BINARY_DIR}/${FIL_WE}.pb.h")
+
+    add_custom_command(
+      OUTPUT "${CMAKE_CURRENT_BINARY_DIR}/${FIL_WE}.pb.cc"
+             "${CMAKE_CURRENT_BINARY_DIR}/${FIL_WE}.pb.h"
+      COMMAND
+        ${Protobuf_PROTOC_EXECUTABLE} ARGS --cpp_out=${CMAKE_BINARY_DIR}/src -I
+        ${PROTOBUF_INCLUDE_DIRS} -I ${CMAKE_SOURCE_DIR}/src ${ABS_FIL}
+      DEPENDS ${ABS_FIL}
+      COMMENT "Running C++ protocol buffer compiler on ${FIL}"
+      VERBATIM)
+  endforeach()
+
+  set_source_files_properties(${${SRCS}} ${${HDRS}} PROPERTIES GENERATED TRUE)
+
+  # remove some warnings from protobuf generated files
+  if(WIN32)
+    set_source_files_properties(${${SRCS}} PROPERTIES COMPILE_FLAGS
+                                                      "/wd4309 /wd4838")
+  else()
+    set_source_files_properties(
+      ${${SRCS}}
+      PROPERTIES
+        COMPILE_FLAGS
+        "-Wno-unused-parameter -Wno-effc++ -Wno-pedantic -Wno-sign-compare -Wno-uninitialized"
+    )
+  endif()
+
+  set(${SRCS}
+      ${${SRCS}}
+      PARENT_SCOPE)
+  set(${HDRS}
+      ${${HDRS}}
+      PARENT_SCOPE)
+endfunction()
+
+function(Service_Compiler_Generate_Cpp SRCS HDRS)
+  set(${SRCS})
+  set(${HDRS})
+
+  foreach(FIL ${ARGN})
+    get_filename_component(ABS_FIL ${FIL} ABSOLUTE)
+    get_filename_component(FIL_WE ${FIL} NAME_WE)
+    get_filename_component(FIL_DIR ${FIL} DIRECTORY)
+    if(FIL_DIR)
+      set(FIL_WE "${FIL_DIR}/${FIL_WE}")
+    endif()
+
+    list(APPEND ${SRCS} "${CMAKE_CURRENT_BINARY_DIR}/${FIL_WE}.pb.cc")
+    list(APPEND ${HDRS} "${CMAKE_CURRENT_BINARY_DIR}/${FIL_WE}.pb.h")
+
+    list(APPEND ${SRCS} "${CMAKE_CURRENT_BINARY_DIR}/${FIL_WE}.client.pb.cc")
+    list(APPEND ${HDRS} "${CMAKE_CURRENT_BINARY_DIR}/${FIL_WE}.client.pb.h")
+
+    list(APPEND ${SRCS} "${CMAKE_CURRENT_BINARY_DIR}/${FIL_WE}.server.pb.cc")
+    list(APPEND ${HDRS} "${CMAKE_CURRENT_BINARY_DIR}/${FIL_WE}.server.pb.h")
+
+    set(TEMPLATES
+        ${CMAKE_CURRENT_SOURCE_DIR}/api/service_compiler/templates/cpp/client.pb.cc
+        ${CMAKE_CURRENT_SOURCE_DIR}/api/service_compiler/templates/cpp/client.pb.h
+        ${CMAKE_CURRENT_SOURCE_DIR}/api/service_compiler/templates/cpp/server.pb.cc
+        ${CMAKE_CURRENT_SOURCE_DIR}/api/service_compiler/templates/cpp/server.pb.h
+    )
+
+    if(WIN32)
+      add_custom_command(
+        OUTPUT "${CMAKE_CURRENT_BINARY_DIR}/${FIL_WE}.pb.cc"
+               "${CMAKE_CURRENT_BINARY_DIR}/${FIL_WE}.pb.h"
+               "${CMAKE_CURRENT_BINARY_DIR}/${FIL_WE}.client.pb.cc"
+               "${CMAKE_CURRENT_BINARY_DIR}/${FIL_WE}.client.pb.h"
+               "${CMAKE_CURRENT_BINARY_DIR}/${FIL_WE}.server.pb.cc"
+               "${CMAKE_CURRENT_BINARY_DIR}/${FIL_WE}.server.pb.h"
+        COMMAND SET PYTHONPATH=${SPARK_PYTHON}
+        COMMAND ${CMAKE_SOURCE_DIR}/.venv/Scripts/activate.bat
+        COMMAND
+          ${Protobuf_PROTOC_EXECUTABLE} ARGS --cpp_out=${CMAKE_BINARY_DIR}/src
+          --plugin=protoc-gen-service=${CMAKE_SOURCE_DIR}/src/lib/api/service_compiler/plugin.cmd
+          --service_out=lang=cpp:${CMAKE_BINARY_DIR}/src -I
+          ${PROTOBUF_INCLUDE_DIRS} -I ${CMAKE_SOURCE_DIR}/src ${ABS_FIL}
+        DEPENDS ${ABS_FIL} ${TEMPLATES}
+                ${CMAKE_CURRENT_SOURCE_DIR}/api/service_compiler/plugin.py
+        COMMENT "Running C++ service compiler on ${FIL}"
+        VERBATIM)
+    else()
+      add_custom_command(
+        OUTPUT "${CMAKE_CURRENT_BINARY_DIR}/${FIL_WE}.pb.cc"
+               "${CMAKE_CURRENT_BINARY_DIR}/${FIL_WE}.pb.h"
+               "${CMAKE_CURRENT_BINARY_DIR}/${FIL_WE}.client.pb.cc"
+               "${CMAKE_CURRENT_BINARY_DIR}/${FIL_WE}.client.pb.h"
+               "${CMAKE_CURRENT_BINARY_DIR}/${FIL_WE}.server.pb.cc"
+               "${CMAKE_CURRENT_BINARY_DIR}/${FIL_WE}.server.pb.h"
+        COMMAND ${CMAKE_SOURCE_DIR}/tools/venv_exec.sh
+        ARGS
+          ${Protobuf_PROTOC_EXECUTABLE}
+          --cpp_out=${CMAKE_BINARY_DIR}/src
+          --plugin=protoc-gen-service=${CMAKE_SOURCE_DIR}/src/lib/api/service_compiler/plugin.py
+          --service_out=lang=cpp,make_file_name=${ABS_FIL}:${CMAKE_BINARY_DIR}/src
+          -I ${PROTOBUF_INCLUDE_DIRS} -I ${CMAKE_SOURCE_DIR}/src ${ABS_FIL}
+        DEPENDS ${ABS_FIL} ${TEMPLATES}
+                ${CMAKE_CURRENT_SOURCE_DIR}/api/service_compiler/plugin.py
+        COMMENT "Running C++ service compiler on ${FIL}"
+        VERBATIM)
+    endif()
+
+  endforeach()
+
+  # remove some warnings from protobuf generated files
+  if(WIN32)
+    set_source_files_properties(${${SRCS}} PROPERTIES COMPILE_FLAGS
+                                                      "/wd4309 /wd4838")
+  else()
+    set_source_files_properties(
+      ${${SRCS}}
+      PROPERTIES
+        COMPILE_FLAGS
+        "-Wno-unused-parameter -Wno-effc++ -Wno-pedantic -Wno-sign-compare -Wno-uninitialized"
+    )
+  endif()
+
+  set(${SRCS}
+      ${${SRCS}}
+      PARENT_SCOPE)
+  set(${HDRS}
+      ${${HDRS}}
+      PARENT_SCOPE)
+
+endfunction()
+
@@ -1,80 +0,0 @@
-include_guard(GLOBAL)
-
-function(protobuf_generate_cpp)
-  set(_options)
-  set(_one_value_args
-    CXX_SOURCES
-    CXX_HEADERS
-    CXX_OUT_DIR
-    PROTOC
-  )
-  set(_multi_value_args
-    PROTO_INCLUDE_DIRS
-    PROTO_SOURCES
-  )
-  cmake_parse_arguments(_ARGS "${_options}" "${_one_value_args}" "${_multi_value_args}" ${ARGN})
-
-  if (NOT DEFINED _ARGS_PROTOC)
-    if (WIN32)
-      set(_ARGS_PROTOC "${EXTERNALS_BIN}/protoc.exe")
-    else()
-      set(_ARGS_PROTOC "${EXTERNALS_BIN}/protoc")
-    endif()
-  endif()
-  set(${_ARGS_CXX_SOURCES})
-  set(${_ARGS_CXX_HEADERS})
-
-  set(_proto_include_cli_args)
-  foreach(_dir ${_ARGS_PROTO_INCLUDE_DIRS})
-    list(APPEND _proto_include_cli_args -I ${_dir})
-  endforeach()
-
-  foreach(_fil ${_ARGS_PROTO_SOURCES})
-    get_filename_component(_abs_fil ${_fil} ABSOLUTE)
-    get_filename_component(_fil_we ${_fil} NAME_WE)
-    get_filename_component(_fil_dir ${_fil} DIRECTORY)
-    if(_fil_dir)
-      set(_fil_we "${_fil_dir}/${_fil_we}")
-    endif()
-    list(APPEND ${_ARGS_CXX_SOURCES} "${_ARGS_CXX_OUT_DIR}/${_fil_we}.pb.cc")
-    list(APPEND ${_ARGS_CXX_HEADERS} "${_ARGS_CXX_OUT_DIR}/${_fil_we}.pb.h")
-    message(STATUS "add_custom_command for ${_fil} -> ${_ARGS_CXX_OUT_DIR}/${_fil_we}.pb.h")
-    add_custom_command(
-      OUTPUT
-        "${_ARGS_CXX_OUT_DIR}/${_fil_we}.pb.cc"
-        "${_ARGS_CXX_OUT_DIR}/${_fil_we}.pb.h"
-      COMMAND
-        ${_ARGS_PROTOC}
-      ARGS
-        --cpp_out=${_ARGS_CXX_OUT_DIR} ${_proto_include_cli_args} ${_fil}
-      DEPENDS ${_abs_fil}
-      COMMENT "Running C++ protocol buffer compiler on ${_fil}"
-      VERBATIM
-    )
-  endforeach()
-
-  set_source_files_properties(
-    ${${_ARGS_CXX_SOURCES}}
-    ${${_ARGS_CXX_HEADERS}}
-    PROPERTIES
-      GENERATED TRUE
-  )
-
-  # remove some warnings from protobuf generated files
-  if(WIN32)
-    set_source_files_properties(
-      ${${_ARGS_CXX_SOURCES}}
-      PROPERTIES
-        COMPILE_FLAGS "/wd4309 /wd4838"
-    )
-  else()
-    set_source_files_properties(
-      ${${_ARGS_CXX_SOURCES}}
-      PROPERTIES
-        COMPILE_FLAGS "-Wno-unused-parameter -Wno-effc++ -Wno-pedantic -Wno-sign-compare -Wno-uninitialized"
-    )
-  endif()
-
-  set(${_ARGS_CXX_SOURCES} ${${_ARGS_CXX_SOURCES}} PARENT_SCOPE)
-  set(${_ARGS_CXX_HEADERS} ${${_ARGS_CXX_HEADERS}} PARENT_SCOPE)
-endfunction()
@@ -1,58 +0,0 @@
-function(protobuf_generate_cpp_config)
-  set(_options)
-  set(_one_value_args
-    PROTO_SOURCES_HEADER
-    PROTO_DEFAULTS_HEADER
-  )
-  set(_multi_value_args
-    PROTO_SOURCES
-    PROTO_DEFAULTS
-  )
-  cmake_parse_arguments(_ARGS "${_options}" "${_one_value_args}" "${_multi_value_args}" ${ARGN})
-
-  # Configure the 'config_proto_files' header. It provides an array of strings populated with the
-  # files in PROTO_SOURCES. At runtime, we use this array to generate a special 'config' proto
-  # message with fields for each message defined in said files. A global instance of this 'config'
-  # proto is then constructed, and populated with [compile-time] defaults, and [runtime] overrides
-  # (i.e., in spark.conf). Calls to `bts::config::get<T>()` retrieve the value from the field
-  # associated with type `T` in this global instance.
-  message(STATUS "PROTO_SOURCES: ${_ARGS_PROTO_SOURCES}")
-  message(STATUS "PROTO_SOURCES_HEADER: ${_ARGS_PROTO_SOURCES_HEADER}")
-  message(STATUS "PROTO_DEFAULTS_HEADER: ${_ARGS_PROTO_DEFAULTS_HEADER}")
-  if (_ARGS_PROTO_SOURCES_HEADER)
-    set(PROTO_SOURCES)
-    set(PROTO_SOURCES_CXX_ARRAY_ITEMS "")
-    foreach(_proto_path ${_ARGS_PROTO_SOURCES})
-      list(APPEND PROTO_SOURCES ${_proto_path})
-      string(APPEND PROTO_SOURCES_CXX_ARRAY_ITEMS "\"${_proto_path}\",\n")
-    endforeach()
-    list(LENGTH PROTO_SOURCES PROTO_SOURCES_LENGTH)
-    message(STATUS "CONFIGURING PROTO_SOURCES_HEADER): ${_ARGS_PROTO_SOURCES_HEADER}")
-    configure_file(${CMAKE_SOURCE_DIR}/cmake/config_proto_files.h.in ${_ARGS_PROTO_SOURCES_HEADER})
-  endif()
-
-  # Generate static defaults. Ideally, this would be a single string, but we use an array because
-  # the compiler complains if the string is longer than 65k.
-  set(_cfg_defaults_str "inline const char * config_static_defaults[] = { ")
-
-  foreach(_cfg_file_path ${_ARGS_PROTO_DEFAULTS})
-    file(READ ${_cfg_file_path} _cfg_file_content)
-    string(REPLACE "\"" "\\\"" _cfg_file_content_escaped ${_cfg_file_content})
-    string(REPLACE "\n" ";" _cfg_lines ${_cfg_file_content_escaped})
-
-    foreach(_line ${_cfg_lines})
-      string(APPEND _cfg_defaults_str "\n  \"${_line}\",")
-    endforeach(_line)
-  endforeach(_cfg_file_path)
-
-  string(APPEND _cfg_defaults_str "};")
-  set(_old_cfg_defaults_str "")
-
-  if(EXISTS ${_ARGS_PROTO_DEFAULTS_HEADER})
-    file(READ ${_ARGS_PROTO_DEFAULTS_HEADER} _old_cfg_defaults_str)
-  endif()
-
-  if(NOT _cfg_defaults_str STREQUAL _old_cfg_defaults_str)
-    file(WRITE "${_ARGS_PROTO_DEFAULTS_HEADER}" "${_cfg_defaults_str}")
-  endif()
-endfunction()
@@ -1,49 +0,0 @@
-include_guard(DIRECTORY)
-
-function(protobuf_generate_fs)
-  set(_options)
-  set(_one_value_args TARGET_NAME OUT_DIR PROTOC)
-  set(_multi_value_args PROTO_INCLUDE_DIRS PROTO_SOURCES)
-  cmake_parse_arguments(_ARGS "${_options}" "${_one_value_args}" "${_multi_value_args}" ${ARGN})
-
-  if (NOT DEFINED _ARGS_TARGET_NAME)
-    set(_ARGS_TARGET_NAME "${PROJECT_NAME}_generate_fs_protos")
-  endif()
-  if (NOT DEFINED _ARGS_PROTOC)
-    set(_ARGS_PROTOC "${EXTERNALS_BIN}/protoc")
-  endif()
-
-  set(_proto_include_cli_args)
-  foreach(_dir ${_ARGS_PROTO_INCLUDE_DIRS})
-    list(APPEND _proto_include_cli_args -I ${_dir})
-  endforeach()
-
-  set(_outputs)
-  foreach(_proto_source ${_ARGS_PROTO_SOURCES})
-    # get_filename_component(_abs_proto_source ${_proto_source} ABSOLUTE)
-    get_filename_component(_proto_source_we ${_proto_source} NAME_WE)
-    # get_filename_component(_proto_source_dir ${_proto_source} DIRECTORY)
-    list(APPEND _outputs ${_ARGS_OUT_DIR}/${_proto_source_we}.fable.fs)
-  endforeach()
-
-  set(_venv_exec ${CMAKE_SOURCE_DIR}/tools/venv_exec.sh)
-  set(_plugin_py ${CMAKE_SOURCE_DIR}/src/lib_spark/spark/api/service_compiler/plugin.py)
-  set(_template_fs ${CMAKE_SOURCE_DIR}/src/lib_spark/spark/api/service_compiler/templates/fs/fable.fs)
-  add_custom_command(
-    OUTPUT ${_outputs}
-    WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}
-    COMMAND ${_venv_exec}
-      ${_ARGS_PROTOC}
-      --plugin=protoc-gen-service=${_plugin_py}
-      --service_out=lang=fs,strip_out_path=1:${_ARGS_OUT_DIR}
-      ${_proto_include_cli_args}
-      ${_ARGS_PROTO_SOURCES}
-    DEPENDS
-      ${_venv_exec}
-      ${_plugin_py}
-      ${_template_fs}
-      ${CMAKE_CURRENT_SOURCE_DIR}/package-lock.json
-      ${_ARGS_PROTO_SOURCES}
-  )
-  add_custom_target(${_ARGS_TARGET_NAME} ALL DEPENDS "${_outputs}")
-endfunction()
@@ -1,33 +0,0 @@
-include_guard(DIRECTORY)
-
-function(protobuf_generate_js)
-  set(_options)
-  set(_one_value_args TARGET_NAME OUTPUT)
-  set(_multi_value_args PROTO_INCLUDE_DIRS PROTO_SOURCES)
-  cmake_parse_arguments(_ARGS "${_options}" "${_one_value_args}" "${_multi_value_args}" ${ARGN})
-
-  if (NOT DEFINED _ARGS_TARGET_NAME)
-    set(_ARGS_TARGET_NAME "${PROJECT_NAME}_generate_js_protos")
-  endif()
-
-  set(_proto_include_cli_args)
-  foreach(_dir ${_ARGS_PROTO_INCLUDE_DIRS})
-    list(APPEND _proto_include_cli_args -p ${_dir})
-  endforeach()
-
-  add_custom_command(
-    OUTPUT ${_ARGS_OUTPUT}
-    WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}
-    COMMAND npx
-      --no-install pbjs
-      -t json-module
-      -w commonjs
-      -o ${_ARGS_OUTPUT}
-      ${_proto_include_cli_args}
-      ${_ARGS_PROTO_SOURCES}
-    DEPENDS
-      ${CMAKE_CURRENT_SOURCE_DIR}/package-lock.json
-      ${_ARGS_PROTO_SOURCES}
-  )
-  add_custom_target(${_ARGS_TARGET_NAME} ALL DEPENDS ${_ARGS_OUTPUT})
-endfunction()
@@ -1,10 +0,0 @@
-include_guard(GLOBAL)
-
-macro(set_first_defined RESULT)
-  foreach(_arg ${ARGN})
-    if (DEFINED ${_arg})
-      set(${RESULT} ${${_arg}})
-      break()
-    endif()
-  endforeach()
-endmacro()
@@ -1,49 +0,0 @@
-include_guard(DIRECTORY)
-include(${CMAKE_SOURCE_DIR}/cmake/TernaryOperator.cmake)
-
-function(sym_link_dir)
-  set(_options)
-  set(_one_value_args TARGET_NAME SOURCE DEST COMMENT)
-  set(_multi_value_args)
-  cmake_parse_arguments(_ARGS "${_options}" "${_one_value_args}" "${_multi_value_args}" ${ARGN})
-  add_custom_command(
-    OUTPUT ${_ARGS_DEST}
-    COMMAND ln ARGS -sf ${_ARGS_SOURCE} ${_ARGS_DEST}
-  )
-  set_ternary(_comment _ARGS_COMMENT "${_ARGS_COMMENT}" "linking dir ${_ARGS_SOURCE} -> ${_ARGS_DEST}")
-  add_custom_target(
-    ${_ARGS_TARGET_NAME} ALL
-    DEPENDS ${_ARGS_DEST}
-    COMMENT "${_comment}"
-  )
-endfunction()
-
-function(sym_link_file)
-  set(_options)
-  set(_one_value_args SOURCE DEST INSTALL_DEST)
-  set(_multi_value_args)
-  cmake_parse_arguments(_ARGS "${_options}" "${_one_value_args}" "${_multi_value_args}" ${ARGN})
-  file(
-    CREATE_LINK
-      ${_ARGS_SOURCE}
-      ${_ARGS_DEST}
-    SYMBOLIC
-  )
-  if (DEFINED _ARGS_INSTALL_DEST)
-    install(
-      FILES
-      ${_ARGS_SOURCE}
-      DESTINATION ${_ARGS_INSTALL_DEST}
-    )
-  endif()
-endfunction()
-
-function(sym_link_local_lib LIB_NAME)
-  set(_lib_file_name "${CMAKE_SHARED_LIBRARY_PREFIX}${LIB_NAME}.so")
-  file(
-    CREATE_LINK
-      "${CMAKE_BINARY_DIR}/src/lib_${LIB_NAME}/${_lib_file_name}"
-      "${CMAKE_CURRENT_BINARY_DIR}/${_lib_file_name}"
-    SYMBOLIC
-  )
-endfunction()
@@ -1,9 +0,0 @@
-include_guard(GLOBAL)
-
-macro(set_ternary RESULT CONDITION TRUE_VALUE FALSE_VALUE)
-  if(${CONDITION})
-    set(${RESULT} ${TRUE_VALUE})
-  else()
-    set(${RESULT} ${FALSE_VALUE})
-  endif()
-endmacro()
@@ -1,11 +0,0 @@
-#pragma once
-
-#include <array>
-
-namespace bts::@PROJECT_NAME@::protobufs {
-
-inline std::array<const char *, @PROTO_SOURCES_LENGTH@> config_proto_files = {
-@PROTO_SOURCES_CXX_ARRAY_ITEMS@
-};
-
-} // namespace bts::@PROJECT_NAME@::protobufs
@@ -5,7 +5,7 @@ cmake_minimum_required(VERSION 3.5)
 cmake_policy(SET CMP0076 NEW)

 # add protos to the library to facilitate source_group
-target_sources(spark PUBLIC ${Config_Protos} ${NonConfig_Protos} ${Service_Protos})
+target_sources(libspark PUBLIC ${Config_Protos} ${NonConfig_Protos} ${Service_Protos})

 function(assign_source_group)
  foreach(_source IN ITEMS ${ARGN})
@@ -1,51 +0,0 @@
-services:
-  sparkd:
-    build:
-      dockerfile: Dockerfile
-      context: ./sparkd
-    command: [ "./sparkd", "run" ]
-    ports: [ "8000:8000", "8001:8001", "8002:8002" ]
-
-  pubsub:
-    build:
-      dockerfile: pubsub.Dockerfile
-      context: ./DataStream
-    command: [ "./PubSubServer" ]
-    environment:
-      - DOTNET_ENVIRONMENT=Production
-    ports:
-      - 22019:22019
-
-  md-publisher:
-    build:
-      dockerfile: md-publisher.Dockerfile
-      context: ./DataStream
-    depends_on:
-      - pubsub
-      - sparkd
-    command:
-      [
-        "dockerize",
-        "-wait",
-        "tcp://pubsub:22019",
-        "-wait",
-        "tcp://sparkd:8000",
-        "-timeout",
-        "10s",
-        "./DataStreamPublisher"
-      ]
-    environment:
-      - DOTNET_ENVIRONMENT=MarketData
-
-  webapp:
-    ports:
-      - 8080:8080
-      - 8443:8443
-    build:
-      dockerfile: webapp.Dockerfile
-      context: ./DataStream
-    command: [ "./Bts.DataStreamWebApp.Server" ]
-    environment:
-      - ASPNETCORE_HTTP_PORT=8080
-      - ASPNETCORE_HTTPS_PORT=8443
-      - ASPNETCORE_ENVIRONMENT=Staging
@@ -1,79 +0,0 @@
-services:
-  sparkd:
-    build:
-      dockerfile: src/app_sparkd/Dockerfile
-      args:
-        - IMAGE=${SPARKD__IMAGE}
-        - SOURCE_DIR=${SPARKD__SOURCE_DIR}
-        - CONF_FILE=${SPARKD__CONF_FILE}
-        - MD_RECORDING_DIR=${SPARKD__MD_RECORDING_DIR}
-    command: [ "./sparkd", "run" ]
-    ports:
-      - "8000:8000"
-      - "8001:8001"
-      - "8002:8002"
-
-  pubsub:
-    environment:
-      - DOTNET_ENVIRONMENT=${PUBSUB__DOTNET_ENVIRONMENT}
-    build:
-      dockerfile: src/DataStreamSuite/PubSubServer/Dockerfile
-      args:
-        - CS_SOURCE_DIR=${PUBSUB__CS_SOURCE_DIR}
-        - CPP_SOURCE_DIR=${PUBSUB__CPP_SOURCE_DIR}
-        - CONF_FILE=${PUBSUB__CONF_FILE}
-    ports:
-      - "22019:22019"
-    command: [ "./PubSubServer" ]
-
-  md-publisher:
-    build:
-      dockerfile: src/DataStreamSuite/DataStreamPublisher/Dockerfile
-      args:
-        - SOURCE_DIR=${MD_PUBLISHER__SOURCE_DIR}
-        - CONF_FILE=${MD_PUBLISHER__CONF_FILE}
-    depends_on:
-      - pubsub
-      - sparkd
-    command: [ "dockerize", "-wait", "tcp://pubsub:22019", "-wait", "tcp://sparkd:8000", "-timeout", "10s", "./DataStreamPublisher" ]
-    environment:
-      - DOTNET_ENVIRONMENT=${MD_PUBLISHER__DOTNET_ENVIRONMENT}
-
-  webapp:
-    ports:
-      - "8080:8080"
-      - "8443:8443"
-    build:
-      dockerfile: src/DataStreamSuite/Bts.DataStreamWebApp.Server/Dockerfile
-      args:
-        - SOURCE_DIR=${WEBAPP__SOURCE_DIR}
-        - WEB_ROOT=${WEBAPP__WEB_ROOT}
-        - CONF_FILE=${WEBAPP__CONF_FILE}
-    command: [ "./Bts.DataStreamWebApp.Server" ]
-    environment:
-      - ASPNETCORE_ENVIRONMENT=${WEBAPP__ASPNETCORE_ENVIRONMENT}
-
-  api-server:
-    depends_on:
-      - sparkd
-    ports:
-      - 8008:8008
-      - 9443:9443
-    build:
-      dockerfile: src/client/spark_api_server/Dockerfile
-    command: [ "dockerize", "-wait", "tcp://sparkd:8000", "-timeout", "10s", "./spark_api_server", "run" ]
-    environment:
-      - ASPNETCORE_ENVIRONMENT=${API_SERVER__ASPNETCORE_ENVIRONMENT}
-
-  sparkd-ping-test:
-    depends_on:
-      - sparkd
-    profiles:
-      - ping-test
-    build:
-      context: .
-      dockerfile_inline: |
-        FROM bluetradingsystems/linux_dotnet7
-        USER "root"
-        RUN apt-get update && apt-get -y install iputils-ping
-    command: [ "ping", "sparkd" ]
@@ -53,7 +53,7 @@ The status callback is called when the market for this security opens, closes, o
 // Subscribe to security trading status
 void IAlgoServices::subscribe_status(SecurityKey);
 // You implement
-virtual void on_status(SecurityKey, domain::protobufs::TradingStatus) override;
+virtual void on_status(SecurityKey, co2::TradingStatus) override;
 ```

 # Child Orders
@@ -1,65 +1,65 @@
-FROM ubuntu:latest
+FROM ubuntu:22.04

 ARG DEBIAN_FRONTEND=noninteractive
-ENV TZ=America/New_York

-# light deps (should be quick to download/install)
+# Foundational Deps
 RUN apt-get update && apt-get install -y \
+    build-essential \
    ca-certificates \
    curl \
    gnupg \
    liblttng-ust-dev \
    libpcap0.8-dev \
+    libssh2-1-dev \
    locales \
-    ninja-build \
+    openssh-server \
    software-properties-common \
-    sqlite3 \
+    sudo \
    tzdata

-# clang
-RUN apt-get update && apt-get install -y \
-    clang-13 \
-    clang-format-13 \
-    clangd-13 \
-    clang-15 \
-    clang-format-15 \
-    clangd-15
+# Clang 17
+RUN curl -fsSL https://apt.llvm.org/llvm-snapshot.gpg.key 2> /dev/null \
+    | gpg --dearmor - \
+    | tee /usr/share/keyrings/llvm-snapshot.gpg > /dev/null \
+    && echo "deb [signed-by=/usr/share/keyrings/llvm-snapshot.gpg] http://apt.llvm.org/jammy/ llvm-toolchain-jammy-17 main" \
+    | tee -a "/etc/apt/sources.list.d/llvm.list" \
+    && apt-get update \
+    && apt-get install -y \
+    clang-17 \
+    clang-format-17 \
+    clangd-17

-# g++
+# G++
 RUN apt-get update && apt-get install -y \
-    g++-11 \
-    g++-12
+    g++

-# python
+# Ninja
 RUN apt-get update && apt-get install -y \
-    python3-pip \
-    python3-venv \
-    && python3 -m pip install --upgrade pip
+    ninja-build

-# dotnet
-RUN apt-get update && apt-get install -y \
-    dotnet7
-
-# ssh
-RUN apt-get update && apt-get install -y \
-    libssh2-1-dev \
-    openssh-server
-
-# cmake
-RUN \
-    curl -fsSL https://apt.kitware.com/keys/kitware-archive-latest.asc 2> /dev/null \
+# CMake
+RUN curl -fsSL https://apt.kitware.com/keys/kitware-archive-latest.asc 2> /dev/null \
    | gpg --dearmor - \
    | tee /usr/share/keyrings/kitware-archive-keyring.gpg > /dev/null \
    && echo "deb [signed-by=/usr/share/keyrings/kitware-archive-keyring.gpg] https://apt.kitware.com/ubuntu/ $(lsb_release -cs) main" \
    | tee /etc/apt/sources.list.d/kitware.list > /dev/null \
    && apt-get update \
-    && rm /usr/share/keyrings/kitware-archive-keyring.gpg \
-    && apt-get install -y kitware-archive-keyring \
-    && echo "deb [signed-by=/usr/share/keyrings/kitware-archive-keyring.gpg] https://apt.kitware.com/ubuntu/ $(lsb_release -cs)-rc main" \
-    | tee -a /etc/apt/sources.list.d/kitware.list >/dev/null \
    && apt-get install -y cmake

-# node
+# Dotnet
+RUN curl -fsSLO "https://packages.microsoft.com/config/ubuntu/$(lsb_release -rs)/packages-microsoft-prod.deb" \
+    && dpkg -i packages-microsoft-prod.deb \
+    && rm packages-microsoft-prod.deb \
+    && apt-get update \
+    && apt-get install -y dotnet-sdk-8.0
+
+# Python
+RUN apt-get update && apt-get install -y \
+    python3-pip \
+    python3-venv \
+    && python3 -m pip install --upgrade pip
+
+# Node
 ARG node_major=18
 RUN mkdir -p /etc/apt/keyrings \
    && curl -fsSL https://deb.nodesource.com/gpgkey/nodesource-repo.gpg.key \
@@ -69,54 +69,35 @@ RUN mkdir -p /etc/apt/keyrings \
    && apt-get update \
    && apt-get install -y nodejs

-# ccache is done here so that ccache sees clang++ installed, not clang-10
+# Sqlite3
 RUN apt-get update && apt-get install -y \
-    ccache 
+    sqlite3

-# emumba (light deps)
-RUN apt-get update && apt-get install -y \
-    build-essential \
-    doxygen \
-    python3-sphinx \
-    graphviz \
-    libgtest-dev \
-    libssl-dev \
-    rapidjson-dev \
-    zlib1g-dev \
-    pkg-config
-
-# emumba (boost)
-RUN apt-get update && apt-get install -y \
-    libboost-filesystem-dev \
-    libboost-program-options-dev \
-    libboost-test-dev \
-    libboost-date-time-dev
-
-# dev utils
+# Dev Utils (install last so they can be added/removed with minimal rebuild)
 RUN apt-get update && apt-get install -y \ 
+    ccache \
    fish \
+    gdb \
    git-all \
-    sudo \
-    vim \
-    wget
+    vim

-# set up default executables (update-alternatives)
-RUN \
-    update-alternatives --install /usr/bin/clang++ clang++ /usr/bin/clang++-13 100; \
-    update-alternatives --install /usr/bin/clang clang /usr/bin/clang-13 100; \
-    update-alternatives --install /usr/bin/clang-format clang-format /usr/bin/clang-format-13 100; \
-    update-alternatives --install /usr/bin/clangd clangd /usr/bin/clangd-13 100; \
-    update-alternatives --install /usr/bin/g++ g++ /usr/bin/g++-11 100; \
-    update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-11 100
+# Set up default executables (update-alternatives)
+RUN update-alternatives --install /usr/bin/clang++ clang++ /usr/bin/clang++-17 100; \
+    update-alternatives --install /usr/bin/clang clang /usr/bin/clang-17 100; \
+    update-alternatives --install /usr/bin/clang-format clang-format /usr/bin/clang-format-17 100; \
+    update-alternatives --install /usr/bin/clangd clangd /usr/bin/clangd-17 100; \
+    update-alternatives --install /usr/bin/g++ g++ /usr/bin/g++-12 100; \
+    update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-12 100

 # Add compiler symlinks for ccache 
-RUN \
-    ln -s /usr/bin/ccache /usr/local/bin/clang; \
+RUN ln -s /usr/bin/ccache /usr/local/bin/clang; \
    ln -s /usr/bin/ccache /usr/local/bin/clang++; \
    ln -s /usr/bin/ccache /usr/local/bin/clang-13; \
    ln -s /usr/bin/ccache /usr/local/bin/clang++-13; \
    ln -s /usr/bin/ccache /usr/local/bin/clang-15; \
    ln -s /usr/bin/ccache /usr/local/bin/clang++-15; \
+    ln -s /usr/bin/ccache /usr/local/bin/clang-17; \
+    ln -s /usr/bin/ccache /usr/local/bin/clang++-17; \
    ln -s /usr/bin/ccache /usr/local/bin/gcc; \
    ln -s /usr/bin/ccache /usr/local/bin/g++; \
    ln -s /usr/bin/ccache /usr/local/bin/gcc-11; \
@@ -130,8 +111,7 @@ ARG docker_user="bts"
 ARG docker_uid="1000"
 ARG docker_pass="bts"

-RUN \
-    groupadd -g "$docker_uid" "$docker_user" \
+RUN groupadd -g "$docker_uid" "$docker_user" \
    && useradd \
    -m \
    -u "$docker_uid" \
@@ -1,39 +0,0 @@
-FROM ubuntu:22.04
-
-ARG DEBIAN_FRONTEND=noninteractive
-
-RUN apt-get update \
-  && apt-get install -y pkg-config \
-  && apt-get install -y locales \
-  && apt-get install -y ca-certificates \
-  && apt-get install -y libssh2-1-dev \
-  && apt-get install -y software-properties-common \
-  && apt-get install -y sudo
-
-# GCC
-ARG gcc_major=13
-RUN add-apt-repository ppa:ubuntu-toolchain-r/test \
-  && apt-get update \
-  && apt-get install -y g++-$gcc_major
-
-RUN apt-get update \
-  && apt-get install -y dotnet7
-
-RUN locale-gen en_US.UTF-8
-
-ARG docker_user="bts"
-ARG docker_uid="1000"
-ARG docker_pass="bts"
-
-RUN \
-  groupadd -g "$docker_uid" "$docker_user" \
-  && useradd \
-  -m \
-  -u "$docker_uid" \
-  -g "$docker_uid" \
-  -G sudo \
-  --password "$(openssl passwd -1 "$docker_pass")" \
-  "$docker_user"
-
-USER bts
-WORKDIR /home/bts
@@ -0,0 +1,45 @@
+/* global use, db */
+// MongoDB Playground
+// To disable this template go to Settings | MongoDB | Use Default Template For Playground.
+// Make sure you are connected to enable completions and to be able to run a playground.
+// Use Ctrl+Space inside a snippet or a string literal to trigger completions.
+// The result of the last command run in a playground is shown on the results panel.
+// By default the first 20 documents will be returned with a cursor.
+// Use 'console.log()' to print to the debug output.
+// For more documentation on playgrounds please refer to
+// https://www.mongodb.com/docs/mongodb-vscode/playgrounds/
+
+// Select the database to use.
+use('mongodbVSCodePlaygroundDB');
+
+// Insert a few documents into the sales collection.
+db.getCollection('sales').insertMany([
+    { 'item': 'abc', 'price': 10, 'quantity': 2, 'date': new Date('2014-03-01T08:00:00Z') },
+    { 'item': 'jkl', 'price': 20, 'quantity': 1, 'date': new Date('2014-03-01T09:00:00Z') },
+    { 'item': 'xyz', 'price': 5, 'quantity': 10, 'date': new Date('2014-03-15T09:00:00Z') },
+    { 'item': 'xyz', 'price': 5, 'quantity': 20, 'date': new Date('2014-04-04T11:21:39.736Z') },
+    { 'item': 'abc', 'price': 10, 'quantity': 10, 'date': new Date('2014-04-04T21:23:13.331Z') },
+    { 'item': 'def', 'price': 7.5, 'quantity': 5, 'date': new Date('2015-06-04T05:08:13Z') },
+    { 'item': 'def', 'price': 7.5, 'quantity': 10, 'date': new Date('2015-09-10T08:43:00Z') },
+    { 'item': 'abc', 'price': 10, 'quantity': 5, 'date': new Date('2016-02-06T20:20:13Z') },
+]);
+
+// Run a find command to view items sold on April 4th, 2014.
+const salesOnApril4th = db.getCollection('sales').find({
+    date: { $gte: new Date('2014-04-04'), $lt: new Date('2014-04-05') }
+}).count();
+
+// Print a message to the output window.
+console.log(`${salesOnApril4th} sales occurred in 2014.`);
+
+// Here we run an aggregation and open a cursor to the results.
+// Use '.toArray()' to exhaust the cursor to return the whole result set.
+// You can use '.hasNext()/.next()' to iterate through the cursor page by page.
+db.getCollection('sales').aggregate([
+    // Find all of the sales that occurred in 2014.
+    { $match: { date: { $gte: new Date('2014-01-01'), $lt: new Date('2015-01-01') } } },
+    // Group the total sales for each product.
+    { $group: { _id: '$item', totalSaleAmount: { $sum: { $multiply: ['$price', '$quantity'] } } } }
+]);
+
+
@@ -8,6 +8,7 @@
   public int Integer;
   public double Real;
   public string Identifier;
+   public DateTime DateTime; 
   public ImmutableList<IArithmeticExpression> Arguments;
   public ImmutableList<Assignment> Assignments;
   public IArithmeticExpression Argument;
@@ -1,7 +1,7 @@
 <Project Sdk="Microsoft.NET.Sdk">

  <PropertyGroup>
-    <TargetFramework>net7.0</TargetFramework>
+    <TargetFramework>net8.0</TargetFramework>
    <ImplicitUsings>enable</ImplicitUsings>
    <Nullable>enable</Nullable>
  </PropertyGroup>
@@ -1,7 +1,7 @@
 <Project Sdk="Microsoft.NET.Sdk">

  <PropertyGroup>
-    <TargetFramework>net7.0</TargetFramework>
+    <TargetFramework>net8.0</TargetFramework>
    <ImplicitUsings>enable</ImplicitUsings>
    <Nullable>enable</Nullable>
  </PropertyGroup>
@@ -1,22 +1,41 @@
-using System.Diagnostics;
+using mbts.spark;

-namespace BCG.Utilities.General;
+using System;
+using System.Runtime;

-/// <summary>
-/// Summary description for Crono.
-/// </summary>
-public class Crono {
-   readonly Stopwatch myTimer = new();
+namespace BCG.Utilities.General
+{
+	/// <summary>
+	/// Summary description for Crono.
+	/// </summary>
+	public class Crono
+	{
+      public Crono()
+      {
+         myTimer = new HighPerformanceTimer();
+      }

-   public void start() =>
-      myTimer.Start();
+      public void start() 
+      {myTimer.start();}

-   public void stop() =>
-      myTimer.Stop();
+      public void stop()
+      {myTimer.stop();}

-   public TimeSpan time =>
-      myTimer.Elapsed;
+      public TimeSpan time
+      {
+         get 
+         {
+            return new TimeSpan((long) (myTimer.totalMilliseconds * TimeSpan.TicksPerMillisecond));
+         }
+      }
+
+      public double miliseconds
+      {
+         get
+         {return myTimer.totalMilliseconds;}
+      }
+
+      private HighPerformanceTimer myTimer;
+	}

-   public double miliseconds =>
-      myTimer.Elapsed.TotalMilliseconds;
 }
@@ -1,192 +1,192 @@
-using System;
-using System.Collections;
-using System.Collections.Generic;
-using System.Collections.Concurrent;
-using System.Reflection;
-using Microsoft.Win32;
-
-namespace BCG
-{
-   public enum RegistryRoot { HKCU, HKLM }
-
-   [Serializable]
-   public class BlueRegistry
-   {
-      public const string BlueSubKey = @"Software\Blue Capital Group\";
-      public static readonly string ApplicationPrefix;
-      protected string myFullPath;
-
-      public BlueRegistry(string subPath)
-      {
-         myFullPath = string.Concat(BlueRegistry.ApplicationPrefix, @"\", subPath);
-      }
-
-      public BlueRegistry(BlueRegistry parent, string subPath)
-      {
-         myFullPath = string.Concat(parent.myFullPath, @"\", subPath);
-      }
-
-      static BlueRegistry()
-      {
-         string app = "Applications";
-
-         Assembly ass = Assembly.GetEntryAssembly();
-         if (ass != null)
-         {
-            app = ass.GetName().Name;
-
-            object[] attrs = ass.GetCustomAttributes(
-               Type.GetType("System.Reflection.AssemblyTitleAttribute"), true);
-
-            if (attrs.Length > 0 && ((AssemblyTitleAttribute)attrs[0]).Title != string.Empty)
-            {
-               app = ((AssemblyTitleAttribute)attrs[0]).Title;
-            }
-         }
-
-         ApplicationPrefix = string.Concat(BlueRegistry.BlueSubKey, app);
-      }
-
-      public object getValue(string name)
-      {
-         return getValue(RegistryRoot.HKCU, name);
-      }
-
-      public object getValue(string name, object defaultValue)
-      {
-         return getValue(RegistryRoot.HKCU, name, defaultValue);
-      }
-
-      public void setValue(string name, object value)
-      {
-         setValue(RegistryRoot.HKCU, name, value);
-      }
-
-      public void deleteValue(string name)
-      {
-         deleteValue(RegistryRoot.HKCU, name);
-      }
-
-      private RegistryKey myGetRegistryKey(RegistryRoot root)
-      {
-         if (root == RegistryRoot.HKCU)
-            return Registry.CurrentUser.CreateSubKey(myFullPath);
-         else
-            return Registry.LocalMachine.CreateSubKey(myFullPath);
-      }
-
-      #region Generic Methods
-      public virtual void setValue(RegistryRoot root, string name, object value)
-      {
-         using (RegistryKey rk = myGetRegistryKey(root))
-         {
-            rk.SetValue(name, value);
-         }
-      }
-
-      internal virtual object getValue(RegistryRoot root, string name)
-      {
-         using (RegistryKey rk = myGetRegistryKey(root))
-         {
-            return rk.GetValue(name);
-         }
-      }
-
-      internal virtual void deleteValue(RegistryRoot root, string name)
-      {
-         using (RegistryKey rk = myGetRegistryKey(root))
-         {
-            rk.DeleteValue(name);
-         }
-      }
-
-      public virtual object getValue(RegistryRoot root, string name, object defaultValue)
-      {
-         using (RegistryKey rk = myGetRegistryKey(root))
-         {
-            return rk.GetValue(name, defaultValue);
-         }
-      }
-      #endregion
-
-      #region Double
-      internal virtual void setDouble(RegistryRoot root, string name, double value)
-      {
-         using (RegistryKey rk = myGetRegistryKey(root))
-         {
-            DoubleRegistryHandler.setValue(rk, name, value);
-         }
-      }
-
-      internal virtual double getDouble(RegistryRoot root, string name)
-      {
-         using (RegistryKey rk = myGetRegistryKey(root))
-         {
-            return DoubleRegistryHandler.getValue(rk, name);
-         }
-      }
-      #endregion
-
-      #region Enum
-      internal virtual void setEnum(RegistryRoot root, string name, Enum value)
-      {
-         using (RegistryKey rk = myGetRegistryKey(root))
-         {
-            EnumRegistryHandler.setValue(rk, name, value);
-         }
-      }
-
-      internal virtual Enum getEnum(RegistryRoot root, string name, Type enumType)
-      {
-         using (RegistryKey rk = myGetRegistryKey(root))
-         {
-            return EnumRegistryHandler.getValue(rk, name, enumType);
-         }
-      }
-      #endregion
-
-      #region Array
-      internal virtual void setArray(RegistryRoot root, string name, Array array)
-      {
-         using (RegistryKey rk = myGetRegistryKey(root))
-         {
-            ArrayRegistryHandler.setValue(rk, name, array);
-         }
-      }
-
-      internal virtual Array getArray(RegistryRoot root, string name, Type elementType)
-      {
-         using (RegistryKey rk = myGetRegistryKey(root))
-         {
-            return ArrayRegistryHandler.getValue(rk, name, elementType);
-         }
-      }
-      #endregion
-
-      #region SortedList
-      internal virtual void setSortedList(RegistryRoot root, string name, SortedList slist)
-      {
-         using (RegistryKey rk = myGetRegistryKey(root))
-         {
-            SortedListRegistryHandler.setValue(rk, name, slist);
-         }
-      }
-
-      internal virtual SortedList getSortedList(RegistryRoot root, string name)
-      {
-         using (RegistryKey rk = myGetRegistryKey(root))
-         {
-            return SortedListRegistryHandler.getValue(rk, name);
-         }
-      }
-
-      internal virtual void deleteSortedList(RegistryRoot root, string name)
-      {
-         using (RegistryKey rk = myGetRegistryKey(root))
-         {
-            SortedListRegistryHandler.deleteValue(rk, name);
-         }
-      }
-      #endregion
-   }
-}
+using System;
+using System.Collections;
+using System.Collections.Generic;
+using System.Collections.Concurrent;
+using System.Reflection;
+using Microsoft.Win32;
+
+namespace BCG
+{
+   public enum RegistryRoot { HKCU, HKLM }
+
+   [Serializable]
+   public class BlueRegistry
+   {
+      public const string BlueSubKey = @"Software\Blue Capital Group\";
+      public static readonly string ApplicationPrefix;
+      protected string myFullPath;
+
+      public BlueRegistry(string subPath)
+      {
+         myFullPath = string.Concat(BlueRegistry.ApplicationPrefix, @"\", subPath);
+      }
+
+      public BlueRegistry(BlueRegistry parent, string subPath)
+      {
+         myFullPath = string.Concat(parent.myFullPath, @"\", subPath);
+      }
+
+      static BlueRegistry()
+      {
+         string app = "Applications";
+
+         Assembly ass = Assembly.GetEntryAssembly();
+         if (ass != null)
+         {
+            app = ass.GetName().Name;
+
+            object[] attrs = ass.GetCustomAttributes(
+               Type.GetType("System.Reflection.AssemblyTitleAttribute"), true);
+
+            if (attrs.Length > 0 && ((AssemblyTitleAttribute)attrs[0]).Title != string.Empty)
+            {
+               app = ((AssemblyTitleAttribute)attrs[0]).Title;
+            }
+         }
+
+         ApplicationPrefix = string.Concat(BlueRegistry.BlueSubKey, app);
+      }
+
+      public object getValue(string name)
+      {
+         return getValue(RegistryRoot.HKCU, name);
+      }
+
+      public object getValue(string name, object defaultValue)
+      {
+         return getValue(RegistryRoot.HKCU, name, defaultValue);
+      }
+
+      public void setValue(string name, object value)
+      {
+         setValue(RegistryRoot.HKCU, name, value);
+      }
+
+      public void deleteValue(string name)
+      {
+         deleteValue(RegistryRoot.HKCU, name);
+      }
+
+      private RegistryKey myGetRegistryKey(RegistryRoot root)
+      {
+         if (root == RegistryRoot.HKCU)
+            return Registry.CurrentUser.CreateSubKey(myFullPath);
+         else
+            return Registry.LocalMachine.CreateSubKey(myFullPath);
+      }
+
+      #region Generic Methods
+      internal virtual void setValue(RegistryRoot root, string name, object value)
+      {
+         using (RegistryKey rk = myGetRegistryKey(root))
+         {
+            rk.SetValue(name, value);
+         }
+      }
+
+      internal virtual object getValue(RegistryRoot root, string name)
+      {
+         using (RegistryKey rk = myGetRegistryKey(root))
+         {
+            return rk.GetValue(name);
+         }
+      }
+
+      internal virtual void deleteValue(RegistryRoot root, string name)
+      {
+         using (RegistryKey rk = myGetRegistryKey(root))
+         {
+            rk.DeleteValue(name);
+         }
+      }
+
+      internal virtual object getValue(RegistryRoot root, string name, object defaultValue)
+      {
+         using (RegistryKey rk = myGetRegistryKey(root))
+         {
+            return rk.GetValue(name, defaultValue);
+         }
+      }
+      #endregion
+
+      #region Double
+      internal virtual void setDouble(RegistryRoot root, string name, double value)
+      {
+         using (RegistryKey rk = myGetRegistryKey(root))
+         {
+            DoubleRegistryHandler.setValue(rk, name, value);
+         }
+      }
+
+      internal virtual double getDouble(RegistryRoot root, string name)
+      {
+         using (RegistryKey rk = myGetRegistryKey(root))
+         {
+            return DoubleRegistryHandler.getValue(rk, name);
+         }
+      }
+      #endregion
+
+      #region Enum
+      internal virtual void setEnum(RegistryRoot root, string name, Enum value)
+      {
+         using (RegistryKey rk = myGetRegistryKey(root))
+         {
+            EnumRegistryHandler.setValue(rk, name, value);
+         }
+      }
+
+      internal virtual Enum getEnum(RegistryRoot root, string name, Type enumType)
+      {
+         using (RegistryKey rk = myGetRegistryKey(root))
+         {
+            return EnumRegistryHandler.getValue(rk, name, enumType);
+         }
+      }
+      #endregion
+
+      #region Array
+      internal virtual void setArray(RegistryRoot root, string name, Array array)
+      {
+         using (RegistryKey rk = myGetRegistryKey(root))
+         {
+            ArrayRegistryHandler.setValue(rk, name, array);
+         }
+      }
+
+      internal virtual Array getArray(RegistryRoot root, string name, Type elementType)
+      {
+         using (RegistryKey rk = myGetRegistryKey(root))
+         {
+            return ArrayRegistryHandler.getValue(rk, name, elementType);
+         }
+      }
+      #endregion
+
+      #region SortedList
+      internal virtual void setSortedList(RegistryRoot root, string name, SortedList slist)
+      {
+         using (RegistryKey rk = myGetRegistryKey(root))
+         {
+            SortedListRegistryHandler.setValue(rk, name, slist);
+         }
+      }
+
+      internal virtual SortedList getSortedList(RegistryRoot root, string name)
+      {
+         using (RegistryKey rk = myGetRegistryKey(root))
+         {
+            return SortedListRegistryHandler.getValue(rk, name);
+         }
+      }
+
+      internal virtual void deleteSortedList(RegistryRoot root, string name)
+      {
+         using (RegistryKey rk = myGetRegistryKey(root))
+         {
+            SortedListRegistryHandler.deleteValue(rk, name);
+         }
+      }
+      #endregion
+   }
+}
@@ -1,9 +1,10 @@
 <Project Sdk="Microsoft.NET.Sdk">

  <PropertyGroup>
-    <TargetFramework>net7.0</TargetFramework>
+    <TargetFramework>net8.0-windows</TargetFramework>
    <ImplicitUsings>enable</ImplicitUsings>
    <Nullable>enable</Nullable>
+    <UseWindowsForms>true</UseWindowsForms>
  </PropertyGroup>

 </Project>
@@ -1,6 +1,4 @@
 using System;
-using System.Runtime.Versioning;
-
 using Microsoft.Win32;

 namespace BCG
@@ -12,7 +10,6 @@ namespace BCG
         rk.SetValue (name, value.ToString ());
      }

-      [SupportedOSPlatform("windows")]
      public static double getValue (RegistryKey rk, string name)
      {
         object v = rk.GetValue (name);
@@ -1,166 +1,166 @@
-using System;
-using System.Reflection;
-using System.Windows.Forms;
-using System.Drawing;
-using System.Diagnostics;
-using Microsoft.Win32;
-
-namespace BCG
-{
-   public class PersistWindowState
-   {
-      // event info that allows form to persist extra window state data
-      public delegate void WindowStateHandler (object sender, BlueRegistry bluereg);
-      public event WindowStateHandler LoadState;
-      public event WindowStateHandler SaveState;
-
-      private Form myParent;
-      private BlueRegistry myRegistry;
-
-      private int myNormalLeft;
-      private int myNormalTop;
-      private int myNormalWidth;
-      private int myNormalHeight;
-      private bool myResizable = true;
-      private FormWindowState myWindowState;
-      private bool myAllowSaveMinimized = false;
-      private bool mySavePosition = true;
-      private bool mySaveState = true;
-
-      public PersistWindowState (bool savePosition)
-      {
-         mySavePosition = savePosition;
-      }
-
-      public PersistWindowState ()
-      {
-      }
-
-      public bool saveState
-      {
-         [DebuggerStepThrough]
-         set { mySaveState = value; }
-      }
-
-      public Form parent
-      {
-         [DebuggerStepThrough]
-         get { return myParent; }
-         set
-         {
-            myParent = value;
-            myRegistry = new BlueRegistry (myParent.Text);
-
-            // subscribe to parent form's events
-            myParent.Closing += new System.ComponentModel.CancelEventHandler(myOnClosing);
-            myParent.Resize += new System.EventHandler(myOnResize);
-            myParent.Move += new System.EventHandler(myOnMove);
-            myParent.Load += new System.EventHandler(myOnLoad);
-
-            // get initial width and height in case form is never resized
-            myNormalWidth = myParent.Width;
-            myNormalHeight = myParent.Height;
-         }
-      }
-
-      public BlueRegistry blueRegistry
-      {
-         [DebuggerStepThrough]
-         get { return myRegistry; }
-         [DebuggerStepThrough]
-         set { myRegistry = value; }
-      }
-
-      public bool resizable
-      {
-         [DebuggerStepThrough]
-         set { myResizable = value; }
-      }
-
-      public bool allowSaveMinimized
-      {
-         [DebuggerStepThrough]
-         set { myAllowSaveMinimized = value; }
-      }
-
-      private void myOnResize(object sender, System.EventArgs e)
-      {
-         // save width and height
-         if (myParent.WindowState == FormWindowState.Normal && myResizable)
-         {
-            myNormalWidth = myParent.Width;
-            myNormalHeight = myParent.Height;
-         }
-      }
-
-      private void myOnMove(object sender, System.EventArgs e)
-      {
-         // save position
-         if(myParent.WindowState == FormWindowState.Normal)
-         {
-            myNormalLeft = myParent.Left;
-            myNormalTop = myParent.Top;
-         }
-         // save state
-         myWindowState = myParent.WindowState;
-      }
-
-      private void myOnClosing(object sender, System.ComponentModel.CancelEventArgs e)
-      {
-         // save position, size and state
-         if (mySavePosition)
-         {
-            myRegistry.setValue (RegistryRoot.HKCU, "Left", myNormalLeft);
-            myRegistry.setValue (RegistryRoot.HKCU, "Top", myNormalTop);
-         }
-
-         if (myResizable)
-         {
-            myRegistry.setValue (RegistryRoot.HKCU, "Width", myNormalWidth);
-            myRegistry.setValue (RegistryRoot.HKCU, "Height", myNormalHeight);
-         }
-
-         // check if we are allowed to save the state as minimized (not normally)
-         if (!myAllowSaveMinimized)
-         {
-            if(myWindowState == FormWindowState.Minimized)
-               myWindowState = FormWindowState.Normal;
-         }
-
-         if (mySaveState)
-            myRegistry.setValue (RegistryRoot.HKCU, "WindowState", (int) myWindowState);
-        
-         // fire SaveState event
-         if (SaveState != null)
-            SaveState (this, myRegistry);
-      }
-
-      private void myOnLoad (object sender, System.EventArgs e)
-      {
-         if (mySavePosition)
-         {
-            int left = (int) myRegistry.getValue (RegistryRoot.HKCU, "Left", myParent.Left);
-            int top = (int) myRegistry.getValue (RegistryRoot.HKCU, "Top", myParent.Top);
-            myParent.Location = new Point (left, top);
-         }
-
-         if (myResizable)
-         {
-            int width = (int) myRegistry.getValue (RegistryRoot.HKCU, "Width", myParent.Width);
-            int height = (int) myRegistry.getValue (RegistryRoot.HKCU, "Height", myParent.Height);
-            myParent.Size = new Size (width, height);
-         }
-            
-         if (mySaveState)
-         {
-            FormWindowState windowState = (FormWindowState) myRegistry.getValue (RegistryRoot.HKCU, "WindowState", 
-               (int) myParent.WindowState);
-            myParent.WindowState = windowState;
-         }
-
-         // fire LoadState event
-         if(LoadState != null)
-            LoadState (this, myRegistry);
-      }
-   }
-}
+using System;
+using System.Reflection;
+using System.Windows.Forms;
+using System.Drawing;
+using System.Diagnostics;
+using Microsoft.Win32;
+
+namespace BCG
+{
+   public class PersistWindowState
+   {
+      // event info that allows form to persist extra window state data
+      public delegate void WindowStateHandler (object sender, BlueRegistry bluereg);
+      public event WindowStateHandler LoadState;
+      public event WindowStateHandler SaveState;
+
+      private Form myParent;
+      private BlueRegistry myRegistry;
+
+      private int myNormalLeft;
+      private int myNormalTop;
+      private int myNormalWidth;
+      private int myNormalHeight;
+      private bool myResizable = true;
+      private FormWindowState myWindowState;
+      private bool myAllowSaveMinimized = false;
+      private bool mySavePosition = true;
+      private bool mySaveState = true;
+
+      public PersistWindowState (bool savePosition)
+      {
+         mySavePosition = savePosition;
+      }
+
+      public PersistWindowState ()
+      {
+      }
+
+      public bool saveState
+      {
+         [DebuggerStepThrough]
+         set { mySaveState = value; }
+      }
+
+      public Form parent
+      {
+         [DebuggerStepThrough]
+         get { return myParent; }
+         set
+         {
+            myParent = value;
+            myRegistry = new BlueRegistry (myParent.Text);
+
+            // subscribe to parent form's events
+            myParent.Closing += new System.ComponentModel.CancelEventHandler(myOnClosing);
+            myParent.Resize += new System.EventHandler(myOnResize);
+            myParent.Move += new System.EventHandler(myOnMove);
+            myParent.Load += new System.EventHandler(myOnLoad);
+
+            // get initial width and height in case form is never resized
+            myNormalWidth = myParent.Width;
+            myNormalHeight = myParent.Height;
+         }
+      }
+
+      public BlueRegistry blueRegistry
+      {
+         [DebuggerStepThrough]
+         get { return myRegistry; }
+         [DebuggerStepThrough]
+         set { myRegistry = value; }
+      }
+
+      public bool resizable
+      {
+         [DebuggerStepThrough]
+         set { myResizable = value; }
+      }
+
+      public bool allowSaveMinimized
+      {
+         [DebuggerStepThrough]
+         set { myAllowSaveMinimized = value; }
+      }
+
+      private void myOnResize(object sender, System.EventArgs e)
+      {
+         // save width and height
+         if (myParent.WindowState == FormWindowState.Normal && myResizable)
+         {
+            myNormalWidth = myParent.Width;
+            myNormalHeight = myParent.Height;
+         }
+      }
+
+      private void myOnMove(object sender, System.EventArgs e)
+      {
+         // save position
+         if(myParent.WindowState == FormWindowState.Normal)
+         {
+            myNormalLeft = myParent.Left;
+            myNormalTop = myParent.Top;
+         }
+         // save state
+         myWindowState = myParent.WindowState;
+      }
+
+      private void myOnClosing(object sender, System.ComponentModel.CancelEventArgs e)
+      {
+         // save position, size and state
+         if (mySavePosition)
+         {
+            myRegistry.setValue (RegistryRoot.HKCU, "Left", myNormalLeft);
+            myRegistry.setValue (RegistryRoot.HKCU, "Top", myNormalTop);
+         }
+
+         if (myResizable)
+         {
+            myRegistry.setValue (RegistryRoot.HKCU, "Width", myNormalWidth);
+            myRegistry.setValue (RegistryRoot.HKCU, "Height", myNormalHeight);
+         }
+
+         // check if we are allowed to save the state as minimized (not normally)
+         if (!myAllowSaveMinimized)
+         {
+            if(myWindowState == FormWindowState.Minimized)
+               myWindowState = FormWindowState.Normal;
+         }
+
+         if (mySaveState)
+            myRegistry.setValue (RegistryRoot.HKCU, "WindowState", (int) myWindowState);
+        
+         // fire SaveState event
+         if (SaveState != null)
+            SaveState (this, myRegistry);
+      }
+
+      private void myOnLoad (object sender, System.EventArgs e)
+      {
+         if (mySavePosition)
+         {
+            int left = (int) myRegistry.getValue (RegistryRoot.HKCU, "Left", myParent.Left);
+            int top = (int) myRegistry.getValue (RegistryRoot.HKCU, "Top", myParent.Top);
+            myParent.Location = new Point (left, top);
+         }
+
+         if (myResizable)
+         {
+            int width = (int) myRegistry.getValue (RegistryRoot.HKCU, "Width", myParent.Width);
+            int height = (int) myRegistry.getValue (RegistryRoot.HKCU, "Height", myParent.Height);
+            myParent.Size = new Size (width, height);
+         }
+            
+         if (mySaveState)
+         {
+            FormWindowState windowState = (FormWindowState) myRegistry.getValue (RegistryRoot.HKCU, "WindowState", 
+               (int) myParent.WindowState);
+            myParent.WindowState = windowState;
+         }
+
+         // fire LoadState event
+         if(LoadState != null)
+            LoadState (this, myRegistry);
+      }
+   }
+}
@@ -1,11 +1,19 @@
 namespace Bts.BooleanExpressions;

+using bts;
+
 class BooleanExpression : IBooleanExpression {
   readonly Func<IDictionary<string, object?>, bool> evaluate;

   public BooleanExpression(Func<IDictionary<string, object?>, bool> evaluate) =>
      this.evaluate = evaluate;

-   public bool Evaluate(IDictionary<string, object?> env) =>
-      evaluate(env);
+   public bool Evaluate(IDictionary<string, object?> env) {
+      try {
+         return evaluate(env);
+      } catch (Exception ex) {
+         Log.assert(false, $"BooleanExpression.Evaluate failed: {ex}");
+         throw;
+      }
+   }
 }
@@ -47,6 +47,11 @@ IsOfType                 '/'

 %%

+{StringConstant} {
+   yylval.String = yytext.Substring(1, yytext.Length - 2).Replace(@"\'", "'");
+   return (int)Token.STRING_CONSTANT;
+} 
+
 {BooleanConstant} {
   yylval.Boolean = yytext.ToLower() switch {
      "true" => true,
@@ -57,11 +62,6 @@ IsOfType                 '/'
   return (int)Token.BOOLEAN_CONSTANT;
 }

-{StringConstant} {
-   yylval.String = yytext.Substring(1, yytext.Length - 2).Replace(@"\'", "'");
-   return (int)Token.STRING_CONSTANT;
-} 
-
 {NullConstant} {
   yylval.Null = DBNull.Value;
   return (int)Token.NULL_CONSTANT;
@@ -1,7 +1,7 @@
 <Project Sdk="Microsoft.NET.Sdk">

  <PropertyGroup>
-    <TargetFramework>net7.0</TargetFramework>
+    <TargetFramework>net8.0</TargetFramework>
    <ImplicitUsings>enable</ImplicitUsings>
    <Nullable>enable</Nullable>
  </PropertyGroup>
@@ -1,7 +1,7 @@
 <Project Sdk="Microsoft.NET.Sdk">

  <PropertyGroup>
-    <TargetFramework>net7.0</TargetFramework>
+    <TargetFramework>net8.0</TargetFramework>
    <ImplicitUsings>enable</ImplicitUsings>
    <Nullable>enable</Nullable>
  </PropertyGroup>
@@ -1,7 +1,7 @@
 <Project Sdk="Microsoft.NET.Sdk">

  <PropertyGroup>
-    <TargetFramework>net7.0</TargetFramework>
+    <TargetFramework>net8.0</TargetFramework>
    <ImplicitUsings>enable</ImplicitUsings>
    <Nullable>enable</Nullable>
  </PropertyGroup>
@@ -0,0 +1,15 @@
+<Project Sdk="Microsoft.NET.Sdk">
+
+  <PropertyGroup>
+    <TargetFramework>net8.0</TargetFramework>
+    <ImplicitUsings>enable</ImplicitUsings>
+    <Nullable>enable</Nullable>
+  </PropertyGroup>
+
+  <ItemGroup>
+    <ProjectReference Include="..\..\client\spark_websocket\spark_websocket.csproj" />
+    <ProjectReference Include="..\DataStreamExpressions\DataStreamExpressions.csproj" />
+    <ProjectReference Include="..\..\client\spark_core\spark_core.csproj" />
+  </ItemGroup>
+
+</Project>
@@ -0,0 +1,9 @@
+namespace Bts.DataStream.Websocket;
+
+class ClientSubscription(WebsocketClientID id, SubscriptionParams @params, IDisposable dispose) : IDisposable {
+   public SubscriptionParams Params => @params;
+   public WebsocketClientID Id => id;
+
+   public void Dispose() =>
+      dispose.Dispose();
+}
@@ -0,0 +1,301 @@
+namespace Bts.DataStream.Websocket;
+
+using BCG;
+
+using bts;
+using bts.api.core;
+using bts.collections;
+
+using Bts.DataStreamExpressions;
+using Bts.Websocket;
+
+using mbts.utils;
+
+using Spark.Core.Disposables;
+using Spark.Core.Json;
+
+using System.Buffers;
+using System.Reactive.Subjects;
+using System.Text;
+using System.Text.Json;
+using System.Text.Json.Nodes;
+
+/// <summary>
+/// Represents an I/O model (<see cref="ISubject{TSource, TResult}"/>), that accepts websocket
+/// requests (from clients) as input, and emits data/notifications (to clients) as output.
+/// This class's chief role is managing data subscriptions to sparkd on the clients' behalf.
+/// </summary>
+public class DataSync(JsonSerializerOptions? serializerOpts = null) :
+   ISubject<WebsocketConnectionUpdate, IEnumerable<WebsocketOutgoingMessage>> {
+   protected readonly object lockObj = new();
+   readonly JsonSerializerOptions serializerOpts = serializerOpts ?? JsonSerializerOptions.Default;
+   readonly Dictionary<WebsocketClientID, List<ClientSubscription>> clients = [];
+
+   event Action<IEnumerable<WebsocketOutgoingMessage>>? requestSend;
+
+   public void OnCompleted() =>
+      Log.Api.info("DataSync incoming request stream has completed.");
+
+   public void OnError(Exception error) =>
+      Log.Api.error($"DataSync error: {error}");
+
+   public void OnNext(WebsocketConnectionUpdate value) {
+      switch (value.Type) {
+         case WebsocketConnectionUpdateType.IncomingMessage:
+            onIncomingMessage(value.IncomingMessage);
+            break;
+         case WebsocketConnectionUpdateType.Connect:
+            onConnect(value.Connect);
+            break;
+         case WebsocketConnectionUpdateType.Disconnect:
+            onDisconnect(value.Disconnect);
+            break;
+         default:
+            Log.assert(false, $"Unexpected WebsocketConnectionUpdate type: {value.Type}");
+            break;
+      }
+   }
+
+   public IDisposable Subscribe(IObserver<IEnumerable<WebsocketOutgoingMessage>> observer) {
+      lock (lockObj) {
+         requestSend += observer.OnNext;
+         return Disposable.Create(Dispose);
+      }
+      void Dispose() {
+         lock (lockObj) {
+            requestSend -= observer.OnNext;
+         }
+      }
+   }
+
+   static JsonRpcId? extractRpcId(string messageBody) {
+      try {
+         return JsonNode.Parse(messageBody)
+            ?.AsObject()
+            ?.GetValueOrDefault("id")
+            ?.Deserialize<JsonRpcId>();
+      } catch {
+         return null;
+      }
+   }
+
+   static string getRpcNotification(DataPacket dp) {
+      using (var buffer = new MemoryStream())
+      using (var writer = new Utf8JsonWriter(buffer))
+      using (writer.WriteObject()) {
+         writer.WriteString("method", "subscription");
+         using (writer.WriteObject("params")) {
+            writer.Write("tableName", dp.schema.tableName);
+            writer.Write("columns", dp.schema.columnInfo);
+            var cellJsonFactories = dp.schema.columnInfo.Select(getCellJsonFactory).ToList();
+            using (writer.WriteArray("added")) {
+               foreach (IBlueRow row in (dp.addedRows?.Cast<IBlueRow>()).empty_if_null()) {
+                  using (writer.WriteObject()) {
+                     for (int i = 0; i < cellJsonFactories.Count; i++) {
+                        writer.Write(dp.schema.columnInfo[i].colName, cellJsonFactories[i](row));
+                     }
+                  }
+               }
+            }
+            using (writer.WriteArray("deleted")) {
+               foreach (IBlueRow row in (dp.deletedRows?.Cast<IBlueRow>()).empty_if_null()) {
+                  using (writer.WriteObject()) {
+                     for (int i = 0; i < cellJsonFactories.Count; i++) {
+                        writer.Write(dp.schema.columnInfo[i].colName, cellJsonFactories[i](row));
+                     }
+                  }
+               }
+            }
+         }
+         writer.WriteString("jsonrpc", "2.0");
+         writer.Flush();
+         return Encoding.UTF8.GetString(buffer.ToArray());
+      }
+   }
+
+   static Func<IBlueRow, JsonNode?> getCellJsonFactory(ColumnInfo colInfo) {
+      var t = colInfo.colType;
+      if (t == typeof(int)) {
+         return NullCheckWrapper(v => (int)v);
+      }
+      if (t == typeof(uint)) {
+         return NullCheckWrapper(v => (uint)v);
+      }
+      if (t == typeof(long)) {
+         return NullCheckWrapper(v => (long)v);
+      }
+      if (t == typeof(ulong)) {
+         return NullCheckWrapper(v => (ulong)v);
+      }
+      if (t == typeof(short)) {
+         return NullCheckWrapper(v => (short)v);
+      }
+      if (t == typeof(ushort)) {
+         return NullCheckWrapper(v => (ushort)v);
+      }
+      if (t == typeof(double)) {
+         return NullCheckWrapper(v => (double)v);
+      }
+      if (t == typeof(float)) {
+         return NullCheckWrapper(v => (float)v);
+      }
+      if (t == typeof(bool)) {
+         return NullCheckWrapper(v => (bool)v);
+      }
+      return NullCheckWrapper(v => (string)v);
+
+      Func<IBlueRow, JsonNode?> NullCheckWrapper(Func<object, JsonNode> toJson) =>
+         r => {
+            var obj = r[colInfo.colNo];
+            if (obj is null) {
+               return null;
+            }
+            return toJson(obj);
+         };
+   }
+
+   void onConnect(WebsocketConnectNotification notification) {
+      var id = notification.Id;
+      lock (lockObj) {
+         clients.Add(id, []);
+      }
+      Log.Api.info($"DataStreamWebsocketDataSync client connected. clientId: {notification.Id}");
+      if (notification.Message is not null) {
+         onIncomingMessage(notification.Message);
+      }
+   }
+
+   void onDisconnect(WebsocketDisconnectNotification notification) {
+      Log.Api.warn($"DataStreamWebsocketDataSync client disconnected. clientId: {notification.Id}");
+      unsubscribeAll(notification.Id);
+      clients.Remove(notification.Id);
+   }
+
+   void onIncomingMessage(WebsocketIncomingMessage message) {
+      var id = message.sender_client_id;
+      if (!JsonRpcRequest.TryDeserialize(message, out var req, out var err)) {
+         onBadRequest(message, err);
+         return;
+      }
+      switch (req.Method!.ToLower()) {
+         case "subscribe":
+            subscribe(id, req);
+            break;
+         case "publish":
+            Log.Api.error("DataStreamWebsocket api 'publish' not implemented.");
+            // TODO
+            break;
+         case "unsubscribe":
+            unsubscribe(id, req);
+            break;
+         case "unsubscribe_all":
+            unsubscribeAll(id);
+            break;
+         default:
+            Log.Api.error($"Received request of unknown type: {req}");
+            break;
+      }
+   }
+
+   void sendResponse<T>(WebsocketClientID id, JsonRpcId rpcId, T rpcParams) {
+      var rpcRes = JsonRpcResponse.Create(rpcId, rpcParams);
+      var rpcResStr = JsonSerializer.Serialize(rpcRes, serializerOpts);
+      send([new(id, rpcResStr)]);
+   }
+
+   void sendError<T>(WebsocketClientID id, JsonRpcId rpcId, JsonRpcError<T> error) {
+      var rpcRes = JsonRpcResponse.CreateError(rpcId, error);
+      var rpcResStr = JsonSerializer.Serialize(rpcRes, serializerOpts);
+      send([new(id, rpcResStr)]);
+   }
+
+   bool send(IEnumerable<WebsocketOutgoingMessage> messages) {
+      lock (lockObj) {
+         if (requestSend is null) {
+            Log.Api.warn(
+               $"Unable to send message(s) because '{nameof(requestSend)}' handler is null");
+            return false;
+         }
+         requestSend(messages);
+         return true;
+      }
+   }
+
+   void subscribe(WebsocketClientID id, JsonRpcRequest<JsonObject?> req) {
+      var subParams = req.Params!.Deserialize<SubscriptionParams>()!;
+      var exp = DataStreamExpressionsInterpreter.Evaluate(subParams.Query!);
+      var stream = exp.getView(new Dictionary<string, StatementExpression>(0));
+      var sub = new ClientSubscription(id, subParams, Disposable.Create(Dispose));
+      lock (lockObj) {
+         clients[id].Add(sub);
+         sendResponse(id, req.Id!.Value, req.Params);
+         stream.onChange += OnChange;
+      }
+
+      void OnChange(object? newValue) {
+         var dp = ExceptionAssert.of_type<DataPacket>(newValue);
+         var notif = getRpcNotification(dp);
+         send([new(id, notif)]);
+      }
+
+      void Dispose() =>
+         stream.onChange -= OnChange;
+   }
+
+   void unsubscribe(WebsocketClientID id, JsonRpcRequest<JsonObject?> req) {
+      var subParams = req.Params!.Deserialize<SubscriptionParams>();
+      var subParamsStr = JsonSerializer.Serialize(subParams, serializerOpts);
+      // Minimize amount of code inside lock (just need it to find/remove the subscription)
+      List<ClientSubscription>? subs = null;
+      ClientSubscription? sub = null;
+      lock (lockObj) {
+         subs = clients.GetValueOrDefault(id);
+         subs?.TryRemoveFirst(s => s!.Params.Equals(subParams), out sub);
+      }
+      if (subs is null) {
+         // Didn't find client (log/throw).
+         var msg = $"Unable to unsubscribe: did not find client. Id: {id}.\nParams:\n{subParamsStr}";
+         Log.Api.error(msg);
+         throw new Exception(msg);
+      }
+      if (sub is null) {
+         // Found client, but didn't find subscription (log/throw).
+         var msg = $"Unable to unsubscribe: did not find the specified subscription. " +
+            $"Id: {id}. Request:\n{subParams}";
+         Log.Api.error(msg);
+         throw new Exception(msg);
+      }
+      // Found client and subscription (dispose/remove subscription).
+      Log.Api.info($"Unsubscribing client. Id: {id}.\nParams:\n{subParamsStr}");
+      sub.Dispose();
+   }
+
+   void unsubscribeAll(WebsocketClientID id) {
+      lock (lockObj) {
+         if (!clients.TryGetValue(id, out var subs)) {
+            throw new Exception(
+               $"Unable to unsubscribe: did not find client with id {id}");
+         }
+         subs.DisposeAll();
+         subs.Clear();
+      }
+   }
+
+   void onBadRequest(WebsocketIncomingMessage req, string? errorMsg) {
+      var rpcId = extractRpcId(req.message_body);
+      var logMsg = $"Recieved invalid request:\n{req.message_body}";
+      if (errorMsg is not null) {
+         logMsg += $"\n\n**Exception**\n{errorMsg}";
+      }
+      Log.Api.error(logMsg);
+      if (rpcId is null) {
+         return;
+      }
+      // Send error back to client
+      var rpcErr = JsonRpcError.Create(
+         (long)JsonRpcErrorCode.InvalidRequest,
+         $"Received invalid request: {errorMsg ?? "unknown error"}",
+         req.message_body);
+      sendError(req.sender_client_id, rpcId.Value, rpcErr);
+   }
+}
@@ -0,0 +1,34 @@
+namespace Bts.DataStream.Websocket;
+
+using Newtonsoft.Json.Linq;
+
+using System.Diagnostics.CodeAnalysis;
+
+public static class JObjectExtensions {
+   public static bool TryParse<T>(
+      this JObject filter, string key, [MaybeNull] out T obj, [NotNullWhen(false)] out string? error) {
+      obj = default;
+      var value = filter.GetValueOrDefault(key)?.ToString();
+      if (value == null) {
+         error = $"{key} not present";
+         return true;
+      }
+      return WebsocketJsonParsers.TryParse(value, key, out obj, out error);
+   }
+
+   public static bool TryParse<T>(
+     this JObject filter, string key, out T result, [NotNullWhen(false)] out string? error, T defaultValue) {
+      var value = filter.GetValueOrDefault(key)?.ToString();
+      if (value == null) {
+         result = defaultValue;
+         error = $"{key} not present";
+         return true;
+      }
+      if (WebsocketJsonParsers.TryParse<T>(value, key, out var maybeRes, out error)) {
+         result = maybeRes ?? defaultValue;
+         return true;
+      }
+      result = defaultValue;
+      return false;
+   }
+}
@@ -0,0 +1,24 @@
+namespace Bts.DataStream.Websocket;
+
+using Newtonsoft.Json.Linq;
+
+class JsonParseResult {
+   public string? MessageId { get; init; }
+   public RequestType Type { get; init; }
+   public JObject? Parameters { get; init; }
+   public string? Error { get; init; }
+   public bool Successful => Error is not null;
+
+   public static JsonParseResult CreateError(string error) =>
+      new() {
+         Error = error,
+         MessageId = "missing"
+      };
+
+   public static JsonParseResult Create(string messageId, RequestType type, JObject parameters) =>
+      new() {
+         MessageId = messageId,
+         Parameters = parameters,
+         Type = type
+      };
+}
@@ -1,4 +1,4 @@
-namespace bts.websocket_core.json_rpc;
+namespace Bts.DataStream.Websocket;

 public enum JsonRpcErrorCode {
   None = 0,
@@ -0,0 +1,48 @@
+namespace Bts.DataStream.Websocket;
+
+using Spark.Core;
+
+using System;
+using System.Text.Json;
+using System.Text.Json.Serialization;
+
+class MultilineStringJsonConverter : JsonConverter<string> {
+   public override string? Read(ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options) {
+      if (typeToConvert != typeof(string)) {
+         throw new ArgumentException($"Unable to parse {typeToConvert}", nameof(typeToConvert));
+      }
+      return reader.TokenType switch {
+         JsonTokenType.String => StringImpl(ref reader),
+         JsonTokenType.StartArray => ArrayImpl(ref reader),
+         _ => throw new ArgumentException(
+            $"Unable to parse {typeof(SubscriptionParams)} from {reader.TokenType}"),
+      };
+
+      static string? StringImpl(ref Utf8JsonReader reader) =>
+         new(reader.GetString()!);
+
+      static string? ArrayImpl(ref Utf8JsonReader reader) {
+         var lines = new List<string>(2);
+         while (reader.Read()) {
+            switch (reader.TokenType) {
+               case JsonTokenType.String:
+                  lines.Add(reader.GetString()!);
+                  break;
+               case JsonTokenType.EndArray:
+                  return string.Join("\n", lines);
+            }
+         }
+         throw new JsonException(
+            $"Failed to parse {typeof(SubscriptionParams)} (did not reach end of array)");
+      }
+   }
+
+   public override void Write(Utf8JsonWriter writer, string value, JsonSerializerOptions options) {
+      var lines = value.Split("\n");
+      if (lines.Length < 2) {
+         writer.WriteRawValue(value);
+         return;
+      }
+      writer.WriteRawValue($"[{", ".join(lines)}]");
+   }
+}
@@ -0,0 +1,8 @@
+namespace Bts.DataStream.Websocket;
+
+public enum RequestType {
+   Unspecified,
+   Subscribe,
+   Unsubscribe,
+   UnsubscribeAll,
+}
@@ -0,0 +1,27 @@
+namespace Bts.DataStream.Websocket;
+
+using bts;
+
+using Bts.Websocket;
+
+using Google.Protobuf;
+
+using System.Text.Json;
+
+public static class Session {
+   static int sessionIdCounter;
+
+   public static async Task Start(
+      WebsocketServerConfig config, JsonSerializerOptions? opts) {
+      opts ??= JsonSerializerOptions.Default;
+      var sessionId = Interlocked.Increment(ref sessionIdCounter);
+      var gateway = new WebsocketGatewayObservable(config);
+      var sync = new DataSync(opts);
+      using (gateway.Subscribe(sync))
+      using (sync.Subscribe(gateway)) {
+         Log.Api.info($"Starting DataStream Websocket Session {sessionId}");
+         Log.Api.info($"DataStream Websocket Session {sessionId} config:\n{JsonFormatter.Default.Format(config)}");
+         await gateway.ConnectAsync();
+      }
+   }
+}
@@ -0,0 +1,110 @@
+namespace Bts.DataStream.Websocket;
+
+using Spark.Core;
+
+using System;
+using System.Diagnostics.CodeAnalysis;
+using System.Text.Json;
+using System.Text.Json.Serialization;
+
+public class SubscriptionParams : IEquatable<SubscriptionParams> {
+   static class PropNames {
+      public const string QUERY = "query";
+   }
+
+   public SubscriptionParams(string? query) {
+      Query = query;
+   }
+
+   // Serializer supports plain string, or an array of lines for convenience
+   [MaybeNull]
+   [JsonPropertyName(PropNames.QUERY)]
+   [JsonConverter(typeof(MultilineStringJsonConverter))]
+   public string? Query { get; set; }
+
+   public bool Equals(SubscriptionParams? other) =>
+      other is not null &&
+      Query == other.Query;
+
+   class SubscriptionParamsConverter : JsonConverter<SubscriptionParams> {
+      public override bool CanConvert(Type typeToConvert) =>
+         typeToConvert == typeof(SubscriptionParams);
+
+      public override SubscriptionParams Read(
+         ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options) {
+         if (typeToConvert != typeof(SubscriptionParams)) {
+            throw new ArgumentException($"Unable to parse {typeToConvert}", nameof(typeToConvert));
+         }
+         return read(ref reader);
+      }
+
+      SubscriptionParams read(ref Utf8JsonReader reader) {
+         return reader.TokenType switch {
+            JsonTokenType.String => stringImpl(ref reader),
+            JsonTokenType.StartArray => arrayImpl(ref reader),
+            JsonTokenType.StartObject => objectImpl(ref reader),
+            _ => throw new ArgumentException(
+               $"Unable to parse {typeof(SubscriptionParams)} from {reader.TokenType}"),
+         };
+
+         static SubscriptionParams stringImpl(ref Utf8JsonReader reader) =>
+            new(reader.GetString()!);
+
+         static SubscriptionParams arrayImpl(ref Utf8JsonReader reader) {
+            var lines = new List<string>(2);
+            while (reader.Read()) {
+               switch (reader.TokenType) {
+                  case JsonTokenType.String:
+                     lines.Add(reader.GetString()!);
+                     break;
+                  case JsonTokenType.EndArray:
+                     return new("\n".join(lines));
+               }
+            }
+            throw new JsonException(
+               $"Failed to parse {typeof(SubscriptionParams)} (did not reach end of array)");
+         }
+
+         static SubscriptionParams objectImpl(ref Utf8JsonReader reader) {
+            var lines = new List<string>(2);
+            while (reader.Read()) {
+               switch (reader.TokenType) {
+                  case JsonTokenType.PropertyName:
+                     var propName = reader.GetString()!;
+                     if (propName == PropNames.QUERY) {
+                        switch (reader.TokenType) {
+                           case JsonTokenType.StartArray:
+                              return arrayImpl(ref reader);
+                           case JsonTokenType.String:
+                              return stringImpl(ref reader);
+                        }
+                     }
+                     break;
+                  case JsonTokenType.String:
+                     lines.Add(reader.GetString()!);
+                     break;
+                  case JsonTokenType.EndArray:
+                     return new("\n".join(lines));
+               }
+            }
+            throw new JsonException(
+               $"Failed to parse {typeof(SubscriptionParams)} (did not reach end of array)");
+         }
+      }
+
+      public override void Write(
+         Utf8JsonWriter writer, SubscriptionParams value, JsonSerializerOptions options) {
+         var q = value.Query;
+         var lines = q?.Split("\n");
+         if (lines is null || lines.Length < 2) {
+            writer.WriteString(PropNames.QUERY, value.Query);
+            return;
+         }
+         writer.WriteStartArray(PropNames.QUERY);
+         foreach (var l in lines) {
+            writer.WriteStringValue(l);
+         }
+         writer.WriteEndArray();
+      }
+   }
+}
@@ -0,0 +1,5 @@
+global using bts.websocket_core;
+
+global using System;
+
+global using System.Collections.Immutable;
@@ -1,9 +1,10 @@
-<Project Sdk="Microsoft.NET.Sdk">
+<Project Sdk="Microsoft.NET.Sdk.Web">

  <PropertyGroup>
    <TargetFramework>net7.0</TargetFramework>
-    <ImplicitUsings>enable</ImplicitUsings>
    <Nullable>enable</Nullable>
+    <ImplicitUsings>enable</ImplicitUsings>
+    <EnableDefaultContentItems>True</EnableDefaultContentItems>
  </PropertyGroup>

 </Project>
@@ -0,0 +1,26 @@
+@page
+@model ErrorModel
+@{
+    ViewData["Title"] = "Error";
+}
+
+<h1 class="text-danger">Error.</h1>
+<h2 class="text-danger">An error occurred while processing your request.</h2>
+
+@if (Model.ShowRequestId)
+{
+    <p>
+        <strong>Request ID:</strong> <code>@Model.RequestId</code>
+    </p>
+}
+
+<h3>Development Mode</h3>
+<p>
+    Swapping to the <strong>Development</strong> environment displays detailed information about the error that occurred.
+</p>
+<p>
+    <strong>The Development environment shouldn't be enabled for deployed applications.</strong>
+    It can result in displaying sensitive information from exceptions to end users.
+    For local debugging, enable the <strong>Development</strong> environment by setting the <strong>ASPNETCORE_ENVIRONMENT</strong> environment variable to <strong>Development</strong>
+    and restarting the app.
+</p>
@@ -0,0 +1,27 @@
+using System.Diagnostics;
+using Microsoft.AspNetCore.Mvc;
+using Microsoft.AspNetCore.Mvc.RazorPages;
+
+namespace Bts.DataStream.WebsocketClient.Pages;
+
+[ResponseCache(Duration = 0, Location = ResponseCacheLocation.None, NoStore = true)]
+[IgnoreAntiforgeryToken]
+public class ErrorModel : PageModel
+{
+    public string? RequestId { get; set; }
+
+    public bool ShowRequestId => !string.IsNullOrEmpty(RequestId);
+
+    private readonly ILogger<ErrorModel> _logger;
+
+    public ErrorModel(ILogger<ErrorModel> logger)
+    {
+        _logger = logger;
+    }
+
+    public void OnGet()
+    {
+        RequestId = Activity.Current?.Id ?? HttpContext.TraceIdentifier;
+    }
+}
+
@@ -0,0 +1,15 @@
+@page
+@model IndexModel
+@{
+  ViewData["Title"] = "Home page";
+}
+
+<div class="text-center">
+  <h1 class="display-4">Welcome</h1>
+  <p>Learn about <a href="https://docs.microsoft.com/aspnet/core">building Web apps with ASP.NET Core</a>.</p>
+  <button id="connectButton">Connect to Websocket</button>
+  <div id="output"></div>
+  <input type="text" id="methodInput" placeholder="Enter method name" />
+  <input type="text" id="paramsInput" placeholder="Enter method parameters (if any)" />
+  <button id="sendMessageButton">Send JSON-RPC Request</button>
+</div>
@@ -0,0 +1,16 @@
+using Microsoft.AspNetCore.Mvc;
+using Microsoft.AspNetCore.Mvc.RazorPages;
+
+namespace Bts.DataStream.WebsocketClient.Pages;
+
+public class IndexModel : PageModel {
+   private readonly ILogger<IndexModel> _logger;
+
+   public IndexModel(ILogger<IndexModel> logger) {
+      _logger = logger;
+   }
+
+   public void OnGet() {
+
+   }
+}
@@ -0,0 +1,8 @@
+@page
+@model PrivacyModel
+@{
+    ViewData["Title"] = "Privacy Policy";
+}
+<h1>@ViewData["Title"]</h1>
+
+<p>Use this page to detail your site's privacy policy.</p>
@@ -0,0 +1,19 @@
+using Microsoft.AspNetCore.Mvc;
+using Microsoft.AspNetCore.Mvc.RazorPages;
+
+namespace Bts.DataStream.WebsocketClient.Pages;
+
+public class PrivacyModel : PageModel
+{
+    private readonly ILogger<PrivacyModel> _logger;
+
+    public PrivacyModel(ILogger<PrivacyModel> logger)
+    {
+        _logger = logger;
+    }
+
+    public void OnGet()
+    {
+    }
+}
+
@@ -0,0 +1,51 @@
+<!DOCTYPE html>
+<html lang="en">
+<head>
+    <meta charset="utf-8" />
+    <meta name="viewport" content="width=device-width, initial-scale=1.0" />
+    <title>@ViewData["Title"] - Bts.DataStream.WebsocketClient</title>
+    <link rel="stylesheet" href="~/lib/bootstrap/dist/css/bootstrap.min.css" />
+    <link rel="stylesheet" href="~/css/site.css" asp-append-version="true" />
+    <link rel="stylesheet" href="~/Bts.DataStream.WebsocketClient.styles.css" asp-append-version="true" />
+</head>
+<body>
+    <header>
+        <nav class="navbar navbar-expand-sm navbar-toggleable-sm navbar-light bg-white border-bottom box-shadow mb-3">
+            <div class="container">
+                <a class="navbar-brand" asp-area="" asp-page="/Index">Bts.DataStream.WebsocketClient</a>
+                <button class="navbar-toggler" type="button" data-bs-toggle="collapse" data-bs-target=".navbar-collapse" aria-controls="navbarSupportedContent"
+                        aria-expanded="false" aria-label="Toggle navigation">
+                    <span class="navbar-toggler-icon"></span>
+                </button>
+                <div class="navbar-collapse collapse d-sm-inline-flex justify-content-between">
+                    <ul class="navbar-nav flex-grow-1">
+                        <li class="nav-item">
+                            <a class="nav-link text-dark" asp-area="" asp-page="/Index">Home</a>
+                        </li>
+                        <li class="nav-item">
+                            <a class="nav-link text-dark" asp-area="" asp-page="/Privacy">Privacy</a>
+                        </li>
+                    </ul>
+                </div>
+            </div>
+        </nav>
+    </header>
+    <div class="container">
+        <main role="main" class="pb-3">
+            @RenderBody()
+        </main>
+    </div>
+
+    <footer class="border-top footer text-muted">
+        <div class="container">
+            &copy; 2023 - Bts.DataStream.WebsocketClient - <a asp-area="" asp-page="/Privacy">Privacy</a>
+        </div>
+    </footer>
+
+    <script src="~/lib/jquery/dist/jquery.min.js"></script>
+    <script src="~/lib/bootstrap/dist/js/bootstrap.bundle.min.js"></script>
+    <script src="~/js/site.js" asp-append-version="true"></script>
+
+    @await RenderSectionAsync("Scripts", required: false)
+</body>
+</html>
@@ -0,0 +1,48 @@
+/* Please see documentation at https://docs.microsoft.com/aspnet/core/client-side/bundling-and-minification
+for details on configuring this project to bundle and minify static web assets. */
+
+a.navbar-brand {
+  white-space: normal;
+  text-align: center;
+  word-break: break-all;
+}
+
+a {
+  color: #0077cc;
+}
+
+.btn-primary {
+  color: #fff;
+  background-color: #1b6ec2;
+  border-color: #1861ac;
+}
+
+.nav-pills .nav-link.active, .nav-pills .show > .nav-link {
+  color: #fff;
+  background-color: #1b6ec2;
+  border-color: #1861ac;
+}
+
+.border-top {
+  border-top: 1px solid #e5e5e5;
+}
+.border-bottom {
+  border-bottom: 1px solid #e5e5e5;
+}
+
+.box-shadow {
+  box-shadow: 0 .25rem .75rem rgba(0, 0, 0, .05);
+}
+
+button.accept-policy {
+  font-size: 1rem;
+  line-height: inherit;
+}
+
+.footer {
+  position: absolute;
+  bottom: 0;
+  width: 100%;
+  white-space: nowrap;
+  line-height: 60px;
+}
@@ -0,0 +1,2 @@
+<script src="~/lib/jquery-validation/dist/jquery.validate.min.js"></script>
+<script src="~/lib/jquery-validation-unobtrusive/jquery.validate.unobtrusive.min.js"></script>
@@ -0,0 +1,3 @@
+@using Bts.DataStream.WebsocketClient
+@namespace Bts.DataStream.WebsocketClient.Pages
+@addTagHelper *, Microsoft.AspNetCore.Mvc.TagHelpers
@@ -0,0 +1,3 @@
+@{
+    Layout = "_Layout";
+}
@@ -0,0 +1,26 @@
+using Microsoft.AspNetCore.Mvc;
+
+var builder = WebApplication.CreateBuilder(args);
+
+// Add services to the container.
+builder.Services.AddRazorPages();
+
+var app = builder.Build();
+
+// Configure the HTTP request pipeline.
+if (!app.Environment.IsDevelopment()) {
+   app.UseExceptionHandler("/Error");
+   // The default HSTS value is 30 days. You may want to change this for production scenarios, see https://aka.ms/aspnetcore-hsts.
+   app.UseHsts();
+}
+
+app.UseHttpsRedirection();
+app.UseStaticFiles();
+
+app.UseRouting();
+
+app.UseAuthorization();
+
+app.MapRazorPages();
+
+app.Run();
@@ -0,0 +1,37 @@
+{
+  "iisSettings": {
+    "windowsAuthentication": false,
+    "anonymousAuthentication": true,
+    "iisExpress": {
+      "applicationUrl": "http://localhost:29312",
+      "sslPort": 44390
+    }
+  },
+  "profiles": {
+    "http": {
+      "commandName": "Project",
+      "dotnetRunMessages": true,
+      "launchBrowser": true,
+      "applicationUrl": "http://localhost:5214",
+      "environmentVariables": {
+        "ASPNETCORE_ENVIRONMENT": "Development"
+      }
+    },
+    "https": {
+      "commandName": "Project",
+      "dotnetRunMessages": true,
+      "launchBrowser": true,
+      "applicationUrl": "https://localhost:7176;http://localhost:5214",
+      "environmentVariables": {
+        "ASPNETCORE_ENVIRONMENT": "Development"
+      }
+    },
+    "IIS Express": {
+      "commandName": "IISExpress",
+      "launchBrowser": true,
+      "environmentVariables": {
+        "ASPNETCORE_ENVIRONMENT": "Development"
+      }
+    }
+  }
+}
@@ -0,0 +1,9 @@
+{
+  "DetailedErrors": true,
+  "Logging": {
+    "LogLevel": {
+      "Default": "Information",
+      "Microsoft.AspNetCore": "Warning"
+    }
+  }
+}
@@ -0,0 +1,9 @@
+{
+  "Logging": {
+    "LogLevel": {
+      "Default": "Information",
+      "Microsoft.AspNetCore": "Warning"
+    }
+  },
+  "AllowedHosts": "*"
+}
@@ -0,0 +1,22 @@
+html {
+  font-size: 14px;
+}
+
+@media (min-width: 768px) {
+  html {
+    font-size: 16px;
+  }
+}
+
+.btn:focus, .btn:active:focus, .btn-link.nav-link:focus, .form-control:focus, .form-check-input:focus {
+  box-shadow: 0 0 0 0.1rem white, 0 0 0 0.25rem #258cfb;
+}
+
+html {
+  position: relative;
+  min-height: 100%;
+}
+
+body {
+  margin-bottom: 60px;
+}
@@ -0,0 +1,67 @@
+// Please see documentation at https://docs.microsoft.com/aspnet/core/client-side/bundling-and-minification
+// for details on configuring this project to bundle and minify static web assets.
+
+// Write your JavaScript code.
+const connectButton = document.getElementById('connectButton');
+const output = document.getElementById('output');
+const methodInput = document.getElementById('methodInput');
+const paramsInput = document.getElementById('paramsInput');
+const sendMessageButton = document.getElementById('sendMessageButton');
+
+let socket;
+
+connectButton.addEventListener('click', () => {
+    if (socket && socket.readyState === WebSocket.OPEN) {
+        return;
+    }
+
+    socket = new WebSocket('ws://localhost:8080'); // Replace with your WebSocket server URL
+
+    socket.onopen = (event) => {
+        output.innerHTML += 'Connected to WebSocket server<br>';
+    };
+
+    socket.onmessage = (event) => {
+        const message = JSON.parse(event.data);
+        output.innerHTML += `Received JSON-RPC response: ${JSON.stringify(message)}<br>`;
+    };
+
+    socket.onclose = (event) => {
+        if (event.wasClean) {
+            output.innerHTML += `Connection closed cleanly, code=${event.code}, reason=${event.reason}<br>`;
+        } else {
+            output.innerHTML += 'Connection abruptly closed<br>';
+        }
+    };
+
+    socket.onerror = (error) => {
+        output.innerHTML += `WebSocket Error: ${error.message}<br>`;
+    };
+});
+
+sendMessageButton.addEventListener('click', () => {
+    // const method = methodInput.value;
+    // const params = paramsInput.value;
+    const method = 'subscribe';
+    const params = `{
+"expression": "literal("n", "2.0")"
+}'
+`
+    if (socket && socket.readyState === WebSocket.OPEN && method) {
+        // const reqParams = {
+        //   'expression': message
+        // }
+        const request = {
+            jsonrpc: '2.0',
+            method: method,
+            params: {
+              query: `literal("n", "2.0")`
+            },
+            id: 1, // You can set a unique ID for each request
+        };
+
+        socket.send(JSON.stringify(request));
+        methodInput.value = ''; // Clear the input fields
+        paramsInput.value = '';
+    }
+});
@@ -0,0 +1,18 @@
+<Project Sdk="Microsoft.NET.Sdk">
+
+  <PropertyGroup>
+    <OutputType>Exe</OutputType>
+    <TargetFramework>net7.0</TargetFramework>
+    <ImplicitUsings>enable</ImplicitUsings>
+    <Nullable>enable</Nullable>
+  </PropertyGroup>
+
+  <ItemGroup>
+    <PackageReference Include="System.CommandLine" Version="2.0.0-beta4.22272.1" />
+  </ItemGroup>
+
+  <ItemGroup>
+    <ProjectReference Include="..\Bts.DataStream.Websocket\Bts.DataStream.Websocket.csproj" />
+  </ItemGroup>
+
+</Project>
@@ -0,0 +1,7 @@
+{
+  "host": "localhost",
+  "port": 8080,
+  "scheme": "ws",
+  "default_end_point": "/",
+  "keep_alive_interval_secs": 1
+}
@@ -0,0 +1,76 @@
+using bts.api.core;
+
+using Bts.DataStream.Websocket;
+
+using Google.Protobuf;
+
+using System.CommandLine.Parsing;
+using System.Text.Json;
+
+static WebsocketServerConfig readConfigFile(string path) {
+   using var fs = File.OpenRead(path);
+   using var sr = new StreamReader(fs);
+   return JsonParser.Default.Parse<WebsocketServerConfig>(sr);
+}
+
+static WebsocketServerConfig? readConfigArg(ArgumentResult result) {
+   var path = result.Tokens.Select(t => t.Value).FirstOrDefault() ?? string.Empty;
+   return readConfigFile(path);
+}
+
+static WebsocketServerConfig getDefaultConfig() {
+   const string defaultCfgPath = "DataStreamWebsocketServer.conf";
+   if (File.Exists(defaultCfgPath)) {
+      return readConfigFile(defaultCfgPath);
+   }
+   return new() {
+      Scheme = "http",
+      Host = "localhost",
+      Port = 8080,
+      DefaultEndPoint = "/",
+      KeepAliveIntervalSecs = 1
+   };
+}
+   
+static async Task run(
+   WebsocketServerConfig? wssConfig, string user, string password, string? sparkdConfigSource) {
+   var opts = new JsonSerializerOptions(JsonSerializerDefaults.Web);
+   wssConfig ??= getDefaultConfig();
+   Console.WriteLine("Starting DataStreamWebsocketServer...");
+   var serviceCtx = ServiceContext.create(sparkdConfigSource);
+   await serviceCtx.login(user, password);
+   await Session.Start(wssConfig, opts);
+   // if (sparkdConfigSource is not null && sparkdConfigSource.Length > 9999) {
+   // } else {
+   //    await ChatGptSession.Start();
+   // }
+}
+
+var userOpt = new Option<string>("--user", () => "bts");
+var passOpt = new Option<string>("--password", () => string.Empty);
+var sparkdConfigSourceOpt = new Option<string>(
+   "--sparkd", description: "The sparkd config server url");
+// var bling = new ParseArgument<WebsocketServerConfig?>(parseWsConfig);
+
+// WebsocketServerConfig? parseWsConfig(ArgumentResult result) {
+//    Token t;
+//    TokenType tt;
+//    Symbol s;
+//    throw new NotImplementedException();
+// }
+
+var configOpt = new Option<WebsocketServerConfig?>(
+   "--config", readConfigArg, description: "WebsocketServerConfig file path");
+configOpt.SetDefaultValue(new());
+var rootCommand = new RootCommand("Starts a DataStream Websocket Server");
+rootCommand.AddOption(configOpt);
+rootCommand.AddOption(userOpt);
+rootCommand.AddOption(passOpt);
+rootCommand.SetHandler(run, configOpt, userOpt, passOpt, sparkdConfigSourceOpt);
+
+try {
+   return await rootCommand.InvokeAsync(args);
+} catch (Exception ex) {
+   Console.WriteLine($"Error: {ex}");
+}
+return 1;
@@ -0,0 +1,7 @@
+global using Bts.Websocket;
+
+global using bts.websocket_core;
+
+global using System.CommandLine;
+
+global using System.Collections.Immutable;
@@ -1 +0,0 @@
-VITE_DATASTREAM_WEBSOCKET_URL=wss://localhost:8443/ws
@@ -1 +0,0 @@
-VITE_DATASTREAM_WEBSOCKET_URL=wss://localhost:8443/ws
@@ -0,0 +1,20 @@
+module.exports = {
+  root: true,
+  env: { browser: true, es2020: true },
+  extends: [
+    'eslint:recommended',
+    'plugin:react/recommended',
+    'plugin:react/jsx-runtime',
+    'plugin:react-hooks/recommended',
+  ],
+  ignorePatterns: ['dist', '.eslintrc.cjs'],
+  parserOptions: { ecmaVersion: 'latest', sourceType: 'module' },
+  settings: { react: { version: '18.2' } },
+  plugins: ['react-refresh'],
+  rules: {
+    'react-refresh/only-export-components': [
+      'warn',
+      { allowConstantExport: true },
+    ],
+  },
+}
@@ -4,8 +4,8 @@
    <JavaScriptTestRoot>src\</JavaScriptTestRoot>
    <JavaScriptTestFramework>Jest</JavaScriptTestFramework>
    <!-- Allows the build (or compile) script located on package.json to run on Build -->
-    <ShouldRunBuildScript>true</ShouldRunBuildScript>
+    <ShouldRunBuildScript>false</ShouldRunBuildScript>
    <!-- Folder where production build objects will be placed -->
-    <PublishAssetsDirectory>$(MSBuildProjectDirectory)\distill</PublishAssetsDirectory>
+    <PublishAssetsDirectory>$(MSBuildProjectDirectory)\dist</PublishAssetsDirectory>
  </PropertyGroup>
-</Project>
+</Project>
@@ -1,36 +0,0 @@
-import importPlugin from "eslint-plugin-import";
-import reactPkg from "eslint-plugin-react";
-const { configs: reactConfigs } = reactPkg;
-import js from "@eslint/js";
-import defineConfig from "./vite.config.js";
-
-export default [
-  js.configs.recommended,
-  importPlugin.flatConfigs.recommended,
-  reactConfigs.flat.recommended,
-  reactConfigs.flat["jsx-runtime"],
-  {
-    files: ["**/*.{js,jsx}"],
-    languageOptions: {
-      ecmaVersion: "latest",
-      sourceType: "module",
-      globals: {
-        console: "readonly"
-      }
-    },
-    settings: {
-      "import/resolver": {
-        vite: {
-          viteConfig: defineConfig({ mode: "development" })
-        }
-      }
-    },
-    rules: {
-      semi: "error",
-      "prefer-const": "error",
-      "no-unused-vars": "warn",
-      "import/no-dynamic-require": "warn",
-      "import/no-nodejs-modules": "warn",
-    }
-  }
-];
@@ -2,9 +2,9 @@
 <html lang="en">
  <head>
    <meta charset="UTF-8" />
-    <link rel="icon" type="image/ico" href="/logo.ico" />
+    <link rel="icon" type="image/svg+xml" href="/vite.svg" />
    <meta name="viewport" content="width=device-width, initial-scale=1.0" />
-    <title>BTS Cloud Streams</title>
+    <title>Vite + React</title>
  </head>
  <body>
    <div id="root"></div>
@@ -10,37 +10,18 @@
    "preview": "vite preview"
  },
  "dependencies": {
-    "@angular/material": "^18.2.8",
-    "@emotion/react": "^11.13.3",
-    "@emotion/styled": "^11.13.0",
-    "@fontsource/noto-sans": "^5.1.0",
-    "@material/data-table": "^15.0.0-canary.423edc3dc.0",
-    "@mui/icons-material": "^6.1.4",
-    "@mui/material": "^6.1.4",
-    "@mui/x-data-grid": "^7.20.0",
-    "bulma": "^1.0.2",
-    "classnames": "^2.5.1",
-    "react": "^18.3.1",
-    "react-data-grid": "^7.0.0-beta.47",
-    "react-data-table-component": "^7.6.2",
-    "react-dom": "^18.3.1",
-    "react-json-view-lite": "^1.5.0",
-    "react-pro-sidebar": "^1.1.0",
-    "react-use-websocket": "^4.9.0",
-    "rpc-websockets": "^9.0.4"
+    "react": "^18.2.0",
+    "react-dom": "^18.2.0",
+    "react-use-websocket": "^4.5.0"
  },
  "devDependencies": {
-    "@types/react": "^18.3.11",
-    "@types/react-dom": "^18.3.1",
-    "@vitejs/plugin-react": "^4.3.2",
-    "eslint": "^9.12.0",
-    "@eslint/js": "^9.12.0",
-    "eslint-plugin-import": "^2.31.0",
-    "eslint-plugin-react": "^7.37.1",
-    "eslint-plugin-react-hooks": "^5.0.0",
-    "eslint-plugin-react-refresh": "^0.4.12",
-    "eslint-import-resolver-vite": "^2.0.1",
-    "react-mui-sidebar": "^1.3.8",
-    "vite": "^5.4.9"
+    "@types/react": "^18.2.43",
+    "@types/react-dom": "^18.2.17",
+    "@vitejs/plugin-react": "^4.2.1",
+    "eslint": "^8.55.0",
+    "eslint-plugin-react": "^7.33.2",
+    "eslint-plugin-react-hooks": "^4.6.0",
+    "eslint-plugin-react-refresh": "^0.4.5",
+    "vite": "^5.0.8"
  }
 }
@@ -1,40 +0,0 @@
-import { Sidebar, Menu, MenuItem, Submenu, Logo } from "react-mui-sidebar";
-import './App.css';
-
-const AnimatedSideBar = () => {
-  return (
-    <Sidebar className={"sidebar-container"} width={"270px"}>
-      <Logo img="https://adminmart.com/wp-content/uploads/2024/03/logo-admin-mart-news.png">
-        AdminMart
-      </Logo>
-      <Menu subHeading="HOME">
-        <MenuItem link="/" badge="true">
-          Modern
-        </MenuItem>
-        <MenuItem>eCommerce</MenuItem>
-        <MenuItem>Analytical</MenuItem>
-      </Menu>
-      <Menu subHeading="APPS">
-        <MenuItem>Chat</MenuItem>
-        <MenuItem>Calendar</MenuItem>
-      </Menu>
-      <Menu subHeading="OTHERS">
-        <Submenu title="Menu Level">
-          <MenuItem>Post</MenuItem>
-          <MenuItem>Details</MenuItem>
-          <Submenu title="Level 2">
-            <MenuItem>new</MenuItem>
-            <MenuItem>Hello</MenuItem>
-          </Submenu>
-        </Submenu>
-
-        <MenuItem>Chip</MenuItem>
-        <MenuItem target="_blank" link="google.com">
-          External Link
-        </MenuItem>
-      </Menu>
-    </Sidebar>
-  );
-};
-
-export default AnimatedSideBar;
Author	SHA1	Message	Date
John Colvin	b8bfc1058d	check point (bc i don't trust googledrive sync yet)	2024-02-09 23:06:26 -06:00
John Colvin	82278a4e94	dev: Add gitpod.yml for cloud workspaces	2024-02-09 23:06:26 -06:00
John Colvin	592ff2a7fe	externals: Change repo url to use read-only access token	2024-02-09 23:06:26 -06:00
John Colvin	d65dc1c8e3	build: Upgrade c# to net8.0 (DataStreamSuite)	2024-02-09 23:06:26 -06:00
John Colvin	464a153582	Add Bts.DataStream.WebsocketClient csproj	2024-02-09 23:06:26 -06:00
John Colvin	0610f9ac04	Add ExpressionTests csproj	2024-02-09 23:06:26 -06:00
John Colvin	91c2f6f033	Add TransportUtilities csproj	2024-02-09 23:06:26 -06:00
John Colvin	486c314f19	Add ExpressionTester csproj	2024-02-09 23:06:26 -06:00
John Colvin	1f0b8bd82d	Add DataStream.Console csproj	2024-02-09 23:06:26 -06:00
John Colvin	6aafb2c5f1	Add json facilities for DataStream websocket api	2024-02-09 23:06:19 -06:00
John Colvin	e4d20be735	refactor: Add handy WebsocketExtensions * IsSocketStateOpen * IsClosedProperly	2024-02-09 23:05:37 -06:00
John Colvin	ca028c58ed	feat: Add Bts.DataStream.Websocket/Server	2024-02-09 22:31:10 -06:00
John Colvin	162969d2f7	Add bones for Bts.DataStreamWebApp.Client	2024-02-09 22:30:30 -06:00
John Colvin	d798594836	Add bones for Bts.DataStreamWebApp.Server	2024-02-09 22:30:07 -06:00
John Colvin	9611462e2c	add util: StringStreamEnumerable	2024-02-09 22:30:07 -06:00
John Colvin	54986aeccd	docker: Clean up, reorganize Dockerfile	2024-02-09 22:30:07 -06:00
John Colvin	6d489ce246	docker: Use explicit release ubuntu:22.04 (not 'latest') We don't want the distro to automatically upgrade one day, when 'latest' suddenly points to a new release. Upgrades like that need to be done deliberately.	2024-02-09 22:30:07 -06:00
John Colvin	6217b190bb	docker: Install gdb (for c++ debugging) VS Code debugger depends on this	2024-02-09 22:30:07 -06:00
John Colvin	6a262719af	cleanup: Remove empty file (clearly added by accident)	2024-02-09 22:30:07 -06:00
John Colvin	cf0282f86d	Add more descriptive logs (via stdout) during initialization The C++ configure/initialize routines were particularly opaque	2024-02-09 22:27:58 -06:00
John Colvin	76be3074cd	fix: Fix improper 'OfType' linq cast Previously, animations were expressed through polymorphism (derived from the base class CellAnimation), but the new impl uses composition. This 'OfType' call was leftover from the polymorphism implementation, an attempt to linq cast elements of base class (CellAnimation) to BackgroundAnimation.	2024-02-09 22:27:58 -06:00
John Colvin	029d89386f	cleanup: Remove static service Locator class It relied on a static singleton 'ServiceContext'. Replace references with a properly scoped (i.e., non-static) ServiceContext	2024-02-09 22:27:58 -06:00
John Colvin	1544a6c3aa	refactor: Reduce code (simplify) in BTS.init Remove the 'args' parameter, since the cmd line can be obtained from Environment.GetCommandLine anyways	2024-02-09 22:27:58 -06:00
John Colvin	158dfdd606	Dispose connections in SimpleTCPServer.Dispose	2024-02-09 22:27:58 -06:00
John Colvin	aaa3ff114b	Polish PubSubServer cli	2024-02-09 22:27:57 -06:00
John Colvin	c98891f5e3	cleanup: Format PubSubServer.csproj	2024-02-09 22:23:08 -06:00
John Colvin	f31d8d011b	improvement: Polish DataStreamPublisher cli Using CommandLineParser nuget pkg	2024-02-09 22:22:50 -06:00
John Colvin	7df97373ef	editorconfig: Use standard c# capitalization rules * Use 'PascalCase' for public members (instead of 'snake_case') * Use 'camelCase' for private members (instead of '_snake_case')	2024-02-09 01:06:35 -06:00
John Colvin	df7f8fc48f	refactor: Change WebsocketServerConfig.port to uint32 It should always be a number between 1-65535	2024-02-09 01:06:35 -06:00
John Colvin	33d9e344ed	refactor: Add WebsocketGatewayObservable An IObservable wrapper for IWebsocketConnectionGateway	2024-02-09 01:06:35 -06:00
John Colvin	368dce37c6	refactor: Remove nullability constraint in GetValueOrDefault	2024-02-09 01:06:34 -06:00
John Colvin	e228b33c5a	refactor: Use events to avoid circular deps for ws sync/gateway	2024-02-09 01:06:34 -06:00
John Colvin	0116a0cfe5	refactor: Let caller specify nullability of T in json parse fn	2024-02-09 01:06:34 -06:00
John Colvin	6b7c115779	refactor: Get rid of redundant UniversalNumericComparer	2024-02-09 01:06:34 -06:00
John Colvin	85593f782d	refactor: Reorganize NumericComparer, support all types	2024-02-09 01:06:34 -06:00
John Colvin	cfa5c5a283	feat: Add JsonRpc types for websocket paradigm	2024-02-09 01:06:34 -06:00
John Colvin	29b901f546	docker: Remove kitware 'rc' (release candidate) apt repo No reason to keep it. Probably shouldn't have been added in the first place	2024-02-09 01:06:34 -06:00
John Colvin	42a1f37dcf	docker: Make clang-17 the default clang	2024-02-09 01:06:34 -06:00
John Colvin	f63e6e2ad7	docker: Install clang 17	2024-02-09 01:06:34 -06:00
John Colvin	d40b34e6d5	docker: Update default compiler versions Via update-alternatives	2024-02-09 01:06:34 -06:00
John Colvin	c1cf6230fa	docker: Upgrade dotnet 7 -> 8	2024-02-09 01:06:34 -06:00
John Colvin	1118e89cf4	cleanup: Remove leftover legacy files	2024-02-09 01:06:33 -06:00
John Colvin	6a536c1208	build: Upgrade c# to net8.0	2024-02-09 01:06:33 -06:00
				`@@ -1 +0,0 @@`
				`VITE_DATASTREAM_WEBSOCKET_URL=wss://localhost:8443/ws`