webapp check pt?

cleanup (build): Remove unused cmake cache var
Put cme config url in GlbxMarketDataConfig proto
2024-10-17 14:07:44 +00:00 · 2024-10-17 05:47:47 +00:00 · 2024-10-17 05:47:47 +00:00 · 2024-10-17 05:47:47 +00:00 · 2024-10-17 05:47:47 +00:00 · 2024-10-17 05:47:47 +00:00
2268 changed files with 37647 additions and 36422 deletions
@@ -66,7 +66,6 @@ RUN \

 # Dotnet
 RUN apt-get update && apt-get install -y dotnet7
-RUN apt-get update && apt-get install -y dotnet8

 # Node
 ARG node_major=18
@@ -88,6 +87,15 @@ RUN dpkg -i /emumba/emumba-deribit-connector-3.3.1-Linux.LICENSE_DISABLED.deb \
 # Pcap
 RUN apt-get update && apt-get install -y libpcap0.8-dev

+# Databento (deps)
+RUN apt-get update && apt-get -y install libzstd-dev
+
+# Docker in Docker
+RUN apt-get update && apt-get install -y \
+    docker.io \
+    docker-compose \
+    docker-compose-v2
+
 # Set up default executables (update-alternatives)
 RUN \
   update-alternatives --install /usr/bin/clang clang /usr/bin/clang-$clang_major 100; \
@@ -121,3 +129,5 @@ RUN \
   -G sudo \
   --password "$(openssl passwd -1 "$docker_pass")" \
   "$docker_user"
+
+RUN usermod -aG docker "$docker_user"
@@ -18,27 +18,12 @@
  "runArgs": [
    "--shm-size",
    "2gb",
-    "-p",
-    "5000:5000",
-    "-p",
-    "5001:5001",
-    "-p",
-    "5173:5173",
-    "-p",
-    "5174:5174",
-    "-p",
-    "8000:8000",
-    "-p",
-    "8001:8001",
-    "-p",
-    "8002:8002",
-    "-p",
-    "8003:8003",
-    "-p",
-    "12345:12345",
+    "--network",
+    "host"
  ],
  "mounts": [
-    "source=${localEnv:HOME}/.bts/devcontainer-mounts/${devcontainerId},target=/home/bts,type=bind"
+    "source=${localEnv:HOME}/.bts/devcontainer-mounts/${devcontainerId},target=/home/bts,type=bind",
+    "source=/var/run/docker.sock,target=/var/run/docker.sock,type=bind"
  ],
  "customizations": {
    "vscode": {
@@ -1,3 +1,4 @@
 *.cmd text=auto eol=crlf
 *.cs text=auto eol=crlf
 *.sh text=auto eol=lf
+*.py text=auto eol=lf
@@ -6,18 +6,52 @@
      "type": "coreclr",
      "request": "attach"
    },
+    {
+      "name": "Api Server",
+      "type": "coreclr",
+      "request": "launch",
+      "program": "",
+      "windows": {
+        "preLaunchTask": "Build Api Server (Debug)",
+        "program": "spark_api_server.exe",
+        "args": [
+          "run",
+          "--sparkd",
+          "http://127.0.0.1:8000/config"
+        ],
+        "cwd": "build/Debug/bin/net7.0",
+      },
+      "stopAtEntry": false,
+      "console": "integratedTerminal"
+    },
+    {
+      "name": "Api Server (Skip Build)",
+      "type": "coreclr",
+      "request": "launch",
+      "program": "",
+      "windows": {
+        "program": "spark_api_server.exe",
+        "args": [
+          "run",
+          "--sparkd",
+          "http://127.0.0.1:8000/config"
+        ],
+        "cwd": "build/Debug/bin/net7.0",
+      },
+      "stopAtEntry": false,
+      "console": "integratedTerminal"
+    },
    {
      "name": "Bts.DataStreamWebApp.Server",
      "type": "coreclr",
      "request": "launch",
      "preLaunchTask": "Build DataStream WebApp (Debug)",
      "program": "${workspaceFolder}/build/Debug/bin/net7.0/Bts.DataStreamWebApp.Server.dll",
-      "args": [
-        "--config",
-        "${workspaceFolder}/src/DataStreamSuite/Bts.DataStreamWebApp.Server/Configurations/DevServer.json",
-      ],
+      "cwd": "${workspaceFolder}/build/Debug/bin/net7.0",
      "env": {
-        "ASPNETCORE_ENVIRONMENT": "Development",
+        "ASPNETCORE_HTTP_PORT": "8080",
+        "ASPNETCORE_HTTPS_PORT": "8443",
+        "ASPNETCORE_ENVIRONMENT": "${input:dataStreamWebAppServerEnvironment}",
        "ASPNETCORE_HOSTINGSTARTUPASSEMBLIES": "Microsoft.AspNetCore.SpaProxy",
      }
    },
@@ -26,12 +60,11 @@
      "type": "coreclr",
      "request": "launch",
      "program": "${workspaceFolder}/build/Debug/bin/net7.0/Bts.DataStreamWebApp.Server.dll",
-      "args": [
-        "--config",
-        "${workspaceFolder}/src/DataStreamSuite/Bts.DataStreamWebApp.Server/Configurations/DevServer.json",
-      ],
+      "cwd": "${workspaceFolder}/build/Debug/bin/net7.0",
      "env": {
-        "ASPNETCORE_ENVIRONMENT": "Development",
+        "ASPNETCORE_HTTP_PORT": "8080",
+        "ASPNETCORE_HTTPS_PORT": "8443",
+        "ASPNETCORE_ENVIRONMENT": "${input:dataStreamWebAppServerEnvironment}",
        "ASPNETCORE_HOSTINGSTARTUPASSEMBLIES": "Microsoft.AspNetCore.SpaProxy",
      }
    },
@@ -47,10 +80,9 @@
      "windows": {
        "program": "DataStreamPublisher.exe",
      },
-      "args": [
-        "--config",
-        "${workspaceFolder}/src/DataStreamSuite/DataStreamPublisher/Configurations/${input:dataStreamPublisherConfig}.json"
-      ],
+      "env": {
+        "DOTNET_ENVIRONMENT": "${input:dataStreamPublisherEnvironment}",
+      },
      "cwd": "${workspaceFolder}/build/Debug/bin/net7.0",
      "stopAtEntry": false,
      "console": "integratedTerminal"
@@ -66,14 +98,25 @@
      "windows": {
        "program": "DataStreamPublisher.exe",
      },
-      "args": [
-        "--config",
-        "${workspaceFolder}/src/DataStreamSuite/DataStreamPublisher/Configurations/${input:dataStreamPublisherConfig}.json"
-      ],
+      "env": {
+        "DOTNET_ENVIRONMENT": "${input:dataStreamPublisherEnvironment}",
+      },
      "cwd": "${workspaceFolder}/build/Debug/bin/net7.0",
      "stopAtEntry": false,
      "console": "integratedTerminal"
    },
+    {
+      "name": "(gdb) Launch",
+      "type": "cppdbg",
+      "request": "launch",
+      "program": "/home/bts/spark/build/sparkd-debug-clang/server/sparkd",
+      "args": ["run"],
+      "stopAtEntry": false,
+      "cwd": "/home/bts/spark/build/sparkd-debug-clang/server",
+      "environment": [],
+      "externalConsole": false,
+      "MIMode": "gdb",
+    },
    {
      "name": "DataStreamQueryAnalyzer",
      "type": "coreclr",
@@ -106,11 +149,10 @@
      "windows": {
        "program": "PubSubServer.exe",
      },
-      "args": [
-        "--config",
-        "${workspaceFolder}/src/DataStreamSuite/PubSubServer/Configurations/${input:pubSubServerConfig}.json"
-      ],
      "cwd": "build/Debug/bin/net7.0",
+      "env": {
+        "DOTNET_ENVIRONMENT": "${input:pubSubServerEnvironment}",
+      },
      "stopAtEntry": false,
      "console": "integratedTerminal"
    },
@@ -125,11 +167,10 @@
      "windows": {
        "program": "PubSubServer.exe",
      },
-      "args": [
-        "--config",
-        "${workspaceFolder}/src/DataStreamSuite/PubSubServer/Configurations/${input:pubSubServerConfig}.json"
-      ],
      "cwd": "build/Debug/bin/net7.0",
+      "env": {
+        "DOTNET_ENVIRONMENT": "${input:pubSubServerEnvironment}",
+      },
      "stopAtEntry": false,
      "console": "integratedTerminal"
    },
@@ -141,6 +182,10 @@
      "windows": {
        "preLaunchTask": "Build Spark (Debug)",
        "program": "Spark.exe",
+        "args": [
+          "--sparkd",
+          "http://127.0.0.1:8000/config"
+        ],
        "cwd": "build/Debug/bin/net7.0-windows",
      },
      "stopAtEntry": false,
@@ -165,43 +210,99 @@
    },
    {
      "name": "Sparkd",
-      "type": "coreclr",
+      "type": "cppdbg",
      "request": "launch",
      "preLaunchTask": "Build Sparkd (Debug | Clang)",
      "program": "",
      "linux": {
-        "program": "sparkd",
+        "program": "${workspaceFolder}/build/sparkd-debug-clang/src/app_sparkd/sparkd",
      },
      "args": [
        "run"
      ],
-      "cwd": "build/sparkd-debug-clang/server",
+      "cwd": "${workspaceFolder}/build/sparkd-debug-clang/src/app_sparkd",
      "stopAtEntry": false,
-      "console": "integratedTerminal"
-    }
+      "externalConsole": false,
+      "MIMode": "gdb",
+      "setupCommands": [
+        {
+          "description": "Enable pretty-printing for gdb",
+          "text": "-enable-pretty-printing",
+          "ignoreFailures": true
+        },
+        {
+          "description": "Set Disassembly Flavor to Intel",
+          "text": "-gdb-set disassembly-flavor intel",
+          "ignoreFailures": true
+        }
+      ]
+    },
+    {
+      "name": "Sparkd (Skip Build)",
+      "type": "cppdbg",
+      "request": "launch",
+      "program": "",
+      "linux": {
+        "program": "${workspaceFolder}/build/sparkd-debug-clang/src/app_sparkd/sparkd",
+      },
+      "args": [
+        "run"
+      ],
+      "cwd": "${workspaceFolder}/build/sparkd-debug-clang/src/app_sparkd",
+      "stopAtEntry": false,
+      "externalConsole": false,
+      "MIMode": "gdb",
+      "setupCommands": [
+        {
+          "description": "Enable pretty-printing for gdb",
+          "text": "-enable-pretty-printing",
+          "ignoreFailures": true
+        },
+        {
+          "description": "Set Disassembly Flavor to Intel",
+          "text": "-gdb-set disassembly-flavor intel",
+          "ignoreFailures": true
+        }
+      ]
+    },
  ],
  "inputs": [
    {
-      "id": "dataStreamPublisherConfig",
+      "id": "dataStreamPublisherEnvironment",
      "type": "pickString",
      "description": "Publisher Configuration",
      "options": [
-        "MarketDataPublisher",
-        "SimpleEchoClient",
-        "SimpleTestDataPublisher",
-        "TestDataPublisher",
+        "Development",
+        "MarketData",
+        "Production",
+        "ProductionAws",
+        "Staging",
      ],
      "default": "MarketDataPublisher",
    },
    {
-      "id": "pubSubServerConfig",
+      "id": "pubSubServerEnvironment",
      "type": "pickString",
      "description": "PubSub Configuration",
      "options": [
-        "PubSubServer",
-        "SimpleEchoClient",
+        "Development",
+        "EchoServer",
+        "Production",
+        "ProductionAws",
+        "Staging",
      ],
-      "default": "PubSubServer",
+      "default": "Development",
+    },
+    {
+      "id": "dataStreamWebAppServerEnvironment",
+      "type": "pickString",
+      "description": "DataStream WebApp Environment",
+      "options": [
+        "Development",
+        "Production",
+        "ProductionAws",
+        "Staging",
+      ]
    },
    {
      "id": "sparkConfigServer",
@@ -209,5 +310,15 @@
      "description": "Config Server URL",
      "default": "http://localhost:8000/config",
    },
+    {
+      "id": "isSparkApiEnabled",
+      "type": "pickString",
+      "description": "Enable Spark Api (connect to sparkd)",
+      "options": [
+        "True",
+        "False",
+      ],
+      "default": "True",
+    }
  ]
 }
@@ -212,6 +212,26 @@
        "color": "terminal.ansiCyan"
      },
    },
+    {
+      "label": "Build DataStream WebApp (Release)",
+      "group": "build",
+      "type": "process",
+      "command": "",
+      "linux": {
+        "command": "tools/build_data_stream_web_app.sh",
+      },
+      "windows": {
+        "command": "tools/build_data_stream_web_app.cmd",
+      },
+      "args": [
+        "--configuration",
+        "Release",
+      ],
+      "icon": {
+        "id": "terminal",
+        "color": "terminal.ansiCyan"
+      },
+    },
    {
      "label": "Build Spark",
      "group": "build",
@@ -587,6 +607,89 @@
        "color": "terminal.ansiGreen",
      },
    },
+    {
+      "label": "Run DataStream WebApp (Debug)",
+      "dependsOn": "Build DataStream WebApp (Debug)",
+      "command": "",
+      "linux": {
+        "command": "Bts.DataStreamWebApp.Server",
+      },
+      "windows": {
+        "command": "Bts.DataStreamWebApp.Server.exe",
+      },
+      "args": [
+        "--config",
+        "${workspaceFolder}/src/DataStreamSuite/Bts.DataStreamWebApp.Server/Configurations/DevServer.json",
+      ],
+      "options": {
+        "cwd": "${workspaceFolder}/build/Debug/bin/net7.0",
+        "env": {
+          "ASPNETCORE_HTTP_PORT": "8080",
+          "ASPNETCORE_HTTPS_PORT": "8443",
+          "ASPNETCORE_ENVIRONMENT": "Development",
+          "ASPNETCORE_HOSTINGSTARTUPASSEMBLIES": "Microsoft.AspNetCore.SpaProxy",
+        }
+      },
+      "icon": {
+        "id": "run",
+        "color": "terminal.ansiGreen",
+      },
+    },
+    {
+      "label": "Run DataStream WebApp (Release)",
+      "dependsOn": "Build DataStream WebApp (Release)",
+      "command": "",
+      "linux": {
+        "command": "Bts.DataStreamWebApp.Server",
+      },
+      "windows": {
+        "command": "Bts.DataStreamWebApp.Server.exe",
+      },
+      "args": [
+        "--config",
+        "${workspaceFolder}/src/DataStreamSuite/Bts.DataStreamWebApp.Server/Configurations/DevServer.json",
+      ],
+      "options": {
+        "cwd": "${workspaceFolder}/build/Release/bin/net7.0",
+        "env": {
+          "ASPNETCORE_HTTP_PORT": "8080",
+          "ASPNETCORE_HTTPS_PORT": "8443",
+          "ASPNETCORE_ENVIRONMENT": "Production",
+          "ASPNETCORE_HOSTINGSTARTUPASSEMBLIES": "Microsoft.AspNetCore.SpaProxy",
+        }
+      },
+      "icon": {
+        "id": "run",
+        "color": "terminal.ansiGreen",
+      },
+    },
+    {
+      "label": "Run DataStream WebApp (Release | Skip Build)",
+      "command": "",
+      "linux": {
+        "command": "Bts.DataStreamWebApp.Server",
+      },
+      "windows": {
+        "command": "Bts.DataStreamWebApp.Server.exe",
+      },
+      "args": [
+        "--config",
+        "${workspaceFolder}/src/DataStreamSuite/Bts.DataStreamWebApp.Server/Configurations/DevServer.json",
+      ],
+      "options": {
+        "cwd": "${workspaceFolder}/build/Release/bin/net7.0",
+        "env": {
+          "ASPNETCORE_HTTP_PORT": "8080",
+          "ASPNETCORE_HTTPS_PORT": "8443",
+          "ASPNETCORE_ENVIRONMENT": "Production",
+          "ASPNETCORE_HOSTINGSTARTUPASSEMBLIES": "Microsoft.AspNetCore.SpaProxy",
+        }
+      },
+      "icon": {
+        "id": "run",
+        "color": "terminal.ansiGreen",
+      },
+    },
    {
      "label": "Run PubSubServer",
      "dependsOn": "Build DataStream",
@@ -18,10 +18,10 @@ set(
 # SHARED_MEMORY_PREFIX serves as a kind of namespace so multiple applications
 # can run simultaneously without fighting over the same shm files (e.g., sparkd,
 # api server, etc.).
-set(SHARED_MEMORY_PREFIX "bts" CACHE STRING "Prefix to use when creating shared memory files")
-set(SPARK_BIN "${EXTERNALS_ROOT}/bin")
-set(SPARK_INCLUDE "${EXTERNALS_ROOT}/include")
-set(SPARK_LIB "${EXTERNALS_ROOT}/lib/$<CONFIG>")
+set(SHARED_MEMORY_PREFIX "\${BTS_SHARED_MEMORY_PREFIX}" CACHE STRING "Prefix to use when creating shared memory files")
+set(EXTERNALS_BIN "${EXTERNALS_ROOT}/bin")
+set(EXTERNALS_INCLUDE "${EXTERNALS_ROOT}/include")
+set(EXTERNALS_LIB "${EXTERNALS_ROOT}/lib/$<CONFIG>")
 set(BUILD_DOCS OFF CACHE BOOL "Build documentation")
 set(BUILD_SDK OFF CACHE BOOL "Build the sdk")
 set(EXPORT_MD_CONSUMER OFF CACHE BOOL "Export (extern) C functions for MD consumer interop")
@@ -33,123 +33,88 @@ else()
  set(BUILD_ADMIN_WEB_APP ON CACHE BOOL "Build the Admin WebApp")
 endif()

-include_directories(BEFORE SYSTEM ${SPARK_INCLUDE} ${EXTERNALS_ROOT}/include/zlib)
-link_directories(${SPARK_LIB})
+include_directories(BEFORE SYSTEM ${EXTERNALS_INCLUDE} ${EXTERNALS_ROOT}/include/zlib)
+link_directories(${EXTERNALS_LIB})

 add_compile_definitions(SHARED_MEMORY_PREFIX="${SHARED_MEMORY_PREFIX}")
-add_compile_definitions(APP_DATA_DIR="${APP_DATA_DIR}")
-if(DEFINED APP_DATA_DIR_OVERRIDE)
-  add_compile_definitions(APP_DATA_DIR_OVERRIDE="${APP_DATA_DIR_OVERRIDE}")
+if(DEFINED APP_DATA_DIR)
+  add_compile_definitions(APP_DATA_DIR="${APP_DATA_DIR}")
 endif()
 add_compile_definitions(EXPORT_MD_CONSUMER="${EXPORT_MD_CONSUMER}")
-if(WIN32)
-  # subdirs
-  add_subdirectory(src/lib)
-else()
-  # make sparkd look for libspark.so in its directory
-  set(CMAKE_BUILD_WITH_INSTALL_RPATH TRUE)
-  set(CMAKE_INSTALL_RPATH ".")
-  set(CMAKE_INSTALL_PREFIX "${CMAKE_SOURCE_DIR}/install")
-  set(SPARK_INSTALL_PREFIX "${CMAKE_INSTALL_PREFIX}/$<CONFIG>")

-  if(BUILD_SPARKD)
-    # Note: All macros that begin w/ 'CCAPI' are ccapi's macros (defined and/or
-    # read by their code). We override some below.
-    add_compile_definitions(SPARK_ENABLE_CCAPI)
+if(BUILD_SPARKD)
+  # Note: All macros that begin w/ 'CCAPI' are ccapi's macros (defined and/or
+  # read by their code). We override some below.
+  add_compile_definitions(SPARK_ENABLE_CCAPI)

-    # This allows us to override hardcoded exchange-urls with ones read from
-    # config (i.e., resolved at runtime rather than compile-time)
-    add_compile_definitions(CCAPI_EXPOSE_INTERNAL)
+  # This allows us to override hardcoded exchange-urls with ones read from
+  # config (i.e., resolved at runtime rather than compile-time)
+  add_compile_definitions(CCAPI_EXPOSE_INTERNAL)

-    # Enable ccapi services
-    add_compile_definitions(CCAPI_ENABLE_SERVICE_MARKET_DATA)
-    add_compile_definitions(CCAPI_ENABLE_SERVICE_EXECUTION_MANAGEMENT)
-    # Enable ccapi exchanges
-    add_compile_definitions(CCAPI_ENABLE_EXCHANGE_BINANCE)
-    add_compile_definitions(CCAPI_ENABLE_EXCHANGE_BINANCE_COIN_FUTURES)
-    add_compile_definitions(CCAPI_ENABLE_EXCHANGE_BINANCE_US)
-    add_compile_definitions(CCAPI_ENABLE_EXCHANGE_BINANCE_USDS_FUTURES)
-    add_compile_definitions(CCAPI_ENABLE_EXCHANGE_COINBASE)
-    add_compile_definitions(CCAPI_ENABLE_EXCHANGE_DERIBIT)
-    add_compile_definitions(CCAPI_ENABLE_EXCHANGE_OKX)
+  # Enable ccapi services
+  add_compile_definitions(CCAPI_ENABLE_SERVICE_MARKET_DATA)
+  add_compile_definitions(CCAPI_ENABLE_SERVICE_EXECUTION_MANAGEMENT)
+  # Enable ccapi exchanges
+  add_compile_definitions(CCAPI_ENABLE_EXCHANGE_BINANCE)
+  add_compile_definitions(CCAPI_ENABLE_EXCHANGE_BINANCE_COIN_FUTURES)
+  add_compile_definitions(CCAPI_ENABLE_EXCHANGE_BINANCE_US)
+  add_compile_definitions(CCAPI_ENABLE_EXCHANGE_BINANCE_USDS_FUTURES)
+  add_compile_definitions(CCAPI_ENABLE_EXCHANGE_COINBASE)
+  add_compile_definitions(CCAPI_ENABLE_EXCHANGE_DERIBIT)
+  add_compile_definitions(CCAPI_ENABLE_EXCHANGE_OKX)

-    set(CCAPI_LOG_LEVEL
-      "Warn"
-      CACHE STRING "<Fatal|Error|Warn|Info|Debug|Trace> Log level for ccapi")
-    string(TOUPPER "${CCAPI_LOG_LEVEL}" _ccapi_log_level_upper)
-    add_compile_definitions("CCAPI_ENABLE_LOG_${_ccapi_log_level_upper}")
+  set(CCAPI_LOG_LEVEL
+    "Warn"
+    CACHE STRING "<Fatal|Error|Warn|Info|Debug|Trace> Log level for ccapi")
+  string(TOUPPER "${CCAPI_LOG_LEVEL}" _ccapi_log_level_upper)
+  add_compile_definitions("CCAPI_ENABLE_LOG_${_ccapi_log_level_upper}")

-    # ** Binance **
-    add_compile_definitions(CCAPI_BINANCE_URL_REST_BASE="https://api.binance.com")
-    add_compile_definitions(BINANCE_INSTRUMENTS_PATH="/api/v3/exchangeInfo")
-    add_compile_definitions(BINANCE_PRICE_STATS_PATH="/api/v3/ticker/24hr")
+  # ** Binance **
+  add_compile_definitions(CCAPI_BINANCE_URL_REST_BASE="https://api.binance.com")
+  add_compile_definitions(BINANCE_INSTRUMENTS_PATH="/api/v3/exchangeInfo")
+  add_compile_definitions(BINANCE_PRICE_STATS_PATH="/api/v3/ticker/24hr")

-    # ** Binance US **
-    add_compile_definitions(BINANCE_US_INSTRUMENTS_PATH="/api/v3/exchangeInfo")
-    add_compile_definitions(BINANCE_US_PRICE_STATS_PATH="/api/v3/ticker/24hr")
+  # ** Binance US **
+  add_compile_definitions(BINANCE_US_INSTRUMENTS_PATH="/api/v3/exchangeInfo")
+  add_compile_definitions(BINANCE_US_PRICE_STATS_PATH="/api/v3/ticker/24hr")

-    # ** Binance USDS Futures **
-    add_compile_definitions(CCAPI_BINANCE_USDS_FUTURES_URL_REST_BASE="https://dapi.binance.com")
-    add_compile_definitions(BINANCE_USDS_FUTURES_INSTRUMENTS_PATH="/dapi/v1/exchangeInfo")
-    add_compile_definitions(BINANCE_USDS_FUTURES_PRICE_STATS_PATH="/dapi/v1/ticker/24hr")
+  # ** Binance USDS Futures **
+  add_compile_definitions(CCAPI_BINANCE_USDS_FUTURES_URL_REST_BASE="https://dapi.binance.com")
+  add_compile_definitions(BINANCE_USDS_FUTURES_INSTRUMENTS_PATH="/dapi/v1/exchangeInfo")
+  add_compile_definitions(BINANCE_USDS_FUTURES_PRICE_STATS_PATH="/dapi/v1/ticker/24hr")

-    # ** Deribit **
-    add_compile_definitions(DERIBIT_INSTRUMENT_PATH="/api/v2/public/get_instrument")
-    add_compile_definitions(DERIBIT_INSTRUMENTS_PATH="/api/v2/public/get_instruments")
-    add_compile_definitions(DERIBIT_COMBOS_PATH="/api/v2/public/get_combos")
-    add_compile_definitions(DERIBIT_TICKER_PATH="/api/v2/public/ticker")
-    add_compile_definitions(DERIBIT_CURRENCIES_METHOD="public/get_currencies")
-    add_compile_definitions(DERIBIT_INSTRUMENT_METHOD="public/get_instrument")
-    add_compile_definitions(DERIBIT_INSTRUMENTS_METHOD="public/get_instruments")
-    add_compile_definitions(DERIBIT_COMBO_DETAILS_METHOD="public/get_combo_details")
-    add_compile_definitions(DERIBIT_COMBOS_METHOD="public/get_combos")
+  # ** Deribit **
+  add_compile_definitions(DERIBIT_INSTRUMENT_PATH="/api/v2/public/get_instrument")
+  add_compile_definitions(DERIBIT_INSTRUMENTS_PATH="/api/v2/public/get_instruments")
+  add_compile_definitions(DERIBIT_COMBOS_PATH="/api/v2/public/get_combos")
+  add_compile_definitions(DERIBIT_TICKER_PATH="/api/v2/public/ticker")
+  add_compile_definitions(DERIBIT_CURRENCIES_METHOD="public/get_currencies")
+  add_compile_definitions(DERIBIT_INSTRUMENT_METHOD="public/get_instrument")
+  add_compile_definitions(DERIBIT_INSTRUMENTS_METHOD="public/get_instruments")
+  add_compile_definitions(DERIBIT_COMBO_DETAILS_METHOD="public/get_combo_details")
+  add_compile_definitions(DERIBIT_COMBOS_METHOD="public/get_combos")

-    # ** Okx **
-    add_compile_definitions(OKX_INSTRUMENTS_PATH="/api/v5/public/instruments")
-    add_compile_definitions(OKX_UNDERLYING_PATH="/api/v5/public/underlying")
-  endif()
+  # ** Okx **
+  add_compile_definitions(OKX_INSTRUMENTS_PATH="/api/v5/public/instruments")
+  add_compile_definitions(OKX_UNDERLYING_PATH="/api/v5/public/underlying")
+endif()

-  # subdirs
-  if (BUILD_SPARKD)
-    add_subdirectory(src/server/sparkd)
-    add_subdirectory(src/data_files)
-  endif()
-  add_subdirectory(src/lib)
-  if (BUILD_ADMIN_WEB_APP)
-    add_subdirectory(src/Spark.Admin.WebApp)
-  endif()
-  if(BUILD_DOCS)
-    add_subdirectory(doc/user)
-  endif()
-  if(BUILD_SDK)
-    add_subdirectory(src/sdk)
-  endif()
-
-  if(BUILD_SPARKD)
-    add_custom_command(
-      OUTPUT ${CMAKE_BINARY_DIR}/server
-      COMMAND mkdir -p ${CMAKE_BINARY_DIR}/server
-      COMMAND ln ARGS -sf ${CMAKE_BINARY_DIR}/src/server/sparkd/sparkd ${CMAKE_BINARY_DIR}/server
-      COMMAND ln ARGS -sf ${CMAKE_BINARY_DIR}/data_files ${CMAKE_BINARY_DIR}/server
-      COMMAND ln ARGS -sf ${CMAKE_BINARY_DIR}/src/lib/libspark.so ${CMAKE_BINARY_DIR}/server
-      COMMAND
-        ln ARGS -sf
-        ${EXTERNALS_ROOT}/lib/Release/libOnixS.IceImpactMulticastPriceFeedHandlerCpp.so.3.16.0.0
-        ${CMAKE_BINARY_DIR}/server
-      COMMAND ln ARGS -sf ${EXTERNALS_ROOT}/lib/Release/libACE-5.7.5.so ${CMAKE_BINARY_DIR}/server
-      COMMAND mkdir ARGS -p ${CMAKE_BINARY_DIR}/server/plugins
-      COMMENT "Creating server runtime layout"
-      DEPENDS sparkd data_files_dir)
-
-    add_custom_target(server_layout ALL DEPENDS ${CMAKE_BINARY_DIR}/server)
-
-    install(
-      DIRECTORY "${CMAKE_BINARY_DIR}/server/plugins"
-      DESTINATION "${SPARK_INSTALL_PREFIX}/bin")
-    install(
-      FILES
-      "${SPARK_LIB}/libACE-5.7.5.so"
-      "${SPARK_LIB}/libOnixS.IceImpactMulticastPriceFeedHandlerCpp.so.3.16.0.0"
-      DESTINATION "${SPARK_INSTALL_PREFIX}/bin")
-  endif()
+# subdirs
+add_subdirectory(src/lib_system)
+add_subdirectory(src/lib_domain)
+add_subdirectory(src/lib_http)
+if (BUILD_SPARKD)
+  add_subdirectory(src/lib_python_md)
+  add_subdirectory(src/app_sparkd)
+  add_subdirectory(src/data_files)
+endif()
+add_subdirectory(src/lib_spark)
+if (BUILD_ADMIN_WEB_APP)
+  add_subdirectory(src/Spark.Admin.WebApp)
+endif()
+if(BUILD_DOCS)
+  add_subdirectory(doc/user)
+endif()
+if(BUILD_SDK)
+  add_subdirectory(src/sdk)
 endif()
@@ -20,8 +20,6 @@
        "SPARK_CMAKE_CXX_FLAGS": "/MP /WX /wd4250 /wd4251 /wd4244 /wd4267 /wd4355 /wd4275 /wd4345 /bigobj /EHsc /DCURL_STATICLIB /D_WINSOCK_DEPRECATED_NO_WARNINGS /D__TBB_NO_IMPLICIT_LINKAGE /DBOOST_ALL_NO_LIB /std:c++20 /D_SCL_SECURE_NO_WARNINGS /D_CRT_SECURE_NO_WARNINGS /D_WIN32_WINNT=0x0600"
      },
      "cacheVariables": {
-        "APP_DATA_DIR": "${dollar}{APPDATA}/bts",
-        "APP_DATA_DIR_OVERRIDE": "${dollar}{SPARK_DATA_DIR}",
        "CMAKE_CXX_FLAGS": "$env{SPARK_CMAKE_CXX_FLAGS}",
        "CMAKE_CXX_FLAGS_DEBUG": "$env{SPARK_CMAKE_CXX_FLAGS} /MTd",
        "CMAKE_CXX_FLAGS_RELEASE": "$env{SPARK_CMAKE_CXX_FLAGS} /MT",
@@ -30,13 +28,12 @@
        "CMAKE_SHARED_LINKER_FLAGS_RELEASE": "/OPT:REF /OPT:ICF /LTCG",
        "CMAKE_TOOLCHAIN_FILE": "${sourceDir}/vcpkg/scripts/buildsystems/vcpkg.cmake",
        "EXTERNALS_ROOT": "${sourceDir}/externals/windows",
-        "CONFIG_DEFAULTS_PATH": "config/profiles/default/*.conf",
+        "CONFIG_DEFAULTS_PATH": "spark/config/profiles/default/*.conf",
        "DOTNET_BINARY_DIR": "${sourceDir}/build/$<CONFIG>/bin/net7.0;${sourceDir}/build/$<CONFIG>/bin/net7.0-windows",
        "EXPORT_MD_CONSUMER": {
          "type": "BOOL",
          "value": "ON"
-        },
-        "SHARED_MEMORY_PREFIX": "spark"
+        }
      }
    },

@@ -52,16 +49,11 @@
      "binaryDir": "${sourceDir}/build/${presetName}",
      "cacheVariables": {
        "platform": "",
-        "APP_DATA_DIR": "${dollar}{HOME}/sparkdata",
-        "APP_DATA_DIR_OVERRIDE": "${dollar}{SPARKD_DATA_DIR}",
-        "CMAKE_BUILD_WITH_INSTALL_RPATH": {
-          "type": "BOOL",
-          "value": "ON"
-        },
        "CMAKE_CXX_STANDARD": "20",
-        "CMAKE_INSTALL_RPATH": ".",
-        "CMAKE_INSTALL_PREFIX": "${sourceDir}/install/${presetName}/sparkd",
+        "CMAKE_INSTALL_PREFIX": "${sourceDir}/install/${presetName}",
+        "CMAKE_PREFIX_PATH": "${sourceDir}/cmake/;${sourceDir}/.venv/lib/python3.10/site-packages/pybind11/share/cmake/pybind11",
        "CMAKE_EXE_LINKER_FLAGS": "-Wno-undef",
+        "CMAKE_TOOLCHAIN_FILE": "${sourceDir}/vcpkg/scripts/buildsystems/vcpkg.cmake",
        "EXTERNALS_ROOT": "${sourceDir}/externals/linux",
        "BUILD_SDK": {
          "type": "BOOL",
@@ -70,8 +62,7 @@
        "BUILD_ADMIN_WEB_APP": {
          "type": "BOOL",
          "value": "ON"
-        },
-        "SHARED_MEMORY_PREFIX": "sparkd"
+        }
      }
    },

@@ -86,7 +77,7 @@
          "value": "ON"
        },
        "CCAPI_LOG_LEVEL": "Info",
-        "CONFIG_DEFAULTS_PATH": "config/profiles/md_dev/*.conf;config/profiles/default/*.conf"
+        "CONFIG_DEFAULTS_PATH": "spark/config/profiles/md_dev/*.conf;spark/config/profiles/default/*.conf"
      }
    },

@@ -97,7 +88,7 @@
      "cacheVariables": {
        "CMAKE_BUILD_TYPE": "Release",
        "CCAPI_LOG_LEVEL": "Warn",
-        "CONFIG_DEFAULTS_PATH": "config/profiles/md_prod/*.conf;config/profiles/default/*.conf"
+        "CONFIG_DEFAULTS_PATH": "spark/config/profiles/md_prod/*.conf;spark/config/profiles/default/*.conf"
      }
    },

@@ -152,11 +143,7 @@
    {
      "name": "spark-api-server-windows",
      "inherits": "spark",
-      "displayName": "Spark Api Server (Windows)",
-      "cacheVariables": {
-        "APP_DATA_DIR_OVERRIDE": "${dollar}{SPARK_API_SERVER_DATA_DIR}",
-        "SHARED_MEMORY_PREFIX": "spark_api_server"
-      }
+      "displayName": "Spark Api Server (Windows)"
    },

    {
@@ -165,7 +152,6 @@
      "displayName": "Spark Api Server (Linux)",
      "generator": "Ninja Multi-Config",
      "cacheVariables": {
-        "APP_DATA_DIR_OVERRIDE": "${dollar}{SPARK_API_SERVER_DATA_DIR}",
        "BUILD_ADMIN_WEB_APP": {
          "type": "BOOL",
          "value": "OFF"
@@ -178,9 +164,12 @@
        "CMAKE_CXX_COMPILER": "clang++",
        "CMAKE_CXX_FLAGS_DEBUG": "-g -fPIC -Wall -pedantic -Wextra -Werror -Wno-missing-braces -Wno-ignored-qualifiers --std=c++20 -fdiagnostics-color -DDEBUG",
        "CMAKE_CXX_FLAGS_RELEASE": "-fPIC -Wall -pedantic -Wextra -Werror -Wno-missing-braces -Wno-ignored-qualifiers --std=c++20 -fdiagnostics-color -fno-omit-frame-pointer -DNDEBUG -O3",
-        "CMAKE_INSTALL_PREFIX": "${sourceDir}/install/${presetName}",
-        "CONFIG_DEFAULTS_PATH": "config/profiles/default/*.conf",
-        "DOTNET_BINARY_DIR": "${sourceDir}/build/$<CONFIG>/bin/net7.0",
+        "CMAKE_EXPORT_COMPILE_COMMANDS": {
+          "type": "BOOL",
+          "value": "ON"
+        },
+        "CONFIG_DEFAULTS_PATH": "spark/config/profiles/default/*.conf",
+        "DOTNET_BINARY_DIR": "${sourceDir}/build/Spark.Api.Server/$<CONFIG>/bin/net7.0",
        "EXPORT_MD_CONSUMER": {
          "type": "BOOL",
          "value": "ON"
@@ -218,11 +207,8 @@
      "description": "Data Stream utils ported over from Edge",
      "binaryDir": "${sourceDir}/build/${presetName}",
      "cacheVariables": {
-        "APP_DATA_DIR": "${dollar}{APPDATA}/bts/DataStream",
-        "APP_DATA_DIR_OVERRIDE": "${dollar}{DATA_STREAM_DATA_DIR}",
-        "CONFIG_DEFAULTS_PATH": "config/profiles/default/*.conf;config/profiles/data_stream/*.conf",
-        "DOTNET_BINARY_DIR": "${sourceDir}/build/$<CONFIG>/bin/net7.0;${sourceDir}/build/$<CONFIG>/bin/net7.0-windows;${sourceDir}/build/$<CONFIG>/bin/net8.0",
-        "SHARED_MEMORY_PREFIX": "data_stream"
+        "CONFIG_DEFAULTS_PATH": "spark/config/profiles/default/*.conf;spark/config/profiles/data_stream/*.conf",
+        "DOTNET_BINARY_DIR": "${sourceDir}/build/$<CONFIG>/bin/net7.0;${sourceDir}/build/$<CONFIG>/bin/net7.0-windows"
      }
    },

@@ -238,8 +224,6 @@
      "inherits": "sparkd",
      "generator": "Ninja Multi-Config",
      "cacheVariables": {
-        "APP_DATA_DIR": "${dollar}{HOME}/.local/share/DataStream",
-        "APP_DATA_DIR_OVERRIDE": "${dollar}{DATA_STREAM_DATA_DIR}",
        "BUILD_ADMIN_WEB_APP": {
          "type": "BOOL",
          "value": "OFF"
@@ -256,14 +240,12 @@
        "CMAKE_CXX_COMPILER": "clang++",
        "CMAKE_CXX_FLAGS_DEBUG": "-g -fPIC -Wall -pedantic -Wextra -Werror -Wno-missing-braces -Wno-ignored-qualifiers --std=c++20 -fdiagnostics-color -DDEBUG",
        "CMAKE_CXX_FLAGS_RELEASE": "-fPIC -Wall -pedantic -Wextra -Werror -Wno-missing-braces -Wno-ignored-qualifiers --std=c++20 -fdiagnostics-color -fno-omit-frame-pointer -DNDEBUG -O3",
-        "CMAKE_INSTALL_PREFIX": "${sourceDir}/install/${presetName}",
-        "CONFIG_DEFAULTS_PATH": "config/profiles/default/*.conf;config/profiles/data_stream/*.conf",
+        "CONFIG_DEFAULTS_PATH": "spark/config/profiles/default/*.conf;spark/config/profiles/data_stream/*.conf",
        "DOTNET_BINARY_DIR": "${sourceDir}/build/$<CONFIG>/bin/net7.0;${sourceDir}/build/$<CONFIG>/bin/net8.0",
        "EXPORT_MD_CONSUMER": {
          "type": "BOOL",
          "value": "ON"
-        },
-        "SHARED_MEMORY_PREFIX": "data_stream"
+        }
      }
    }
  ],
@@ -302,16 +284,7 @@
        "lhs": "${hostSystemName}",
        "rhs": "Linux"
      },
-      "hidden": true,
-      "targets": [
-        "data_files_dir",
-        "admin_web_app_bundle",
-        "install",
-        "libspark",
-        "rapidjson_project",
-        "server_layout",
-        "sparkd"
-      ]
+      "hidden": true
    },

    {
@@ -321,6 +294,14 @@
      "inherits": "sparkd"
    },

+    {
+      "name": "sparkd-debug-clang-j6",
+      "configurePreset": "sparkd-debug-clang",
+      "displayName": "Sparkd (Debug | Clang | jobs=6)",
+      "inherits": "sparkd",
+      "jobs": 6
+    },
+
    {
      "name": "sparkd-debug-gcc",
      "configurePreset": "sparkd-debug-gcc",
@@ -0,0 +1,7 @@
+include_guard(GLOBAL)
+
+macro(export_to_parent)
+  foreach(var IN ITEMS ${ARGN})
+    set(${var} ${${var}} PARENT_SCOPE)
+  endforeach()
+endmacro()
@@ -0,0 +1,133 @@
+include_guard(DIRECTORY)
+include(${CMAKE_SOURCE_DIR}/cmake/ExportToParent.cmake)
+include(${CMAKE_SOURCE_DIR}/cmake/InstallBasic.cmake)
+include(${CMAKE_SOURCE_DIR}/cmake/ProtobufGenerateCpp.cmake)
+include(${CMAKE_SOURCE_DIR}/cmake/SetFirstDefined.cmake)
+include(${CMAKE_SOURCE_DIR}/cmake/safe_dep_glob.cmake)
+
+function(init_library)
+  set(_options SHARED STATIC)
+  set(_one_value_args
+    NAME
+    SOURCE_DIR
+    INCLUDE_DIR
+  )
+  set(_multi_value_args)
+  cmake_parse_arguments(_ARGS "${_options}" "${_one_value_args}" "${_multi_value_args}" ${ARGN})
+  if (_ARGS_SHARED AND _ARGS_STATIC)
+    message(FATAL_ERROR "Invalid args passed to init_library: cannot specify STATIC and SHARED")
+  endif()
+  set_first_defined(_name _ARGS_NAME PROJECT_NAME)
+  set_first_defined(_source_dir _ARGS_SOURCE_DIR _name)
+  set_first_defined(_include_dir _ARGS_INCLUDE_DIR _source_dir)
+
+  # Cpp sources
+  file(
+    GLOB_RECURSE ${_name}_SOURCES
+    RELATIVE ${CMAKE_CURRENT_SOURCE_DIR}
+    ${_source_dir}/*.cpp
+  )
+  # Note: cpp files inside 'tests' dir(s) are considered tests.
+  # Remove tests from main sources, and add them to test sources
+  foreach(item ${${_name}_SOURCES})
+    if(${item} MATCHES "tests/.*\.cpp$")
+      list(APPEND ${_name}_TEST_SOURCES ${item})
+    endif()
+  endforeach()
+  foreach(item ${${_name}_TEST_SOURCES})
+    list(REMOVE_ITEM ${_name}_SOURCES ${item})
+  endforeach()
+
+  # Proto sources 
+  file(
+    GLOB_RECURSE ${_name}_PROTO_SOURCES
+    RELATIVE ${CMAKE_CURRENT_SOURCE_DIR}
+    ${_source_dir}/protobufs/*.proto
+  )
+  update_deps_file("${${_name}_PROTO_SOURCES};")
+  protobuf_generate_cpp(
+    CXX_SOURCES ${_name}_CXX_PROTO_SOURCES
+    CXX_HEADERS ${_name}_CXX_PROTO_HEADERS
+    CXX_OUT_DIR ${CMAKE_CURRENT_BINARY_DIR}
+    PROTO_INCLUDE_DIRS
+      ${EXTERNALS_INCLUDE}
+      ${CMAKE_CURRENT_SOURCE_DIR}
+    PROTO_SOURCES
+      ${${_name}_PROTO_SOURCES}
+  )
+
+  export_to_parent(
+    ${_name}_SOURCES
+    ${_name}_TEST_SOURCES
+    ${_name}_PROTO_SOURCES
+    ${_name}_CXX_PROTO_SOURCES
+    ${_name}_CXX_PROTO_HEADERS
+  )
+
+  # Add main lib
+  if (_ARGS_STATIC)
+    add_library(${_name} STATIC ${${_name}_SOURCES} ${${_name}_CXX_PROTO_SOURCES})
+  elseif(_ARGS_SHARED)
+    add_library(${_name} SHARED ${${_name}_SOURCES} ${${_name}_CXX_PROTO_SOURCES})
+  else()
+    add_library(${_name} ${${_name}_SOURCES} ${${_name}_CXX_PROTO_SOURCES})
+  endif()
+  target_include_directories(
+    ${_name}
+    PUBLIC
+      $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}>
+      $<INSTALL_INTERFACE:${CMAKE_INSTALL_INCLUDEDIR}>
+  )
+  # Include protos (if any)
+  if (${_name}_PROTO_SOURCES)
+    target_include_directories(
+      ${_name}
+      PUBLIC
+        $<BUILD_INTERFACE:${CMAKE_CURRENT_BINARY_DIR}>
+    )
+  endif()
+  target_link_directories(
+    ${_name}
+    PUBLIC
+      $<BUILD_INTERFACE:${CMAKE_CURRENT_BINARY_DIR}>
+      $<INSTALL_INTERFACE:${CMAKE_INSTALL_LIBDIR}>
+  )
+  # Install main lib
+  foreach(_hdr ${${_name}_CXX_PROTO_HEADERS})
+    file(RELATIVE_PATH _relative_path ${CMAKE_CURRENT_BINARY_DIR} ${_hdr})
+    install(FILES ${_hdr} DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}/${_relative_path})
+  endforeach()
+  install_basic(
+    NAME ${_name}
+    INCLUDE_DIR ${_include_dir}
+  )
+
+  # Add tests executable (if there are any)
+  if (${_name}_TEST_SOURCES)
+    add_executable(${_name}-tests ${${_name}_TEST_SOURCES})
+
+    get_target_property(_lib_type ${_name} TYPE)
+    if (_lib_type STREQUAL "SHARED_LIBRARY")
+      # Set RPATH so the tests exe knows where to find the shared lib.
+      set_target_properties(
+        ${_name}-tests
+        PROPERTIES
+          BUILD_RPATH "$ORIGIN"
+          INSTALL_RPATH "$ORIGIN/../${CMAKE_INSTALL_LIBDIR}"
+      )
+    endif()
+
+    # Link main lib
+    target_link_directories(
+      ${_name}-tests
+      PRIVATE
+        $<BUILD_INTERFACE:${CMAKE_CURRENT_BINARY_DIR}>
+        $<INSTALL_INTERFACE:${CMAKE_INSTALL_LIBDIR}>
+    )
+    target_link_libraries(${_name}-tests PRIVATE ${_name})
+
+    # Catch2
+    find_package(Catch2 CONFIG REQUIRED COMPONENTS Catch2WithMain)
+    target_link_libraries(${_name}-tests PRIVATE Catch2::Catch2WithMain)
+  endif()
+endfunction()
@@ -0,0 +1,39 @@
+include_guard(DIRECTORY)
+include(GNUInstallDirs)
+include(${CMAKE_SOURCE_DIR}/cmake/PackBasic.cmake)
+include(${CMAKE_SOURCE_DIR}/cmake/SetFirstDefined.cmake)
+
+function(install_basic)
+  set(_options)
+  set(_one_value_args NAME INCLUDE_DIR)
+  set(_multi_value_args)
+  cmake_parse_arguments(_ARGS "${_options}" "${_one_value_args}" "${_multi_value_args}" ${ARGN})
+  set_first_defined(_name _ARGS_NAME PROJECT_NAME)
+  set_first_defined(_include_dir _ARGS_INCLUDE_DIR _name)
+
+  file(
+    GLOB_RECURSE ${_name}_HEADERS
+    RELATIVE ${CMAKE_CURRENT_SOURCE_DIR}
+    ${_include_dir}/*.h)
+
+  target_sources(${_name}
+    PUBLIC
+      FILE_SET HEADERS FILES ${${_name}_HEADERS})
+
+  install(
+    TARGETS ${_name}
+    EXPORT ${_name}Targets
+    RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR}
+    LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR}
+    ARCHIVE DESTINATION ${CMAKE_INSTALL_LIBDIR}
+    FILE_SET HEADERS DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}
+  )
+
+  install(
+    EXPORT ${_name}Targets
+    FILE ${_name}Targets.cmake
+    NAMESPACE ${_name}::
+    DESTINATION ${CMAKE_INSTALL_LIBDIR}/cmake/${_name})
+
+  pack_basic()
+endfunction()
@@ -0,0 +1,44 @@
+include_guard(DIRECTORY)
+include(CMakePackageConfigHelpers)
+include(${CMAKE_SOURCE_DIR}/cmake/SetFirstDefined.cmake)
+
+function(pack_basic)
+  set(_options)
+  set(_one_value_args NAME)
+  set(_multi_value_args)
+  cmake_parse_arguments(_ARGS "${_options}" "${_one_value_args}" "${_multi_value_args}" ${ARGN})
+  set_first_defined(_name _ARGS_NAME PROJECT_NAME)
+
+  find_package(Git QUIET)
+
+  if(GIT_FOUND)
+    execute_process(
+      COMMAND ${GIT_EXECUTABLE} describe --tags --always --dirty
+      WORKING_DIRECTORY ${CMAKE_SOURCE_DIR}
+      OUTPUT_VARIABLE PROJECT_VERSION
+      OUTPUT_STRIP_TRAILING_WHITESPACE
+    )
+  endif()
+
+  if(NOT GIT_FOUND OR PROJECT_VERSION STREQUAL "")
+    set(PROJECT_VERSION "0.0.0")
+  endif()
+
+  write_basic_package_version_file(
+    "${CMAKE_CURRENT_BINARY_DIR}/${_name}ConfigVersion.cmake"
+    VERSION ${PROJECT_VERSION}
+    COMPATIBILITY AnyNewerVersion
+  )
+
+  configure_package_config_file(
+    "${CMAKE_CURRENT_SOURCE_DIR}/Config.cmake.in"
+    "${CMAKE_CURRENT_BINARY_DIR}/${_name}Config.cmake"
+    INSTALL_DESTINATION ${CMAKE_INSTALL_LIBDIR}/cmake/${_name}
+  )
+
+  install(
+    FILES
+      "${CMAKE_CURRENT_BINARY_DIR}/${_name}Config.cmake"
+      "${CMAKE_CURRENT_BINARY_DIR}/${_name}ConfigVersion.cmake"
+    DESTINATION ${CMAKE_INSTALL_LIBDIR}/cmake/${_name})
+endfunction()
@@ -1,145 +0,0 @@
-set(Protobuf_PROTOC_EXECUTABLE "${SPARK_BIN}/protoc")
-set(PROTOBUF_INCLUDE_DIRS "${SPARK_INCLUDE}")
-set(PROTOBUF_STATIC_LIB "libprotobuf")
-
-function(Protobuf_Generate_Cpp SRCS HDRS)
-  set(${SRCS})
-  set(${HDRS})
-
-  foreach(FIL ${ARGN})
-    get_filename_component(ABS_FIL ${FIL} ABSOLUTE)
-    get_filename_component(FIL_WE ${FIL} NAME_WE)
-    get_filename_component(FIL_DIR ${FIL} DIRECTORY)
-    if(FIL_DIR)
-      set(FIL_WE "${FIL_DIR}/${FIL_WE}")
-    endif()
-
-    list(APPEND ${SRCS} "${CMAKE_CURRENT_BINARY_DIR}/${FIL_WE}.pb.cc")
-    list(APPEND ${HDRS} "${CMAKE_CURRENT_BINARY_DIR}/${FIL_WE}.pb.h")
-
-    add_custom_command(
-      OUTPUT "${CMAKE_CURRENT_BINARY_DIR}/${FIL_WE}.pb.cc"
-             "${CMAKE_CURRENT_BINARY_DIR}/${FIL_WE}.pb.h"
-      COMMAND
-        ${Protobuf_PROTOC_EXECUTABLE} ARGS --cpp_out=${CMAKE_BINARY_DIR}/src -I
-        ${PROTOBUF_INCLUDE_DIRS} -I ${CMAKE_SOURCE_DIR}/src ${ABS_FIL}
-      DEPENDS ${ABS_FIL}
-      COMMENT "Running C++ protocol buffer compiler on ${FIL}"
-      VERBATIM)
-  endforeach()
-
-  set_source_files_properties(${${SRCS}} ${${HDRS}} PROPERTIES GENERATED TRUE)
-
-  # remove some warnings from protobuf generated files
-  if(WIN32)
-    set_source_files_properties(${${SRCS}} PROPERTIES COMPILE_FLAGS
-                                                      "/wd4309 /wd4838")
-  else()
-    set_source_files_properties(
-      ${${SRCS}}
-      PROPERTIES
-        COMPILE_FLAGS
-        "-Wno-unused-parameter -Wno-effc++ -Wno-pedantic -Wno-sign-compare -Wno-uninitialized"
-    )
-  endif()
-
-  set(${SRCS}
-      ${${SRCS}}
-      PARENT_SCOPE)
-  set(${HDRS}
-      ${${HDRS}}
-      PARENT_SCOPE)
-endfunction()
-
-function(Service_Compiler_Generate_Cpp SRCS HDRS)
-  set(${SRCS})
-  set(${HDRS})
-
-  foreach(FIL ${ARGN})
-    get_filename_component(ABS_FIL ${FIL} ABSOLUTE)
-    get_filename_component(FIL_WE ${FIL} NAME_WE)
-    get_filename_component(FIL_DIR ${FIL} DIRECTORY)
-    if(FIL_DIR)
-      set(FIL_WE "${FIL_DIR}/${FIL_WE}")
-    endif()
-
-    list(APPEND ${SRCS} "${CMAKE_CURRENT_BINARY_DIR}/${FIL_WE}.pb.cc")
-    list(APPEND ${HDRS} "${CMAKE_CURRENT_BINARY_DIR}/${FIL_WE}.pb.h")
-
-    list(APPEND ${SRCS} "${CMAKE_CURRENT_BINARY_DIR}/${FIL_WE}.client.pb.cc")
-    list(APPEND ${HDRS} "${CMAKE_CURRENT_BINARY_DIR}/${FIL_WE}.client.pb.h")
-
-    list(APPEND ${SRCS} "${CMAKE_CURRENT_BINARY_DIR}/${FIL_WE}.server.pb.cc")
-    list(APPEND ${HDRS} "${CMAKE_CURRENT_BINARY_DIR}/${FIL_WE}.server.pb.h")
-
-    set(TEMPLATES
-        ${CMAKE_CURRENT_SOURCE_DIR}/api/service_compiler/templates/cpp/client.pb.cc
-        ${CMAKE_CURRENT_SOURCE_DIR}/api/service_compiler/templates/cpp/client.pb.h
-        ${CMAKE_CURRENT_SOURCE_DIR}/api/service_compiler/templates/cpp/server.pb.cc
-        ${CMAKE_CURRENT_SOURCE_DIR}/api/service_compiler/templates/cpp/server.pb.h
-    )
-
-    if(WIN32)
-      add_custom_command(
-        OUTPUT "${CMAKE_CURRENT_BINARY_DIR}/${FIL_WE}.pb.cc"
-               "${CMAKE_CURRENT_BINARY_DIR}/${FIL_WE}.pb.h"
-               "${CMAKE_CURRENT_BINARY_DIR}/${FIL_WE}.client.pb.cc"
-               "${CMAKE_CURRENT_BINARY_DIR}/${FIL_WE}.client.pb.h"
-               "${CMAKE_CURRENT_BINARY_DIR}/${FIL_WE}.server.pb.cc"
-               "${CMAKE_CURRENT_BINARY_DIR}/${FIL_WE}.server.pb.h"
-        COMMAND SET PYTHONPATH=${SPARK_PYTHON}
-        COMMAND ${CMAKE_SOURCE_DIR}/.venv/Scripts/activate.bat
-        COMMAND
-          ${Protobuf_PROTOC_EXECUTABLE} ARGS --cpp_out=${CMAKE_BINARY_DIR}/src
-          --plugin=protoc-gen-service=${CMAKE_SOURCE_DIR}/src/lib/api/service_compiler/plugin.cmd
-          --service_out=lang=cpp:${CMAKE_BINARY_DIR}/src -I
-          ${PROTOBUF_INCLUDE_DIRS} -I ${CMAKE_SOURCE_DIR}/src ${ABS_FIL}
-        DEPENDS ${ABS_FIL} ${TEMPLATES}
-                ${CMAKE_CURRENT_SOURCE_DIR}/api/service_compiler/plugin.py
-        COMMENT "Running C++ service compiler on ${FIL}"
-        VERBATIM)
-    else()
-      add_custom_command(
-        OUTPUT "${CMAKE_CURRENT_BINARY_DIR}/${FIL_WE}.pb.cc"
-               "${CMAKE_CURRENT_BINARY_DIR}/${FIL_WE}.pb.h"
-               "${CMAKE_CURRENT_BINARY_DIR}/${FIL_WE}.client.pb.cc"
-               "${CMAKE_CURRENT_BINARY_DIR}/${FIL_WE}.client.pb.h"
-               "${CMAKE_CURRENT_BINARY_DIR}/${FIL_WE}.server.pb.cc"
-               "${CMAKE_CURRENT_BINARY_DIR}/${FIL_WE}.server.pb.h"
-        COMMAND ${CMAKE_SOURCE_DIR}/tools/venv_exec.sh
-        ARGS
-          ${Protobuf_PROTOC_EXECUTABLE}
-          --cpp_out=${CMAKE_BINARY_DIR}/src
-          --plugin=protoc-gen-service=${CMAKE_SOURCE_DIR}/src/lib/api/service_compiler/plugin.py
-          --service_out=lang=cpp,make_file_name=${ABS_FIL}:${CMAKE_BINARY_DIR}/src
-          -I ${PROTOBUF_INCLUDE_DIRS} -I ${CMAKE_SOURCE_DIR}/src ${ABS_FIL}
-        DEPENDS ${ABS_FIL} ${TEMPLATES}
-                ${CMAKE_CURRENT_SOURCE_DIR}/api/service_compiler/plugin.py
-        COMMENT "Running C++ service compiler on ${FIL}"
-        VERBATIM)
-    endif()
-
-  endforeach()
-
-  # remove some warnings from protobuf generated files
-  if(WIN32)
-    set_source_files_properties(${${SRCS}} PROPERTIES COMPILE_FLAGS
-                                                      "/wd4309 /wd4838")
-  else()
-    set_source_files_properties(
-      ${${SRCS}}
-      PROPERTIES
-        COMPILE_FLAGS
-        "-Wno-unused-parameter -Wno-effc++ -Wno-pedantic -Wno-sign-compare -Wno-uninitialized"
-    )
-  endif()
-
-  set(${SRCS}
-      ${${SRCS}}
-      PARENT_SCOPE)
-  set(${HDRS}
-      ${${HDRS}}
-      PARENT_SCOPE)
-
-endfunction()
-
@@ -0,0 +1,76 @@
+include_guard(GLOBAL)
+
+function(protobuf_generate_cpp)
+  set(_options)
+  set(_one_value_args
+    CXX_SOURCES
+    CXX_HEADERS
+    CXX_OUT_DIR
+    PROTOC
+  )
+  set(_multi_value_args
+    PROTO_INCLUDE_DIRS
+    PROTO_SOURCES
+  )
+  cmake_parse_arguments(_ARGS "${_options}" "${_one_value_args}" "${_multi_value_args}" ${ARGN})
+
+  if (NOT DEFINED _ARGS_PROTOC)
+    set(_ARGS_PROTOC "${EXTERNALS_BIN}/protoc")
+  endif()
+  set(${_ARGS_CXX_SOURCES})
+  set(${_ARGS_CXX_HEADERS})
+
+  set(_proto_include_cli_args)
+  foreach(_dir ${_ARGS_PROTO_INCLUDE_DIRS})
+    list(APPEND _proto_include_cli_args -I ${_dir})
+  endforeach()
+
+  foreach(_fil ${_ARGS_PROTO_SOURCES})
+    get_filename_component(_abs_fil ${_fil} ABSOLUTE)
+    get_filename_component(_fil_we ${_fil} NAME_WE)
+    get_filename_component(_fil_dir ${_fil} DIRECTORY)
+    if(_fil_dir)
+      set(_fil_we "${_fil_dir}/${_fil_we}")
+    endif()
+    list(APPEND ${_ARGS_CXX_SOURCES} "${_ARGS_CXX_OUT_DIR}/${_fil_we}.pb.cc")
+    list(APPEND ${_ARGS_CXX_HEADERS} "${_ARGS_CXX_OUT_DIR}/${_fil_we}.pb.h")
+    message(STATUS "add_custom_command for ${_fil} -> ${_ARGS_CXX_OUT_DIR}/${_fil_we}.pb.h")
+    add_custom_command(
+      OUTPUT
+        "${_ARGS_CXX_OUT_DIR}/${_fil_we}.pb.cc"
+        "${_ARGS_CXX_OUT_DIR}/${_fil_we}.pb.h"
+      COMMAND
+        ${_ARGS_PROTOC}
+      ARGS
+        --cpp_out=${_ARGS_CXX_OUT_DIR} ${_proto_include_cli_args} ${_fil}
+      DEPENDS ${_abs_fil}
+      COMMENT "Running C++ protocol buffer compiler on ${_fil}"
+      VERBATIM
+    )
+  endforeach()
+
+  set_source_files_properties(
+    ${${_ARGS_CXX_SOURCES}}
+    ${${_ARGS_CXX_HEADERS}}
+    PROPERTIES
+      GENERATED TRUE
+  )
+
+  # remove some warnings from protobuf generated files
+  if(WIN32)
+    set_source_files_properties(
+      ${${_ARGS_CXX_SOURCES}}
+      PROPERTIES
+        COMPILE_FLAGS "/wd4309 /wd4838"
+    )
+  else()
+    set_source_files_properties(
+      ${${_ARGS_CXX_SOURCES}}
+      PROPERTIES
+        COMPILE_FLAGS "-Wno-unused-parameter -Wno-effc++ -Wno-pedantic -Wno-sign-compare -Wno-uninitialized"
+    )
+  endif()
+
+  set(${_ARGS_CXX_SOURCES} ${${_ARGS_CXX_SOURCES}} PARENT_SCOPE)
+  set(${_ARGS_CXX_HEADERS} ${${_ARGS_CXX_HEADERS}} PARENT_SCOPE)
+endfunction()
@@ -0,0 +1,58 @@
+function(protobuf_generate_cpp_config)
+  set(_options)
+  set(_one_value_args
+    PROTO_SOURCES_HEADER
+    PROTO_DEFAULTS_HEADER
+  )
+  set(_multi_value_args
+    PROTO_SOURCES
+    PROTO_DEFAULTS
+  )
+  cmake_parse_arguments(_ARGS "${_options}" "${_one_value_args}" "${_multi_value_args}" ${ARGN})
+
+  # Configure the 'config_proto_files' header. It provides an array of strings populated with the
+  # files in PROTO_SOURCES. At runtime, we use this array to generate a special 'config' proto
+  # message with fields for each message defined in said files. A global instance of this 'config'
+  # proto is then constructed, and populated with [compile-time] defaults, and [runtime] overrides
+  # (i.e., in spark.conf). Calls to `bts::config::get<T>()` retrieve the value from the field
+  # associated with type `T` in this global instance.
+  message(STATUS "PROTO_SOURCES: ${_ARGS_PROTO_SOURCES}")
+  message(STATUS "PROTO_SOURCES_HEADER: ${_ARGS_PROTO_SOURCES_HEADER}")
+  message(STATUS "PROTO_DEFAULTS_HEADER: ${_ARGS_PROTO_DEFAULTS_HEADER}")
+  if (_ARGS_PROTO_SOURCES_HEADER)
+    set(PROTO_SOURCES)
+    set(PROTO_SOURCES_CXX_ARRAY_ITEMS "")
+    foreach(_proto_path ${_ARGS_PROTO_SOURCES})
+      list(APPEND PROTO_SOURCES ${_proto_path})
+      string(APPEND PROTO_SOURCES_CXX_ARRAY_ITEMS "\"${_proto_path}\",\n")
+    endforeach()
+    list(LENGTH PROTO_SOURCES PROTO_SOURCES_LENGTH)
+    message(STATUS "CONFIGURING PROTO_SOURCES_HEADER): ${_ARGS_PROTO_SOURCES_HEADER}")
+    configure_file(${CMAKE_SOURCE_DIR}/cmake/config_proto_files.h.in ${_ARGS_PROTO_SOURCES_HEADER})
+  endif()
+
+  # Generate static defaults. Ideally, this would be a single string, but we use an array because
+  # the compiler complains if the string is longer than 65k.
+  set(_cfg_defaults_str "inline const char * config_static_defaults[] = { ")
+
+  foreach(_cfg_file_path ${_ARGS_PROTO_DEFAULTS})
+    file(READ ${_cfg_file_path} _cfg_file_content)
+    string(REPLACE "\"" "\\\"" _cfg_file_content_escaped ${_cfg_file_content})
+    string(REPLACE "\n" ";" _cfg_lines ${_cfg_file_content_escaped})
+
+    foreach(_line ${_cfg_lines})
+      string(APPEND _cfg_defaults_str "\n  \"${_line}\",")
+    endforeach(_line)
+  endforeach(_cfg_file_path)
+
+  string(APPEND _cfg_defaults_str "};")
+  set(_old_cfg_defaults_str "")
+
+  if(EXISTS ${_ARGS_PROTO_DEFAULTS_HEADER})
+    file(READ ${_ARGS_PROTO_DEFAULTS_HEADER} _old_cfg_defaults_str)
+  endif()
+
+  if(NOT _cfg_defaults_str STREQUAL _old_cfg_defaults_str)
+    file(WRITE "${_ARGS_PROTO_DEFAULTS_HEADER}" "${_cfg_defaults_str}")
+  endif()
+endfunction()
@@ -0,0 +1,49 @@
+include_guard(DIRECTORY)
+
+function(protobuf_generate_fs)
+  set(_options)
+  set(_one_value_args TARGET_NAME OUT_DIR PROTOC)
+  set(_multi_value_args PROTO_INCLUDE_DIRS PROTO_SOURCES)
+  cmake_parse_arguments(_ARGS "${_options}" "${_one_value_args}" "${_multi_value_args}" ${ARGN})
+
+  if (NOT DEFINED _ARGS_TARGET_NAME)
+    set(_ARGS_TARGET_NAME "${PROJECT_NAME}_generate_fs_protos")
+  endif()
+  if (NOT DEFINED _ARGS_PROTOC)
+    set(_ARGS_PROTOC "${EXTERNALS_BIN}/protoc")
+  endif()
+
+  set(_proto_include_cli_args)
+  foreach(_dir ${_ARGS_PROTO_INCLUDE_DIRS})
+    list(APPEND _proto_include_cli_args -I ${_dir})
+  endforeach()
+
+  set(_outputs)
+  foreach(_proto_source ${_ARGS_PROTO_SOURCES})
+    # get_filename_component(_abs_proto_source ${_proto_source} ABSOLUTE)
+    get_filename_component(_proto_source_we ${_proto_source} NAME_WE)
+    # get_filename_component(_proto_source_dir ${_proto_source} DIRECTORY)
+    list(APPEND _outputs ${_ARGS_OUT_DIR}/${_proto_source_we}.fable.fs)
+  endforeach()
+
+  set(_venv_exec ${CMAKE_SOURCE_DIR}/tools/venv_exec.sh)
+  set(_plugin_py ${CMAKE_SOURCE_DIR}/src/lib_spark/spark/api/service_compiler/plugin.py)
+  set(_template_fs ${CMAKE_SOURCE_DIR}/src/lib_spark/spark/api/service_compiler/templates/fs/fable.fs)
+  add_custom_command(
+    OUTPUT ${_outputs}
+    WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}
+    COMMAND ${_venv_exec}
+      ${_ARGS_PROTOC}
+      --plugin=protoc-gen-service=${_plugin_py}
+      --service_out=lang=fs,strip_out_path=1:${_ARGS_OUT_DIR}
+      ${_proto_include_cli_args}
+      ${_ARGS_PROTO_SOURCES}
+    DEPENDS
+      ${_venv_exec}
+      ${_plugin_py}
+      ${_template_fs}
+      ${CMAKE_CURRENT_SOURCE_DIR}/package-lock.json
+      ${_ARGS_PROTO_SOURCES}
+  )
+  add_custom_target(${_ARGS_TARGET_NAME} ALL DEPENDS "${_outputs}")
+endfunction()
@@ -0,0 +1,33 @@
+include_guard(DIRECTORY)
+
+function(protobuf_generate_js)
+  set(_options)
+  set(_one_value_args TARGET_NAME OUTPUT)
+  set(_multi_value_args PROTO_INCLUDE_DIRS PROTO_SOURCES)
+  cmake_parse_arguments(_ARGS "${_options}" "${_one_value_args}" "${_multi_value_args}" ${ARGN})
+
+  if (NOT DEFINED _ARGS_TARGET_NAME)
+    set(_ARGS_TARGET_NAME "${PROJECT_NAME}_generate_js_protos")
+  endif()
+
+  set(_proto_include_cli_args)
+  foreach(_dir ${_ARGS_PROTO_INCLUDE_DIRS})
+    list(APPEND _proto_include_cli_args -p ${_dir})
+  endforeach()
+
+  add_custom_command(
+    OUTPUT ${_ARGS_OUTPUT}
+    WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}
+    COMMAND npx
+      --no-install pbjs
+      -t json-module
+      -w commonjs
+      -o ${_ARGS_OUTPUT}
+      ${_proto_include_cli_args}
+      ${_ARGS_PROTO_SOURCES}
+    DEPENDS
+      ${CMAKE_CURRENT_SOURCE_DIR}/package-lock.json
+      ${_ARGS_PROTO_SOURCES}
+  )
+  add_custom_target(${_ARGS_TARGET_NAME} ALL DEPENDS ${_ARGS_OUTPUT})
+endfunction()
@@ -0,0 +1,10 @@
+include_guard(GLOBAL)
+
+macro(set_first_defined RESULT)
+  foreach(_arg ${ARGN})
+    if (DEFINED ${_arg})
+      set(${RESULT} ${${_arg}})
+      break()
+    endif()
+  endforeach()
+endmacro()
@@ -0,0 +1,51 @@
+include_guard(DIRECTORY)
+include(${CMAKE_SOURCE_DIR}/cmake/TernaryOperator.cmake)
+
+function(sym_link_dir)
+  set(_options)
+  set(_one_value_args TARGET_NAME SOURCE DEST COMMENT)
+  set(_multi_value_args)
+  cmake_parse_arguments(_ARGS "${_options}" "${_one_value_args}" "${_multi_value_args}" ${ARGN})
+  add_custom_command(
+    OUTPUT ${_ARGS_DEST}
+    COMMAND ln ARGS -sf ${_ARGS_SOURCE} ${_ARGS_DEST}
+  )
+  set_ternary(_comment _ARGS_COMMENT "${_ARGS_COMMENT}" "linking dir ${_ARGS_SOURCE} -> ${_ARGS_DEST}")
+  add_custom_target(
+    ${_ARGS_TARGET_NAME} ALL
+    DEPENDS ${_ARGS_DEST}
+    COMMENT "${_comment}"
+  )
+endfunction()
+
+function(sym_link_file)
+  set(_options)
+  set(_one_value_args TARGET_NAME SOURCE DEST INSTALL_DEST)
+  set(_multi_value_args)
+  cmake_parse_arguments(_ARGS "${_options}" "${_one_value_args}" "${_multi_value_args}" ${ARGN})
+  add_custom_command(
+    OUTPUT ${_ARGS_DEST}
+    COMMAND ln ARGS -s ${_ARGS_SOURCE} ${_ARGS_DEST}
+  )
+  add_custom_target(
+    ${_ARGS_TARGET_NAME} ALL
+    DEPENDS ${_ARGS_DEST}
+  )
+  if (DEFINED _ARGS_INSTALL_DEST)
+    install(
+      FILES
+      ${_ARGS_SOURCE}
+      DESTINATION ${_ARGS_INSTALL_DEST}
+    )
+  endif()
+endfunction()
+
+function(sym_link_local_lib LIB_NAME)
+  set(_lib_file_name "${CMAKE_SHARED_LIBRARY_PREFIX}${LIB_NAME}.so")
+  file(
+    CREATE_LINK
+      "${CMAKE_BINARY_DIR}/src/lib_${LIB_NAME}/${_lib_file_name}"
+      "${CMAKE_CURRENT_BINARY_DIR}/${_lib_file_name}"
+    SYMBOLIC
+  )
+endfunction()
@@ -0,0 +1,9 @@
+include_guard(GLOBAL)
+
+macro(set_ternary RESULT CONDITION TRUE_VALUE FALSE_VALUE)
+  if(${CONDITION})
+    set(${RESULT} ${TRUE_VALUE})
+  else()
+    set(${RESULT} ${FALSE_VALUE})
+  endif()
+endmacro()
@@ -0,0 +1,11 @@
+#pragma once
+
+#include <array>
+
+namespace bts::@PROJECT_NAME@::protobufs {
+
+inline std::array<const char *, @PROTO_SOURCES_LENGTH@> config_proto_files = {
+@PROTO_SOURCES_CXX_ARRAY_ITEMS@
+};
+
+} // namespace bts::@PROJECT_NAME@::protobufs
@@ -0,0 +1,51 @@
+services:
+  sparkd:
+    build:
+      dockerfile: Dockerfile
+      context: ./sparkd
+    command: [ "./sparkd", "run" ]
+    ports: [ "8000:8000", "8001:8001", "8002:8002" ]
+
+  pubsub:
+    build:
+      dockerfile: pubsub.Dockerfile
+      context: ./DataStream
+    command: [ "./PubSubServer" ]
+    environment:
+      - DOTNET_ENVIRONMENT=Production
+    ports:
+      - 22019:22019
+
+  md-publisher:
+    build:
+      dockerfile: md-publisher.Dockerfile
+      context: ./DataStream
+    depends_on:
+      - pubsub
+      - sparkd
+    command:
+      [
+        "dockerize",
+        "-wait",
+        "tcp://pubsub:22019",
+        "-wait",
+        "tcp://sparkd:8000",
+        "-timeout",
+        "10s",
+        "./DataStreamPublisher"
+      ]
+    environment:
+      - DOTNET_ENVIRONMENT=MarketData
+
+  webapp:
+    ports:
+      - 8080:8080
+      - 8443:8443
+    build:
+      dockerfile: webapp.Dockerfile
+      context: ./DataStream
+    command: [ "./Bts.DataStreamWebApp.Server" ]
+    environment:
+      - ASPNETCORE_HTTP_PORT=8080
+      - ASPNETCORE_HTTPS_PORT=8443
+      - ASPNETCORE_ENVIRONMENT=Staging
@@ -0,0 +1,72 @@
+services:
+  sparkd:
+    build:
+      dockerfile: src/server/sparkd/Dockerfile
+    command: [ "./sparkd", "run" ]
+    ports: [ "8000:8000", "8001:8001", "8002:8002" ]
+
+  pubsub:
+    environment:
+      - DOTNET_ENVIRONMENT=Staging
+    build:
+      dockerfile: src/DataStreamSuite/PubSubServer/Dockerfile
+    ports:
+      - 22019:22019
+    command: [ "./PubSubServer" ]
+
+  md-publisher:
+    build:
+      dockerfile: src/DataStreamSuite/DataStreamPublisher/Dockerfile
+    depends_on:
+      - pubsub
+      - sparkd
+    command:
+      [
+        "dockerize",
+        "-wait",
+        "tcp://pubsub:22019",
+        "-wait",
+        "tcp://sparkd:8000",
+        "-timeout",
+        "10s",
+        "./DataStreamPublisher"
+      ]
+    environment:
+      - DOTNET_ENVIRONMENT=MarketData
+
+  webapp:
+    ports:
+      - 8080:8080
+      - 8443:8443
+    build:
+      dockerfile: src/DataStreamSuite/Bts.DataStreamWebApp.Server/Dockerfile
+    command: [ "./Bts.DataStreamWebApp.Server" ]
+    environment:
+      - ASPNETCORE_ENVIRONMENT=Staging
+
+  api-server:
+    profiles:
+      - api-server
+    depends_on:
+      - sparkd
+    ports:
+      - 8008:8008
+      - 9443:9443
+    build:
+      dockerfile: src/client/spark_api_server/Dockerfile
+    command: [ "./spark_api_server", "run" ]
+    environment:
+      - ASPNETCORE_ENVIRONMENT=Staging
+
+  sparkd-ping-test:
+    depends_on:
+      - sparkd
+    profiles:
+      - ping-test
+    build:
+      context: .
+      dockerfile_inline: |
+        FROM bluetradingsystems/linux_dotnet7
+        USER "root"
+        RUN apt-get update && apt-get -y install iputils-ping
+    command: [ "ping", "sparkd" ]
@@ -53,7 +53,7 @@ The status callback is called when the market for this security opens, closes, o
 // Subscribe to security trading status
 void IAlgoServices::subscribe_status(SecurityKey);
 // You implement
-virtual void on_status(SecurityKey, co2::TradingStatus) override;
+virtual void on_status(SecurityKey, domain::protobufs::TradingStatus) override;
 ```

 # Child Orders
@@ -0,0 +1,39 @@
+FROM ubuntu:22.04
+
+ARG DEBIAN_FRONTEND=noninteractive
+
+RUN apt-get update \
+  && apt-get install -y pkg-config \
+  && apt-get install -y locales \
+  && apt-get install -y ca-certificates \
+  && apt-get install -y libssh2-1-dev \
+  && apt-get install -y software-properties-common \
+  && apt-get install -y sudo
+
+# GCC
+ARG gcc_major=13
+RUN add-apt-repository ppa:ubuntu-toolchain-r/test \
+  && apt-get update \
+  && apt-get install -y g++-$gcc_major
+
+RUN apt-get update \
+  && apt-get install -y dotnet7
+
+RUN locale-gen en_US.UTF-8
+
+ARG docker_user="bts"
+ARG docker_uid="1000"
+ARG docker_pass="bts"
+
+RUN \
+  groupadd -g "$docker_uid" "$docker_user" \
+  && useradd \
+  -m \
+  -u "$docker_uid" \
+  -g "$docker_uid" \
+  -G sudo \
+  --password "$(openssl passwd -1 "$docker_pass")" \
+  "$docker_user"
+
+USER bts
+WORKDIR /home/bts
@@ -1,19 +1,11 @@
 namespace Bts.BooleanExpressions;

-using bts;
-
 class BooleanExpression : IBooleanExpression {
   readonly Func<IDictionary<string, object?>, bool> evaluate;

   public BooleanExpression(Func<IDictionary<string, object?>, bool> evaluate) =>
      this.evaluate = evaluate;

-   public bool Evaluate(IDictionary<string, object?> env) {
-      try {
-         return evaluate(env);
-      } catch (Exception ex) {
-         Log.assert(false, $"BooleanExpression.Evaluate failed: {ex}");
-         throw;
-      }
-   }
+   public bool Evaluate(IDictionary<string, object?> env) =>
+      evaluate(env);
 }
@@ -47,11 +47,6 @@ IsOfType                 '/'

 %%

-{StringConstant} {
-   yylval.String = yytext.Substring(1, yytext.Length - 2).Replace(@"\'", "'");
-   return (int)Token.STRING_CONSTANT;
-} 
-
 {BooleanConstant} {
   yylval.Boolean = yytext.ToLower() switch {
      "true" => true,
@@ -62,6 +57,11 @@ IsOfType                 '/'
   return (int)Token.BOOLEAN_CONSTANT;
 }

+{StringConstant} {
+   yylval.String = yytext.Substring(1, yytext.Length - 2).Replace(@"\'", "'");
+   return (int)Token.STRING_CONSTANT;
+} 
+
 {NullConstant} {
   yylval.Null = DBNull.Value;
   return (int)Token.NULL_CONSTANT;
@@ -0,0 +1 @@
+VITE_DATASTREAM_WEBSOCKET_URL=wss://localhost:443
@@ -0,0 +1 @@
+VITE_DATASTREAM_WEBSOCKET_URL=wss://localhost:443
@@ -12,6 +12,18 @@ module.exports = {
  settings: { react: { version: '18.3.1' } },
  plugins: ['react-refresh'],
  rules: {
+    "import/order": [
+      "error",
+      {
+        "groups": [
+          "builtin",
+          "external",
+          "internal",
+          ["parent", "sibling", "index"]
+        ],
+        "newlines-between": "always"
+      }
+    ],
    'react-refresh/only-export-components': [
      'warn',
      { allowConstantExport: true },
@@ -19,9 +19,13 @@
    "@mui/material": "^5.15.20",
    "@mui/x-data-grid": "^7.7.1",
    "bulma": "^1.0.1",
+    "classnames": "^2.5.1",
    "react": "^18.3.1",
+    "react-data-grid": "^7.0.0-beta.44",
    "react-data-table-component": "^7.6.2",
    "react-dom": "^18.3.1",
+    "react-json-view-lite": "^1.4.0",
+    "react-pro-sidebar": "^1.1.0",
    "react-use-websocket": "^4.8.1",
    "rpc-websockets": "^7.11.2"
  },
@@ -30,9 +34,11 @@
    "@types/react-dom": "^18.3.0",
    "@vitejs/plugin-react": "^4.3.1",
    "eslint": "^8.57.0",
+    "eslint-plugin-import": "^2.29.1",
    "eslint-plugin-react": "^7.34.2",
    "eslint-plugin-react-hooks": "^4.6.2",
    "eslint-plugin-react-refresh": "^0.4.7",
+    "react-mui-sidebar": "^1.3.5",
    "vite": "^5.2.13"
  }
 }
@@ -0,0 +1,40 @@
+import { Sidebar, Menu, MenuItem, Submenu, Logo } from "react-mui-sidebar";
+import './App.css';
+
+const AnimatedSideBar = () => {
+  return (
+    <Sidebar className={"sidebar-container"} width={"270px"}>
+      <Logo img="https://adminmart.com/wp-content/uploads/2024/03/logo-admin-mart-news.png">
+        AdminMart
+      </Logo>
+      <Menu subHeading="HOME">
+        <MenuItem link="/" badge="true">
+          Modern
+        </MenuItem>
+        <MenuItem>eCommerce</MenuItem>
+        <MenuItem>Analytical</MenuItem>
+      </Menu>
+      <Menu subHeading="APPS">
+        <MenuItem>Chat</MenuItem>
+        <MenuItem>Calendar</MenuItem>
+      </Menu>
+      <Menu subHeading="OTHERS">
+        <Submenu title="Menu Level">
+          <MenuItem>Post</MenuItem>
+          <MenuItem>Details</MenuItem>
+          <Submenu title="Level 2">
+            <MenuItem>new</MenuItem>
+            <MenuItem>Hello</MenuItem>
+          </Submenu>
+        </Submenu>
+
+        <MenuItem>Chip</MenuItem>
+        <MenuItem target="_blank" link="google.com">
+          External Link
+        </MenuItem>
+      </Menu>
+    </Sidebar>
+  );
+};
+
+export default AnimatedSideBar;
@@ -2,6 +2,54 @@
@use "@material/icon-button";
@use "@material/data-table/data-table";

+.root-div {
+  display: flex;
+  flex-direction: row;
+  align-items: flex-start;
+}
+
+.main-content-div {
+  display: flex;
+  flex-direction: row;
+}
+
+.title-div {
+  width: 100%;
+}
+
+.container {
+  display: flex;
+}
+
+.container-column {
+  display: flex;
+  flex-direction: column;
+}
+
+.container-row {
+  display: flex;
+  flex-direction: row;
+  align-items: stretch;
+}
+
+box.query-input-form {
+   /* p: 2, display: "flex", flexFlow: "column", alignItems: "stretch"  */
+  display: flex;
+  flex-flow: column;
+  align-items: stretch;
+}
+
+/* li.ps-menuitem-root:hover {
+  background-color: #ff9922;  
+} */
+
+.container-row-centered {
+  display: flex;
+  flex-direction: row;
+  justify-content: center;
+  align-items: center;
+}
+
 h1#main-header {
  padding: 4px 8px 4px 8px;
  text-align: center;
@@ -26,3 +74,28 @@ td#cell-string {
 td#cell-date-time {
  text-align: center;
 }
+
+/* Sidebar.css */
+/* .sidebar-container {
+  display: flex;
+  flex-direction: column;
+}
+
+.sidebar {
+  position: relative;
+  top: 0;
+  left: 0;
+  height: 100vh;
+  background-color: #333;
+  color: white;
+  overflow: hidden;
+  transition: transform 0.3s ease;
+}
+
+.sidebar.closed {
+  width: 0;
+}
+
+.sidebar.open {
+  width: 250px;
+} */
@@ -1,22 +1,171 @@
-import { createTheme, ThemeProvider, CssBaseline } from '@mui/material';
+import { ThemeProvider, CssBaseline, GlobalStyles } from '@mui/material';
+import QueryStatusView from './QueryStatusView';
 import useMediaQuery from '@mui/material/useMediaQuery';
 import QueryControl from './QueryControl';
+import DataTable from './DataTable';
+import { useState, useEffect, useMemo } from 'react';
+import useWebSocket, { ReadyState } from 'react-use-websocket';
+import Sidebar from './SideBar';
 import getTheme from './Theme';
+
 import '@fontsource/noto-sans';
 import './App.css';
+import './SideBar.css';
+
+const webSocketUrl = import.meta.env.VITE_DATASTREAM_WEBSOCKET_URL;
+// Always use the same subscription id (1), because we only show 1 table at a time.
+// The server will automatically dispose the previous subscription and start a new one
+// if we're currently subscribed to another query.
+const subscriptionId = "1";
+
+var rpcReqIdCounter = 1;
+
+function subscribeReq(query) {
+  return {
+    jsonrpc: "2.0",
+    id: rpcReqIdCounter++,
+    method: "subscribe",
+    params: {
+      subscription_id: subscriptionId,
+      query: query
+    }
+  };
+}
+
+function unsubscribeReq() {
+  return {
+    jsonrpc: "2.0",
+    id: rpcReqIdCounter++,
+    method: "unsubscribe",
+    params: {
+      subscription_id: subscriptionId,
+    }
+  };
+}

 const App = () => {
  const prefersDarkMode = useMediaQuery('(prefers-color-scheme: dark)');
  const theme = getTheme({ mode: prefersDarkMode ? 'dark' : 'light' });
+  const globalStyles = {
+    ':root': {
+      '--primary-color': theme.palette.primary.main,
+      '--primary-dark-color': theme.palette.primary.dark,
+      '--secondary-color': theme.palette.secondary.main,
+      '--secondary-dark-color': theme.palette.secondary.dark,
+      '--background-color': theme.palette.background.default,
+    },
+  };
+  const [blueTable, setBlueTable] = useState({ columns: null, rows: null });
+  const [queryError, setQueryError] = useState(null);
+  const [queryState, setQueryState] = useState("Unsubscribed");
+  const queryStateDetails = useMemo(() => {
+    if (queryState == "Unsubscribed") {
+      return "Enter query above, then click the play button to subscribe";
+    }
+    if (queryState == "Subscribed") {
+      return null;
+    }
+  }, [queryState]);
+  const [sidebarIsOpen, setSidebarIsOpen] = useState(false);
+  const [cannedQuery, setCannedQuery] = useState(null);

+  const onCannedQueryClicked = (cannedQuery) => {
+    console.info(`Executing canned query: ${JSON.stringify(cannedQuery)}`);
+    const queryText = cannedQuery.query.join("\n");
+    setCannedQuery({ queryText: queryText, clickTime: Date.now() });
+    subscribe(queryText);
+  };
+
+  const { sendJsonMessage, getWebSocket, readyState } = useWebSocket(
+    webSocketUrl,
+    {
+      onOpen: (e) => console.info(`WebSocket open. ${JSON.stringify(e)}`),
+      onClose: (e) => console.info(`WebSocket closed. ${JSON.stringify(e)}`),
+      onError: (e) => console.error(`WebSocket error: ${JSON.stringify(e)}`),
+      shouldReconnect: () => true,
+    });
+
+  useEffect(() => {
+    console.info(`WebSocket readyState: ${readyState}`)
+    if (readyState != ReadyState.OPEN) {
+      return;
+    }
+    const ws = getWebSocket();
+    ws.onmessage = e => {
+      console.info(`Received message: ${e.data}`);
+      const msg = JSON.parse(e.data);
+      setQueryError(msg.error);
+      if (msg.error != null) {
+        setQueryState("Error");
+      } else if (msg.id != null) {
+        setQueryState(msg.result.status);
+      } else {
+        console.assert(msg.params.subscription_id == subscriptionId);
+        console.info(`Received rows: ${msg.params.table.all}`);
+        setBlueTable({ columns: msg.params.table.columns, rows: msg.params.table.all });
+      }
+    };
+  }, [getWebSocket, readyState]);
+
+  const subscribe = (query) => {
+    const req = subscribeReq(query);
+    console.info(`Subscribing: ${JSON.stringify(subscribeReq(query))}`);
+    sendJsonMessage(req);
+  };
+
+  const unsubscribe = () => {
+    const req = unsubscribeReq();
+    console.info(`Unsubscribing: ${JSON.stringify(req)}`);
+    sendJsonMessage(req);
+  };
+
+  const onOpened = () => {
+    setSidebarIsOpen(true);
+  };
+
+  const onClosed = () => {
+    setSidebarIsOpen(false);
+  };
+
+  const sidebarWidth = sidebarIsOpen ? '250px' : '65px';
  return (
    <ThemeProvider theme={theme}>
      <CssBaseline />
-      <h1 id='main-header' style={{
-        color: theme.palette.text.primary,
-        fontFamily: 'Noto Sans',
-      }}>BTS Cloud Streams</h1>
-      <QueryControl websocketUrl="ws://localhost:5000/ws" theme={theme} />
+      <GlobalStyles styles={globalStyles} />
+      <div className='root-div'>
+        <div style={{
+          display: 'flex',
+          flexDirection: 'column',
+          alignItems: 'flex-start',
+          flexWrap: 'nowrap',
+          width: sidebarWidth,
+          transition: 'width 0.3s ease',
+        }}>
+          <Sidebar onSelection={onCannedQueryClicked} onOpened={onOpened} onClosed={onClosed} />
+        </div>
+        <div style={{
+          display: 'flex',
+          flexDirection: 'column',
+          left: sidebarWidth,
+          width: '100%',
+          transition: 'width 0.3s ease',
+        }}>
+          <div style={{ width: '100%', display: 'flex', flexDirection: 'column', alignItems: 'flex-start', margin: theme.spacing(2, 0, 0, 0) }}>
+            <div style={{ width: '90%', rightMargin: '100px' }}>
+              <QueryControl
+                queryState={queryState}
+                queryText={cannedQuery?.queryText}
+                queryTextResetTime={cannedQuery?.clickTime}
+                subscribe={subscribe}
+                unsubscribe={unsubscribe} />
+            </div>
+            <div style={{ width: '90%', rightMargin: '100px' }}>
+              <QueryStatusView queryState={queryState} details={queryStateDetails} error={queryError} />
+              <DataTable columns={blueTable?.columns ?? []} rows={blueTable?.rows ?? []} />
+            </div>
+          </div>
+        </div>
+      </div>
    </ThemeProvider>
  );
 };
@@ -0,0 +1,53 @@
+@use "@material/checkbox";
+@use "@material/icon-button";
+@use "@material/data-table/data-table";
+
+.data-grid-container {
+  overflow: auto;
+  white-space: nowrap;
+}
+
+
+.data-grid {
+  max-height: 1000px;
+}
+
+div[role=columnheader] {
+  border-top: 0px;
+  border-bottom: 0px;
+  background-color: var(--primary-dark-color);
+}
+
+/* div[role=columnheader] {
+  border-top: 0px;
+  border-bottom: 0px;
+  background-color: var(--primary-dark-color);
+}
+
+.rdg-row {
+  border: 0px;
+  border-top: 0px;
+  border-bottom: 0px;
+  background-color: rgba(var(--primary-dark-color), 0.8);
+  color: var(--primary-color);
+}
+
+.rdg-row-odd {
+  border: 0px;
+  border-top: 0px;
+  border-bottom: 0px;
+  background-color: var(--primary-dark-color);
+  color: var(--primary-color);
+}
+
+.rdg-cell {
+  border-bottom: 0px;
+}
+
+.rdg-cell.even {
+  border-bottom: 5px;
+}
+
+.rdg-cell.odd {
+  border-bottom: 5px;
+} */
@@ -1,19 +1,40 @@
 import PropTypes from 'prop-types';
+import { useTheme } from '@mui/material/styles';
 import { DataGrid } from '@mui/x-data-grid';
-
-function getGridColDef(col) {
-  return {
-    field: col.name,
-    headerName: col.name,
-  };
-}
+import './DataGrid.css'
+import { useMemo, useState } from 'react';

 export default function DataTable({ columns, rows }) {
+  const theme = useTheme();
+  const [columnWidths, setColumnWidths] = useState({});
+
+  const handleColumnResize = (params) => {
+    const newWidth = params.width;
+    setColumnWidths((prevWidths) => ({
+      ...prevWidths,
+      [params.colDef.field]: params.width,
+    }));
+  };
+
+  const getGridColDef = useMemo(() => {
+    return col => {
+      return {
+        field: col.name,
+        headerName: col.name,
+        width: columnWidths[col.name],
+      };
+    }
+  }, [columnWidths]);
+
  return (
    <DataGrid
-      columns={columns.map(getGridColDef)}
-      rows={rows}
+      className='data-grid-container'
+      autosizeOnMount={false}
+      columns={columns?.map(getGridColDef) ?? []}
+      onColumnResize={handleColumnResize}
+      rows={rows ?? []}
      sx={{
+        margin: theme.spacing(2, 0, 0, 0),
        borderColor: 'primary.main',
        backgroundColor: 'primary.dark',
        color: 'text.primary',
@@ -49,4 +70,5 @@ DataTable.propTypes = {
    })
  ).isRequired,
  rows: PropTypes.arrayOf(PropTypes.array).isRequired,
+  apiRef: PropTypes.object,
 };
@@ -1,100 +1,119 @@
 import PropTypes from 'prop-types';
 import DataTable from './DataTable';
-import QueryInputForm from './QueryInputForm';
-import Container from '@mui/material/Container';
-import React from 'react';
+import { useRef, useState, useEffect } from 'react';
+import { useGridApiRef } from '@mui/x-data-grid';
+import Box from '@mui/material/Box';
+import Button from '@mui/material/Button';
+import Grid from '@mui/material/Grid';
+import Paper from '@mui/material/Paper';
+import PlayArrowIcon from '@mui/icons-material/PlayArrow';
+import StopIcon from '@mui/icons-material/Stop';
+import TextField from '@mui/material/TextField';
+import Tooltip from '@mui/material/Tooltip';

-var rpcReqIdCounter = 1;
+const QueryControl = (props) => {
+  const { queryState, queryText, queryTextResetTime, subscribe, unsubscribe } = props;
+  // const theme = useTheme();
+  let textInput = useRef();
+  const [queryTextChangeTime, setQueryTextChangeTime] = useState(null);
+  useEffect(() => {
+    if (queryText == null || textInput.current == null) {
+      return;
+    }
+    // Init text field (unless a non-empty string is already present, or the user has modified
+    // the query - we don't want to overwrite their changes).
+    if (queryTextChangeTime == null || queryTextResetTime > queryTextChangeTime) {
+      console.info(`Initializing query text: ${queryText}`)
+      textInput.current.value = queryText;
+    }
+  }, [queryText, queryTextChangeTime, queryTextResetTime]);
+  const onClearClick = () => {
+    textInput.current.value = '';
+  };

-function subscribeReq(query) {
-  return {
-    jsonrpc: "2.0",
-    id: rpcReqIdCounter++,
-    method: "subscribe",
-    params: {
-      subscription_id: "1",
-      query: query
+  const onRunClick = () => {
+    subscribe(textInput.current.value);
+  };
+
+  const onStopClick = () => {
+    unsubscribe();
+  };
+
+  const onKeyDown = (e) => {
+    // Detect keyboard shortcuts.
+    //   Ctrl+Enter : Run
+    if (e.ctrlKey && e.key === 'Enter') {
+      onRunClick();
    }
  };
-}

-function unsubscribeReq() {
-  return {
-    jsonrpc: "2.0",
-    id: rpcReqIdCounter++,
-    method: "unsubscribe",
-    params: {
-      subscription_id: "1",
-    }
+  const onChange = (e) => {
+    // So we know not to overwrite the changes with the canned query on re-render
+    setQueryTextChangeTime(Date.now());
  };
-}

-class QueryControl extends React.Component {
-  constructor({ websocketUrl, theme }) {
-    super(websocketUrl);
-    console.log('Connecting to WebSocket:', websocketUrl);
-    this.state = {
-      rows: null,
-      ws: new WebSocket(websocketUrl),
-      columns: null,
-      isRunning: false,
-      theme: theme,
-    };
-    this.state.ws.onopen = () => {
-      console.info('WebSocket open');
-    };
-    this.state.ws.onclose = () => {
-      console.info('WebSocket closed');
-    };
-    this.state.ws.onmessage = e => {
-      console.info(`Received message: ${e.data}`);
-      const msg = JSON.parse(e.data);
-      if (msg.id != null) {
-        let newState = { ...this.state };
-        newState.isRunning = msg.result.status == 'Subscribed';
-        this.setState(newState);
-      } else {
-        let newState = { ...this.state };
-        newState.rows = msg.params.all;
-        console.info(`Received rows: ${newState.rows}`);
-        newState.columns = msg.params.columns;
-        this.setState(newState);
-      }
-    };
-
-    this.start = (query) => {
-      this.state.ws.send(JSON.stringify(subscribeReq(query)));
-    };
-
-    this.stop = () => {
-      this.state.ws.send(JSON.stringify(unsubscribeReq()));
-    };
-  }
-
-  componentWillUnmount() {
-    if (this.state.ws) {
-      console.warn('Closing websocket because component is unmounting');
-      this.state.ws.close();
-    }
-  }
-
-  render() {
-    console.info('Rendering');
-    return (
-      <Container>
-        <QueryInputForm isRunning={this.state.isRunning} onStart={this.start} onStop={this.stop} />
-        {
-          this.state.rows == null ? <p style={{ color: this.state.theme.palette.warning.main }}>No data received</p> :
-            <DataTable columns={this.state.columns} rows={this.state.rows} />
-        }
-      </Container>
-    );
-  }
-}
+  return (
+    <Box sx={{ display: "flex", flexFlow: "column" }}>
+      <Paper elevation={3} sx={{ color: 'primary.main' }}>
+        <Box sx={{ p: 2 }}>
+          <TextField
+            fullWidth
+            multiline
+            rows={10}
+            variant="outlined"
+            onChange={onChange}
+            onKeyDown={onKeyDown}
+            inputRef={textInput}
+            placeholder="Enter Query..."
+          />
+        </Box>
+        <Box sx={{ p: 2, borderTop: 1 }}>
+          <Grid container spacing={2} justifyContent="flex-start">
+            <Grid item xs>
+              <Tooltip title="Clear Query Text">
+                <Button onClick={onClearClick} sx={{ padding: '11px', color: 'text.dark' }}>
+                  Clear
+                </Button>
+              </Tooltip>
+            </Grid>
+            <Grid item>
+              <Tooltip title="Run (Ctrl+Enter)">
+                <Button onClick={onRunClick} sx={{
+                  color: `queryStartButton.${queryState}.color`,
+                  bgcolor: `queryStartButton.${queryState}.backgroundColor`,
+                  padding: '11px'
+                }}>
+                  <PlayArrowIcon />
+                </Button>
+              </Tooltip>
+            </Grid>
+            <Grid item>
+              <Tooltip title="Stop">
+                <Button onClick={onStopClick} sx={{
+                  color: `queryStopButton.${queryState}.color`,
+                  bgcolor: `queryStopButton.${queryState}.backgroundColor`,
+                  padding: '11px'
+                }}>
+                  <StopIcon />
+                </Button>
+              </Tooltip>
+            </Grid>
+          </Grid>
+        </Box>
+      </Paper>
+    </Box>
+  );
+};

 QueryControl.propTypes = {
-  websocketUrl: PropTypes.string.isRequired,
-  theme: PropTypes.object.isRequired,
+  columns: PropTypes.array,
+  rows: PropTypes.array,
+  isRunning: PropTypes.bool,
+  queryText: PropTypes.string,
+  queryTextResetTime: PropTypes.number,
+  queryState: PropTypes.string,
+  subscribe: PropTypes.func,
+  unsubscribe: PropTypes.func,
 };

 export default QueryControl;
@@ -1,93 +0,0 @@
-import { useRef } from 'react';
-import Box from '@mui/material/Box';
-import Button from '@mui/material/Button';
-import Grid from '@mui/material/Grid';
-import Paper from '@mui/material/Paper';
-import PlayArrowIcon from '@mui/icons-material/PlayArrow';
-import PropTypes from 'prop-types';
-import StopIcon from '@mui/icons-material/Stop';
-import TextField from '@mui/material/TextField';
-import Tooltip from '@mui/material/Tooltip';
-
-const QueryInputForm = ({ isRunning, onStart, onStop }) => {
-  let textInput = useRef(null);
-
-  const onClearClick = () => {
-    textInput.current.value = '';
-  };
-
-  const onRunClick = () => {
-    onStart(textInput.current.value);
-  };
-
-  const onStopClick = () => {
-    onStop();
-  };
-
-  const onKeyDown = (e) => {
-    // Detect keyboard shortcuts.
-    //   Ctrl+Enter : Run
-    if (e.ctrlKey && e.key === 'Enter') {
-      onRunClick();
-    }
-  };
-
-  return (
-    <Box sx={{ p: 2 }}>
-      <Paper elevation={3} sx={{ color: 'primary.main' }}>
-        <Box sx={{ p: 2 }}>
-          <TextField
-            fullWidth
-            multiline
-            rows={10}
-            variant="outlined"
-            onKeyDown={onKeyDown}
-            inputRef={textInput}
-            placeholder="Enter Query..."
-          />
-        </Box>
-        <Box sx={{ p: 2, borderTop: 1 }}>
-          <Grid container spacing={2} justifyContent="flex-start">
-            <Grid item xs>
-              <Tooltip title="Clear Query Text">
-                <Button onClick={onClearClick} sx={{ padding: '11px', color: 'text.dark' }}>
-                  Clear
-                </Button>
-              </Tooltip>
-            </Grid>
-            <Grid item>
-              <Tooltip title="Run (Ctrl+Enter)">
-                <Button onClick={onRunClick} sx={{
-                  color: `query_start_button.${isRunning ? 'active' : 'inactive'}.color`,
-                  bgcolor: `query_start_button.${isRunning ? 'active' : 'inactive'}.background_color`,
-                  padding: '11px'
-                }}>
-                  <PlayArrowIcon />
-                </Button>
-              </Tooltip>
-            </Grid>
-            <Grid item>
-              <Tooltip title="Stop">
-                <Button onClick={onStopClick} sx={{
-                  color: `query_stop_button.${isRunning ? 'inactive' : 'active'}.color`,
-                  bgcolor: `query_stop_button.${isRunning ? 'inactive' : 'active'}.background_color`,
-                  padding: '11px'
-                }}>
-                  <StopIcon />
-                </Button>
-              </Tooltip>
-            </Grid>
-          </Grid>
-        </Box>
-      </Paper>
-    </Box>
-  );
-};
-
-QueryInputForm.propTypes = {
-  isRunning: PropTypes.bool.isRequired,
-  onStart: PropTypes.func.isRequired,
-  onStop: PropTypes.func.isRequired,
-};
-
-export default QueryInputForm;
@@ -0,0 +1,73 @@
+import { useState } from 'react';
+import PropTypes from 'prop-types';
+import { Box, Typography, Collapse } from '@mui/material';
+import Alert from '@mui/material/Alert';
+import AlertTitle from '@mui/material/AlertTitle';
+import { useTheme } from '@mui/material/styles';
+import { alpha } from '@mui/system';
+
+const QueryStatusView = ({ queryState, details, error }) => {
+  const theme = useTheme();
+  const [open, setOpen] = useState(false);
+
+  let header = queryState;
+  let headerContent = null;
+  let detailElements = [];
+  let severity = queryState == "Subscribed" ? "success" : "info";
+  let cursor = details == null ? "default" : "pointer";
+  if (error != null) {
+    severity = "error";
+    cursor = "pointer";
+    header = `Error Code: ${error.code}`;
+    headerContent = error.message;
+    detailElements.push(<Typography variant="body1">{error.data.error_message}</Typography>);
+    detailElements.push(<Typography variant="subtitle1">Stack Trace:</Typography>);
+    (error.data.stack_trace ?? []).forEach(line => detailElements.push(
+      <Typography variant="body2" component="pre" sx={{ whiteSpace: 'pre-wrap' }}>
+        {line}
+      </Typography>));
+  } else if (details != null) {
+    detailElements.push(<Typography variant="body1">{details}</Typography>);
+  }
+  return (
+    <Box sx={{ margin: theme.spacing(2, 0, 0, 0) }}>
+      <Alert
+        severity={severity}
+        onClick={() => setOpen(!open)}
+        sx={{
+          margin: theme.spacing(2, 0, 0, 0),
+          cursor: cursor,
+        }}>
+        <AlertTitle>{header}</AlertTitle>
+        {headerContent}
+      </Alert>
+      {detailElements.length == 0
+        ? null
+        : <Collapse in={open}>
+          <Box
+            sx={{
+              backgroundColor: alpha(theme.palette[severity].light, 0.75),
+              color: theme.palette[severity].contrastText,
+              padding: theme.spacing(2),
+              marginTop: theme.spacing(0),
+              borderRadius: theme.shape.borderRadius,
+            }}
+          >
+            {detailElements}
+          </Box>
+        </Collapse>}
+    </Box>
+  );
+};
+
+QueryStatusView.propTypes = {
+  queryState: PropTypes.string,
+  details: PropTypes.string,
+  error: PropTypes.shape({
+    code: PropTypes.number,
+    message: PropTypes.string,
+    data: PropTypes.object,
+  })
+};
+
+export default QueryStatusView;
@@ -0,0 +1,27 @@
+import 'react-data-grid/lib/styles.css';
+import PropTypes from 'prop-types';
+import DataGrid from 'react-data-grid';
+import './DataGrid.css';
+
+function getGridColDef(col) {
+  return {
+    key: col.name,
+    name: col.name,
+  };
+}
+
+export default function ReactDataGrid({columns, rows}) {
+  return (
+    <DataGrid className={"data-grid"} rows={rows} columns={columns.map(getGridColDef)} />
+  )
+}
+
+ReactDataGrid.propTypes = {
+  columns: PropTypes.arrayOf(
+    PropTypes.shape({
+      name: PropTypes.string.isRequired,
+      type: PropTypes.string.isRequired,
+    })
+  ).isRequired,
+  rows: PropTypes.arrayOf(PropTypes.array).isRequired,
+};
@@ -0,0 +1,98 @@
+/* Sidebar.css */
+.app-container {
+  display: flex;
+  width: 100%;
+  overflow: hidden;
+  flex-direction: row;
+  justify-content: flex-start;
+  align-items: flex-start;
+}
+
+.sidebar {
+  display: flex;
+  flex-direction: row;
+  justify-content: flex-start;
+  align-items: flex-start;
+  background-color: transparent;
+  color: white;
+  overflow: hidden;
+  transition: width 0.3s ease;
+}
+
+.sidebar.open {
+  width: 100%;
+}
+
+.sidebar.closed {
+  width: 0%;
+}
+
+.sidebar-toggle-button {
+  border: 0px;
+  border-radius: 50%;
+  box-shadow: rgba(9, 30, 66, 0.08) 0px 0px 0px 1px, rgba(9, 30, 66, 0.08) 0px 2px 4px 1px;
+  color: var(--primary-color);
+  cursor: pointer;
+  height: 24px;
+  opacity: 1;
+  outline: 0px;
+  padding: 0px;
+  background-color: transparent;
+  transition: background-color 100ms linear 0s, color 100ms linear 0s, opacity 300ms cubic-bezier(0.2, 0, 0, 1) 0s, transform 300ms cubic-bezier(0.2, 0, 0, 1) 0s;
+  width: 20%;
+}
+
+.sidebar-toggle-button:hover {
+  transition: background-color 100ms linear 0s, color 100ms linear 0s, opacity 300ms cubic-bezier(0.2, 0, 0, 1) 0s, transform 300ms cubic-bezier(0.2, 0, 0, 1) 0s;
+}
+
+.main-content.open {
+  width: 165px;
+  transition: width 0.3s ease;
+  background-color: transparent;
+}
+
+.main-content.closed {
+  width: 0%;
+  /* min-width: 25px; */
+  transition: width 0.3s ease;
+  background-color: transparent;
+}
+
+.sidebar-menu-item {
+  width: 100%;
+  box-sizing: border-box;
+}
+
+.sidebar-menu-item.open {
+  width: 100%;
+  /* justify-content: flex-start; */
+  transition: width 0.3s ease;
+}
+
+.sidebar-menu-item.closed {
+  width: 0%;
+  min-width: 0px;
+  /* justify-content: flex-start; */
+  transition: width 0.3s ease;
+}
+
+.sidebar-menu-item-content.open {
+  width: 100%;
+  /* justify-content: flex-start; */
+  transition: width 0.3s ease;
+}
+
+.sidebar-menu-item-content.closed {
+  width: 0px;
+  /* justify-content: flex-start; */
+  transition: width 0.3s ease;
+}
+
+/* Ensure span can shrink */
+.button-text {
+  display: block;    /* Make span a block element */
+  width: 100%;       /* Ensure it takes the full width of its parent */
+  overflow: hidden;  /* Hide any overflow content */
+  white-space: nowrap; /* Prevent text wrapping */
+}
@@ -0,0 +1,109 @@
+import { useState, useEffect } from 'react';
+import Button from '@mui/material/Button';
+import { useTheme } from '@mui/material/styles';
+import PropTypes from 'prop-types';
+import Tooltip from '@mui/material/Tooltip';
+import ChevronLeftRounded from "@mui/icons-material/ChevronLeftRounded"
+import ChevronRightRounded from '@mui/icons-material/ChevronRightRounded';
+import FileCopyIcon from '@mui/icons-material/FileCopy';
+import './SideBar.css';
+
+const Sidebar = ({ onSelection, onOpened, onClosed }) => {
+  const theme = useTheme();
+  const [isOpen, setIsOpen] = useState(false);
+  const [cannedQueries, setCannedQueries] = useState([]);
+  const Icon = isOpen ? ChevronLeftRounded : ChevronRightRounded;
+
+  const toggleIsOpen = () => {
+    if (isOpen) {
+      setIsOpen(false);
+      if (onClosed != null) {
+        onClosed();
+      }
+    } else {
+      setIsOpen(true);
+      if (onOpened != null) {
+        onOpened();
+      }
+    }
+  };
+
+  const onMenuItemClick = (event, cannedQuery) => {
+    console.info(`menu item clicked: ${cannedQuery.name}`);
+    onSelection(cannedQuery);
+  };
+
+  // const onMenuItemClick = async (event, cannedQuery) => {
+  //   try {
+  //     await navigator.clipboard.writeText(cannedQuery.query.join("\n"));
+  //     alert(`Copied ${cannedQuery.name} to clipboard`);
+  //   } catch (err) {
+  //     console.error('Failed to copy query text: ', err);
+  //   }
+  // };
+
+  useEffect(() => {
+    const fetchCannedQueries = async () => {
+      try {
+        const response = await fetch('api/cannedqueries');
+        const data = await response.json();
+        setCannedQueries(data);
+      } catch (error) {
+        console.error('Error fetching WebSocket config:', error);
+      }
+    }
+    fetchCannedQueries();
+  }, []);
+
+  const mainContentClass = `main-content ${isOpen ? 'open' : 'closed'}`;
+  const menuItemClass = `sidebar-menu-item ${isOpen ? 'open' : 'closed'}`;
+  const menuItemContentClass = `sidebar-menu-item-content ${isOpen ? 'open' : 'closed'}`;
+  const toggleButtonTitle = `${isOpen ? 'Hide' : 'Show'} Common Queries`;
+
+  return (
+    <div
+      className={"sidebar"}
+      style={{
+        display: 'flex',
+        flexDirection: 'row',
+        justifyContent: 'flexStart',
+        alignItems: 'flexStart',
+        margin: theme.spacing(2, 0, 0, 0)
+      }}
+    >
+      <div className={mainContentClass}>
+        {cannedQueries.map(cq =>
+          <Button key={cq.name} className={menuItemClass} onClick={(event) => onMenuItemClick(event, cq)}>
+            <div className={menuItemContentClass}>
+              <span className='button-text'>{cq.name}</span>
+            </div>
+          </Button>)
+        }
+        {/* {cannedQueries.map(cq =>
+          <Button
+            variant="contained"
+            color="primary"
+            startIcon={<FileCopyIcon />}
+            onClick={onMenuItemClick}
+          >
+            <div className={menuItemContentClass}>
+              <span className='button-text'>{cq.name}</span>
+            </div>
+          </Button>)} */}
+      </div>
+      <Tooltip title={toggleButtonTitle}>
+        <Button onClick={toggleIsOpen}>
+          <Icon className="w-5 h-5" />
+        </Button>
+      </Tooltip>
+    </div>
+  );
+};
+
+export default Sidebar;
+
+Sidebar.propTypes = {
+  onSelection: PropTypes.func,
+  onOpened: PropTypes.func,
+  onClosed: PropTypes.func,
+};
@@ -42,32 +42,56 @@ const getDarkPalette = () => {
      dark: '#ffffff12',
      contrastText: '#c8d4f7',
    },
+    action: {
+      hoverOpacity: 0.08
+    },
    text: {
      primary: '#eaf8f8',
      dark: '#eaf8f8a0'
    },
-    query_start_button: {
-      active: {
+    queryStartButton: {
+      Subscribed: {
        color: '#33baf9',
-        background_color: '#ffffff40',
+        backgroundColor: '#ffffff40',
      },
-      inactive: {
+      Unsubscribed: {
        color: '#ffffff40',
-        background_color: '#ffffff00',
+        backgroundColor: '#ffffff00',
      }
    },
+    queryStopButton: {
+      Subscribed: {
+        color: '#ff4444a0',
+        backgroundColor: '#ffffff40',
+      },
+      Unsubscribed: {
+        color: '#ff444450',
+        backgroundColor: '#ffffff00',
+      }
+    },
+    error: {
+      light: '#e57373',
+      main: '#f44336',
+      dark: '#d32f2f',
+      contrastText: '#fff',
+    },
    warning: {
-      main: '#c4823f'
+      light: '#ffb74d',
+      main: '#ff9800',
+      dark: '#f57c00',
+      contrastText: '#000',
    },
-    query_stop_button: {
-      active: {
-        color: '#ff4444a0',
-        background_color: '#ffffff40',
-      },
-      inactive: {
-        color: '#ff444450',
-        background_color: '#ffffff00',
-      }
+    info: {
+      light: '#64b5f6',
+      main: '#2196f3',
+      dark: '#303030',
+      contrastText: '#fff',
+    },
+    success: {
+      light: '#81c784',
+      main: '#4caf50',
+      dark: '#388e3c',
+      contrastText: '#000',
    },
  };
 };
@@ -3,7 +3,5 @@ import ReactDOM from 'react-dom/client';
 import App from './App';

 ReactDOM.createRoot(document.getElementById('root')).render(
-  <React.StrictMode>
-    <App />
-  </React.StrictMode>
+  <App />
 )
@@ -1,5 +1,5 @@
 import { fileURLToPath, URL } from 'node:url';
-import { defineConfig } from 'vite';
+import { defineConfig, loadEnv } from 'vite';
 import plugin from '@vitejs/plugin-react';
 import fs from 'fs';
 import path from 'path';
@@ -29,13 +29,12 @@ if (!fs.existsSync(certFilePath) || !fs.existsSync(keyFilePath)) {
  }
 }

-const target = env.ASPNETCORE_HTTPS_PORT
-  ? `https://localhost:${env.ASPNETCORE_HTTPS_PORT}`
-  : env.ASPNETCORE_URLS
-    ? env.ASPNETCORE_URLS.split(';')[0]
-    : 'https://localhost:5001';
-
 export default defineConfig(({ $$command, mode, $$isSsrBuild, $$isPreview }) => {
+  const target = env.ASPNETCORE_HTTPS_PORT
+    ? `https://localhost:${env.ASPNETCORE_HTTPS_PORT}`
+    : env.ASPNETCORE_URLS
+      ? env.ASPNETCORE_URLS.split(';')[0]
+      : 'https://localhost:8443';
  return {
    plugins: [plugin()],
    resolve: {
@@ -29,7 +29,12 @@
    <ProjectReference Include="..\..\client\spark_websocket\spark_websocket.csproj" />
  </ItemGroup>

-  <!-- <ItemGroup>
-    <Content Update="wwwroot\alt-index.html" CopyToPublishDirectory="Never" />
-  </ItemGroup> -->
+  <ItemGroup>
+    <ConfigurationProfile Include="appsettings.Development.json" />
+    <ConfigurationProfile Include="appsettings.Production.json" />
+    <ConfigurationProfile Include="appsettings.ProductionAws.json" />
+    <ConfigurationProfile Include="appsettings.Staging.json" />
+    <ConfigurationProfile Include="appsettings.json" />
+  </ItemGroup>
+
 </Project>
@@ -0,0 +1,5 @@
+namespace Bts.DataStreamWebApp.Server;
+
+public class CannedQueriesConfig {
+   public List<CannedQuery>? Items { get; set; }
+}
@@ -0,0 +1,6 @@
+namespace Bts.DataStreamWebApp.Server;
+
+public class CannedQuery {
+   public string? Name { get; set; }
+   public List<string>? Query { get; set; }
+}
@@ -1,4 +0,0 @@
-{
-  "content_root_path": "build/Bts.DataStreamWebApp.Client/production",
-  "web_root_path": "dist"
-}
@@ -0,0 +1,18 @@
+namespace Bts.DataStreamWebApp.Server;
+
+using Microsoft.AspNetCore.Mvc;
+
+[ApiController]
+[Route("api/[controller]")]
+public class CannedQueriesController : ControllerBase {
+   readonly IConfiguration config;
+
+   public CannedQueriesController(IConfiguration config) =>
+      this.config = config;
+
+   [HttpGet(Name = "GetCannedQueries")]
+   public IEnumerable<CannedQuery>? Get() {
+      var cannedQueries = config.GetSection("CannedQueries").Get<List<CannedQuery>>()?.ToArray() ?? Array.Empty<CannedQuery>();
+      return cannedQueries;
+   }
+}
@@ -0,0 +1,22 @@
+// using Bts.DataStreamWebApp.Server;
+
+// using Microsoft.AspNetCore.Mvc;
+
+// [ApiController]
+// [Route("api/[controller]")]
+// public class ConfigController : ControllerBase {
+//    private readonly IConfiguration _configuration;
+
+//    public ConfigController(IConfiguration configuration) {
+//       _configuration = configuration;
+//    }
+
+//    [HttpGet("websocket")]
+//    public ActionResult<DataStreamWebSocketConfig> GetWebSocketConfig() {
+//       var endpoint = $"{_configuration["Kestrel:Endpoints:Http:Url"]}/ws";
+//       var webSocketConfig = new DataStreamWebSocketConfig {
+//          Endpoint = endpoint // or Https based on your setup
+//       };
+//       return Ok(webSocketConfig);
+//    }
+// }
@@ -7,6 +7,7 @@ using bts.websocket_core.json_rpc;
 using Bts.DataStreamCore;
 using Bts.DataStreamWebApp.Server.Messages;

+using System;
 using System.Net.WebSockets;
 using System.Text;
 using System.Text.Json;
@@ -96,7 +97,10 @@ class DataStreamSubscription : IDisposable, IHostedService {
         IncludeCurrent = true,
         UseObjectRows = true,
      });
-      var res = JsonRpcRequest.CreateNotification(method, dp);
+      var res = JsonRpcRequest.CreateNotification(method, new DataStreamSubscriptionUpdate() {
+         SubscriptionId = Id,
+         Table = dp
+      });
      var str = serialize(res, serializerOpts);
      if (str is null) {
         logger.LogError("Unsubscribing from stream early because we failed to serialize an outgoing DataPacket");
@@ -109,8 +113,12 @@ class DataStreamSubscription : IDisposable, IHostedService {
         dp.schema.columnNames.Length,
         str.Length);
      var bytes = Encoding.UTF8.GetBytes(str);
-      await webSocket.SendAsync(bytes, WebSocketMessageType.Text, true, CancellationToken.None);
-      LastFired = dp;
+      try {
+         await webSocket.SendAsync(bytes, WebSocketMessageType.Text, true, CancellationToken.None);
+         LastFired = dp;
+      } catch (WebSocketException ex) {
+         logger.LogError($"sendDataPacket failed on webSocket.SendAsync: {ex}");
+      }
   }

   string? serialize<T>(T message, JsonSerializerOptions? opts) {
@@ -0,0 +1,13 @@
+namespace Bts.DataStreamWebApp.Server;
+
+using BCG;
+
+using System.Text.Json.Serialization;
+
+public class DataStreamSubscriptionUpdate {
+   [JsonPropertyName("subscription_id")]
+   public string? SubscriptionId { get; set; }
+
+   [JsonPropertyName("table")]
+   public DataPacket? Table { get; set; }
+}
@@ -0,0 +1,5 @@
+namespace Bts.DataStreamWebApp.Server;
+
+public class DataStreamWebSocketConfig {
+   public string? Endpoint { get; set; }
+}
@@ -58,40 +58,67 @@ public class DataStreamWebSocketServer : IDisposable, IAsyncDisposable {
   async Task onSubscribe(JsonRpcRequest<JsonNode> req, CancellationToken cancellationToken) {
      ArgumentNullException.ThrowIfNull(req.Id);
      ArgumentException.ThrowIfNullOrEmpty(req.Method);
-      if (!SubscribeRequest.ParseQuery(req, out var vs, out var subId, out var errMsg, out var errCode)) {
-         logger.LogError("Invalid 'subscribe' request: {Reason}", errMsg);
-         var rpcError = new JsonRpcError<JsonNode>() {
-            Code = (long)errCode,
-            Message = $"{errCode}: {errMsg}",
-         };
+      if (!SubscribeRequest.Parse(req, out var subReq, out var parseErr, out var errCode)) {
+         logger.LogError("Invalid 'subscribe' request: {Reason}", parseErr);
+         var rpcError = JsonRpcResponse.CreateError<SubscriptionError>(
+            req.Id, new() { Code = (long)errCode, Message = "Failed to parse subscription request", Data = parseErr });
+         await send(rpcError, cancellationToken);
+         return;
+      }
+      var subId = subReq.SubscriptionId ?? throw new InvalidOperationException("SubscriptionId is null (parse false)");
+      disposeSubscription(subId);
+      if (!subReq.ParseQuery(out var vs, out parseErr, out errCode)) {
+         logger.LogError("Invalid 'subscribe' request: {Reason}", parseErr);
+         var rpcError = JsonRpcResponse.CreateError<SubscriptionError>(
+            req.Id, new() { Code = (long)errCode, Message = "Failed to parse subscription request", Data = parseErr });
         await send(rpcError, cancellationToken);
         return;
      }
      if (subscriptions.TryGetValue(subId, out var oldSub)) {
+         logger.LogInformation("Disposing existing subscription with id {SubscriptionId}", subId);
         oldSub.Dispose();
      }
      var sub = new DataStreamSubscription(subId, req.Id.Value, vs, webSocket, logger);
      subscriptions[subId] = sub;
      await sendSubscriptionStatus(req.Id.Value, subId, req.Method, SubscriptionStatus.Subscribed, cancellationToken);
+      vs.ErrorChanged += OnStreamError;
      await sub.StartAsync(cancellationToken);
+
+      async void OnStreamError(Exception exception) {
+         logger.LogError("Error in ViewStream (disposing subscription): {Exception}", exception);
+         subscriptions.Remove(sub.Id);
+         sub.Dispose();
+         var error = new SubscriptionError() {
+            ErrorMessage = "Error in ViewStream (disposing subscription)"
+         };
+         if (exception is not null) {
+            error.ErrorMessage += ". " + exception.Message;
+            error.StackTrace = exception.StackTrace?.Split("\n")?.ToList();
+         }
+         var rpcError = JsonRpcResponse.CreateError<SubscriptionError>(
+            req.Id, new() { Code = (long)ErrorCode.ViewStreamError, Message = "Error in ViewStream (disposing subscription)", Data = error });
+         await send(rpcError, cancellationToken);
+      }
   }

   async Task onUnsubscribe(JsonRpcRequest<UnsubscribeRequest> req, CancellationToken cancellationToken) {
      ArgumentNullException.ThrowIfNull(req.Id);
      ArgumentException.ThrowIfNullOrEmpty(req.Method);
      var subId = req.Params?.SubscriptionId ?? string.Empty;
-      if (!subscriptions.TryGetValue(subId, out var sub)) {
-         logger.LogInformation("Ignoring '{Method}' request because no subscription was found with id {Id}.", req.Method, subId);
-         return;
+      if (!disposeSubscription(subId)) {
+         logger.LogInformation("Unable to dispose subscription. Subscription with id {Id} not found.", subId);
      }
-      logger.LogInformation("Disposing subscription");
-      sub.Dispose();
-      subscriptions.Remove(subId);
-      await sendSubscriptionStatus(req.Id.Value, subId, req.Method, SubscriptionStatus.Unsubscribed, cancellationToken);
+      await sendSubscriptionStatus(req.Id.Value, subId, "unsubscribe", SubscriptionStatus.Unsubscribed, cancellationToken);
   }

-   async Task unsubscribe(JsonRpcId requestId, string subscriptionId, CancellationToken cancellationToken) {
-      await sendSubscriptionStatus(requestId, subscriptionId, "unsubscribe", SubscriptionStatus.Unsubscribed, cancellationToken);
+   bool disposeSubscription(string subId) {
+      if (!subscriptions.TryGetValue(subId, out var sub)) {
+         return false;
+      }
+      logger.LogInformation("Disposing subscription with id {Id}", subId);
+      sub.Dispose();
+      subscriptions.Remove(subId);
+      return true;
   }

   Task sendSubscriptionStatus(JsonRpcId reqId, string subId, string method, SubscriptionStatus status, CancellationToken cancellationToken) {
@@ -0,0 +1,11 @@
+FROM bluetradingsystems/linux_dotnet7
+
+ARG SOURCE_DIR="build/Release/bin/net7.0/linux-x64/publish"
+ARG WEB_ROOT="build/Bts.DataStreamWebApp.Client/production/dist"
+ARG CONF_FILE="src/DataStreamSuite/Bts.DataStreamWebApp.Server/spark.conf"
+
+RUN dotnet dev-certs https --trust
+COPY --chown=bts:bts "${SOURCE_DIR}" "bin"
+COPY --chown=bts:bts "${WEB_ROOT}" "bin/wwwroot"
+COPY --chown=bts:bts "${CONF_FILE}" ".local/share/bts/Bts.DataStreamWebApp.Server/conf/spark.conf"
+WORKDIR "/home/bts/bin"
@@ -0,0 +1,5 @@
+namespace Bts.DataStreamWebApp.Server;
+
+public class CannedQueriesResponse {
+   public List<CannedQuery>? Items { get; set; }
+}
@@ -3,4 +3,5 @@ namespace Bts.DataStreamWebApp.Server.Messages;
 enum ErrorCode {
   None = 0,
   InvalidSubscriptionRequest,
+   ViewStreamError,
 }
@@ -18,47 +18,52 @@ public class SubscribeRequest {
   [JsonPropertyName("subscription_id")]
   public string? SubscriptionId { get; set; }

-   internal static bool ParseQuery(JsonRpcRequest<JsonNode> req, [NotNullWhen(true)] out ViewStream? stream, [NotNullWhen(true)] out string? subscriptionId, [NotNullWhen(false)] out string? errorMessage, out ErrorCode errorCode) {
-      stream = null;
-      subscriptionId = null;
+   internal static bool Parse(JsonRpcRequest<JsonNode> req, [NotNullWhen(true)]out SubscribeRequest? subscription, [NotNullWhen(false)]out SubscriptionError? error, out ErrorCode errorCode) {
+      subscription = null;
      errorCode = ErrorCode.None;
      var paramsJsonObj = req.Params?.AsObject();
+      error = null;
      if (paramsJsonObj is null) {
         errorCode = ErrorCode.InvalidSubscriptionRequest;
-         errorMessage = "'params' object not found";
+         error = new() { ErrorMessage = "'params' object not found" };
         return false;
      }
-      var subReq = paramsJsonObj.Deserialize<SubscribeRequest>();
-      if (subReq is null) {
+      subscription = paramsJsonObj.Deserialize<SubscribeRequest>();
+      if (subscription is null) {
         errorCode = ErrorCode.InvalidSubscriptionRequest;
-         errorMessage = "'params' object invalid";
+         error = new() { ErrorMessage = "'params' object invalid" };
         return false;
      }
-      subscriptionId = subReq.SubscriptionId ?? string.Empty;
-      var query = subReq.Query;
-      if (query is null) {
+      return true;
+   }
+
+   internal bool ParseQuery([NotNullWhen(true)] out ViewStream? stream, [NotNullWhen(false)] out SubscriptionError? error, out ErrorCode errorCode) {
+      stream = null;
+      errorCode = ErrorCode.None;
+      error = null;
+      if (Query is null) {
         errorCode = ErrorCode.InvalidSubscriptionRequest;
-         errorMessage = "'params.query' field not found (expected a string)";
+         error = new() { ErrorMessage = "'params.query' field not found (expected a string)" };
         return false;
      }
-      errorMessage = null;
-      var exp = DataStreamExpressionsInterpreter.Evaluate(query);
+      var exp = DataStreamExpressionsInterpreter.Evaluate(Query);
      if (exp is null) {
         errorCode = ErrorCode.InvalidSubscriptionRequest;
-         errorMessage = "'params.query' expression has invalid syntax";
+         error = new() { ErrorMessage = "'params.query' expression has invalid syntax" };
         return false;
      }
-      string? exceptionMsg = null;
+      Exception? exception = null;
      try {
         stream = exp.getView(new Dictionary<string, StatementExpression>(0));
      } catch (Exception ex) {
-         exceptionMsg = ex.ToString();
+         exception = ex;
      }
      if (stream is null) {
         errorCode = ErrorCode.InvalidSubscriptionRequest;
-         errorMessage = "Failed to create ViewStream from 'params.query' expression (check for typos in arguments/typenames)";
-         if (exceptionMsg is not null) {
-            errorMessage += ". Error" + exceptionMsg;
+         error = new() { ErrorMessage = "Failed to create ViewStream from 'params.query' expression (check for typos in arguments/typenames)" };
+         if (exception is not null) {
+            error.ErrorMessage += ". " + exception.Message;
+            error.StackTrace = exception.StackTrace?.Split("\n")?.ToList();
         }
         return false;
      }
@@ -0,0 +1,11 @@
+namespace Bts.DataStreamWebApp.Server.Messages;
+
+using System.Text.Json.Serialization;
+
+public class SubscriptionError {
+   [JsonPropertyName("error_message")]
+   public string? ErrorMessage { get; set; }
+
+   [JsonPropertyName("stack_trace")]
+   public List<string>? StackTrace { get; set; }
+}
@@ -1,59 +1,82 @@
-using Bts.DataStreamCore.Services;
-using Bts.PubSub.Services;
-
-using Spark.Api.Services;
-
-using System.Reactive.Linq;
-
-var builder = WebApplication.CreateBuilder(new WebApplicationOptions {
-   Args = args,
-   ApplicationName = "Bts.DataStreamWebApp.Server",
-});
-ConfigureApp(builder.Configuration);
-ConfigureServices(builder.Services, builder.Configuration);
-
-var app = builder.Build();
-if (app.Environment.IsDevelopment()) {
-   app.UseDirectoryBrowser();
-}
-var wsOpts = new WebSocketOptions {
-   KeepAliveInterval = TimeSpan.FromMinutes(2)
-};
-app.UseWebSockets(wsOpts);
-app.UseDefaultFiles();
-app.UseStaticFiles();
-if (app.Environment.IsDevelopment()) {
-   app.UseSwagger();
-   app.UseSwaggerUI();
-}
-app.UseHttpsRedirection();
-app.UseAuthorization();
-app.MapControllers();
-try {
-   Console.Write($"Starting Application");
-   app.Run();
-   Console.Write($"Application exited gracefully.");
-} catch (Exception ex) {
-   Console.Error.Write($"Application crashed: {ex}");
-   return 1;
-}
-return 0;
-
-static void ConfigureApp(ConfigurationManager configBuilder) {
-   configBuilder.AddCommandLine(Environment.GetCommandLineArgs());
-   var cfgRoot = configBuilder.GetSection("config");
-   if (cfgRoot?.Value is not null) {
-      configBuilder.AddJsonFile(cfgRoot.Value);
-   }
-}
-
-static void ConfigureServices(IServiceCollection services, IConfiguration config) {
-   services.AddDirectoryBrowser();
-   services.AddControllers();
-   services.AddEndpointsApiExplorer();
-   services.AddSwaggerGen();
-   services.AddMasterWorker(config.GetSection("MasterWorker"));
-   services.AddViewStreamDatabase(config.GetSection("ViewStreamDatabase"));
-   services.AddPubSubClient(config.GetSection("PubSub"));
-   services.AddSparkd(config.GetSection("Sparkd"));
-}
+using Bts.DataStreamCore.Services;
+using Bts.PubSub.Services;
+
+using Spark.Api.Services;
+using Spark.Core.Services;
+
+using System.Reactive.Linq;
+using System.Reflection;
+
+var builder = WebApplication.CreateBuilder(new WebApplicationOptions {
+   Args = args,
+   ApplicationName = "Bts.DataStreamWebApp.Server",
+});
+ConfigureApp(builder.Configuration, builder.Environment.EnvironmentName);
+ConfigureServices(builder.Services, builder.Configuration);
+
+var app = builder.Build();
+var wsOpts = new WebSocketOptions {
+   KeepAliveInterval = TimeSpan.FromMinutes(2)
+};
+if (app.Environment.IsDevelopment()) {
+   app.UseDirectoryBrowser("/browse");
+   app.UseSwagger();
+   app.UseSwaggerUI();
+}
+app.UseHttpsRedirection();
+app.UseWebSockets(wsOpts);
+app.UseDefaultFiles();
+app.UseStaticFiles();
+app.UseAuthorization();
+app.MapControllers();
+try {
+   Console.Write($"Starting Application");
+   app.Run();
+   Console.Write($"Application exited gracefully.");
+} catch (Exception ex) {
+   Console.Error.Write($"Application crashed: {ex}");
+   return 1;
+}
+return 0;
+
+static void ConfigureApp(ConfigurationManager configBuilder, string environment) {
+   // We prefix appsettings with the exe name because multiple exes will usually be in the build dir
+   // (this way, each exe can still load its own appsettings files from the current directory).
+   var exeName = Assembly.GetCallingAssembly().GetName().Name;
+   AddJsonFile($"{exeName}.appsettings.json");
+   AddJsonFile($"{exeName}.appsettings.{environment}.json");
+
+   var cfg = configBuilder.GetSection("config")?.Value;
+   if (!SetBasePath(cfg)) {
+      AddJsonFile(cfg);
+   }
+
+   bool SetBasePath(string? path) {
+      if (!Directory.Exists(path)) {
+         return false;
+      }
+      configBuilder.SetBasePath(path);
+      return true;
+   }
+
+   bool AddJsonFile(string? path) {
+      if (!File.Exists(path)) {
+         return false;
+      }
+      configBuilder.AddJsonFile(path);
+      return true;
+   }
+}
+
+static void ConfigureServices(IServiceCollection services, IConfiguration config) {
+   services.AddDirectoryBrowser()
+      .AddControllers().Services
+      .AddEndpointsApiExplorer()
+      .AddSwaggerGen();
+   services.AddSpaStaticFiles();
+   services.AddSpark(config.GetSection("Spark"))
+      .AddSparkApi(config.GetSection("SparkApi"))
+      .AddMasterWorker(config.GetSection("MasterWorker"))
+      .AddViewStreamDatabase()
+      .AddPubSubClient(config.GetSection("PubSub"));
+}
@@ -6,8 +6,9 @@
      "dotnetRunMessages": true,
      "launchBrowser": true,
      "launchUrl": "swagger",
-      "applicationUrl": "http://localhost:5000",
+      "applicationUrl": "http://localhost:8080",
      "environmentVariables": {
+        "ASPNETCORE_HTTP_PORT": "8080",
        "ASPNETCORE_ENVIRONMENT": "Development",
        "ASPNETCORE_HOSTINGSTARTUPASSEMBLIES": "Microsoft.AspNetCore.SpaProxy",
      }
@@ -16,8 +17,10 @@
      "commandName": "Project",
      "dotnetRunMessages": true,
      "launchBrowser": true,
-      "applicationUrl": "http://localhost:5000;https://localhost:5001",
+      "applicationUrl": "http://localhost:8080;https://localhost:8443",
      "environmentVariables": {
+        "ASPNETCORE_HTTP_PORT": "8080",
+        "ASPNETCORE_HTTPS_PORT": "8443",
        "ASPNETCORE_ENVIRONMENT": "Development",
        "ASPNETCORE_HOSTINGSTARTUPASSEMBLIES": "Microsoft.AspNetCore.SpaProxy",
      }
@@ -0,0 +1,62 @@
+{
+  "Kestrel": {
+    "Endpoints": {
+      "Http": {
+        "Url": "http://localhost:8080"
+      },
+      "Https": {
+        "Url": "https://localhost:8443"
+      }
+    }
+  },
+  "PubSub": {
+    "Routing": {
+      "PublisherRoutes": [
+        {
+          "Name": "default",
+          "ServerAddress": "bts-cloud://localhost:22019",
+          "TagRegExp": ".*"
+        }
+      ],
+      "SubscriberRoutes": [
+        {
+          "Name": "default",
+          "ServerAddress": "bts-cloud://localhost:22019",
+          "TagRegExp": ".*"
+        }
+      ]
+    }
+  },
+  "CannedQueries": [
+    {
+      "Name": "BBO",
+      "Query": [
+        "bbo"
+      ]
+    },
+    {
+      "Name": "Trades",
+      "Query": [
+        "timesales"
+      ]
+    },
+    {
+      "Name": "Instruments",
+      "Query": [
+        "instruments"
+      ]
+    },
+    {
+      "Name": "Books",
+      "Query": [
+        "books"
+      ]
+    },
+    {
+      "Name": "Benzinga News",
+      "Query": [
+        "benzinga_news"
+      ]
+    }
+  ]
+}
@@ -0,0 +1,8 @@
+{
+  "Spark": {
+    "ConfigServer": "http://sparkd:8000/config"
+  },
+  "SparkApi": {
+    "IsEnabled": true
+  }
+}
@@ -5,10 +5,10 @@
  "ViewStreamDatabase": {
    "Interval": "00:00:00.1"
  },
-  "Sparkd": {
-    "Config": {
-      "ConfigServer": "http://localhost:8000/config"
-    },
+  "Spark": {
+    "ConfigServer": "http://172.30.2.181:8000/config"
+  },
+  "SparkApi": {
    "Credentials": {
      "User": "bts",
      "Password": ""
@@ -18,38 +18,25 @@
    "EnablePosition": true
  },
  "PubSub": {
-    "MillisecondsBetweenSubInfoUpdates": 1000,
-    "MillisecondsBetweenTimeSliceUpdates": 5000,
-    "MillisecondsBetweenNoBufferSpaceRetries": 100,
-    "RetriesOnNoBufferSpace": 1,
-    "DoItAllSecondsBetweenLogs": 0,
    "HeartbeatInterval": 15000,
    "MissingHeartbeatsBeforeGc": 50,
-    "UseManagedSockets": true,
-    "StateUpdatePeriod": 5000,
-    "StatisticsUpdatePeriod": 5000,
-    "DisposedConnectionCheckPeriod": 1000,
-    "MessageProcessingWarningThreshold": 1000,
-    "ServerPort": 12345,
+    "ServerPort": 22019,
    "Routing": {
      "PublisherRoutes": [
        {
          "Name": "default",
-          "ServerAddress": "bts-cloud://127.0.0.1:12345",
+          "ServerAddress": "bts-cloud://172.30.2.181:22019",
          "TagRegExp": ".*"
        }
      ],
      "SubscriberRoutes": [
        {
          "Name": "default",
-          "ServerAddress": "bts-cloud://127.0.0.1:12345",
+          "ServerAddress": "bts-cloud://172.30.2.181:22019",
          "TagRegExp": ".*"
        }
      ]
    },
-    "MaxPacketSize": 500000000,
-    "BlueCommMustDie": true,
-    "CrushableTags": [],
    "TagTypeToAssemblyMap": {
      "Entries": [
        {
@@ -58,26 +45,6 @@
        }
      ]
    },
-    "ServerQueue": "PubSubQueue",
-    "PortRange": {
-      "First": 30400,
-      "Last": 31700
-    },
-    "BindToRandomPort": false,
-    "DebugTags": "^NOTAGS$",
-    "LatencyReportThresholdMs": 1500,
-    "UseSubstations": false,
-    "UseDynamicClients": false,
-    "IsSubstation": false,
-    "InternallyPublishSubInfo": true,
-    "UseManagedClientsInNativeCode": false,
-    "CompressedTags": "^NOTAGS$",
-    "TagsToReportCompressionRatio": "^NOTAGS$",
-    "DisableMsmq": false,
-    "UsePrepublishedServerState": false,
-    "IpTranslation": null,
-    "GcActivationThreshold": 0,
-    "WaitPeriodAfterGc": 1000,
    "TagHandlerFactory": {
      "Handlers": [
        {
@@ -98,9 +65,51 @@
        {
          "TagType": ".*",
          "Assembly": "DataStreamCore",
-         "Class": "bts.PubsubServerGenericDataHandler"
+          "Class": "bts.PubsubServerGenericDataHandler"
        }
      ]
    }
-  }
+  },
+  "Kestrel": {
+    "Endpoints": {
+      "Http": {
+        "Url": "http://172.30.2.181:8080"
+      },
+      "Https": {
+        "Url": "https://172.30.2.181:8443"
+      }
+    }
+  },
+  "CannedQueries": [
+    {
+      "Name": "BBO",
+      "Query": [
+        "bbo"
+      ]
+    },
+    {
+      "Name": "Trades",
+      "Query": [
+        "timesales"
+      ]
+    },
+    {
+      "Name": "Instruments",
+      "Query": [
+        "instruments"
+      ]
+    },
+    {
+      "Name": "Books",
+      "Query": [
+        "books"
+      ]
+    },
+    {
+      "Name": "Benzinga News",
+      "Query": [
+        "benzinga_news"
+      ]
+    }
+  ]
 }
@@ -0,0 +1,34 @@
+{
+  "CannedQueries": [
+    {
+      "Name": "BBO",
+      "Query": [
+        "bbo"
+      ]
+    },
+    {
+      "Name": "Trades",
+      "Query": [
+        "timesales"
+      ]
+    },
+    {
+      "Name": "Instruments",
+      "Query": [
+        "instruments"
+      ]
+    },
+    {
+      "Name": "Books",
+      "Query": [
+        "books"
+      ]
+    },
+    {
+      "Name": "Benzinga News",
+      "Query": [
+        "benzinga_news"
+      ]
+    }
+  ]
+}
@@ -5,5 +5,81 @@
      "Microsoft.AspNetCore": "Warning"
    }
  },
-  "AllowedHosts": "*"
+  "AllowedHosts": "*",
+  "Kestrel": {
+    "Endpoints": {
+      "Http": {
+        "Url": "http://0.0.0.0:8080"
+      },
+      "Https": {
+        "Url": "https://0.0.0.0:8443"
+      }
+    }
+  },
+  "MasterWorker": {
+    "Interval": "00:00:00.1"
+  },
+  "ViewStreamDatabase": {
+    "Interval": "00:00:00.1"
+  },
+  "SparkApi": {
+    "IsEnabled": false,
+    "Credentials": {
+      "User": "bts",
+      "Password": ""
+    }
+  },
+  "PubSub": {
+    "HeartbeatInterval": 15000,
+    "MissingHeartbeatsBeforeGc": 50,
+    "ServerPort": 22019,
+    "Routing": {
+      "PublisherRoutes": [
+        {
+          "Name": "default",
+          "ServerAddress": "bts-cloud://pubsub:22019",
+          "TagRegExp": ".*"
+        }
+      ],
+      "SubscriberRoutes": [
+        {
+          "Name": "default",
+          "ServerAddress": "bts-cloud://pubsub:22019",
+          "TagRegExp": ".*"
+        }
+      ]
+    },
+    "TagTypeToAssemblyMap": {
+      "Entries": [
+        {
+          "HandlerClass": "BCG.PubSubServerTagHandler",
+          "Assembly": "DataStreamCore.dll"
+        }
+      ]
+    },
+    "TagHandlerFactory": {
+      "Handlers": [
+        {
+          "TagType": "DataStream",
+          "Assembly": "DataStreamCore",
+          "Class": "bts.PubsubServerStreamTagHandler"
+        },
+        {
+          "TagType": "System_Sub",
+          "Assembly": "PubSub",
+          "Class": "BCG.SystemSubscriptionMapPublisher"
+        },
+        {
+          "TagType": "System_Stats",
+          "Assembly": "PubSub",
+          "Class": "BCG.SystemStatsPublisher"
+        },
+        {
+          "TagType": ".*",
+          "Assembly": "DataStreamCore",
+         "Class": "bts.PubsubServerGenericDataHandler"
+        }
+      ]
+    }
+  }
 }
@@ -0,0 +1,15 @@
+Logging {
+  value {
+    console {
+      default_level: "Info"
+    }
+    syslog {
+      default_level: "Info"
+    }
+    file {
+      default_level: "Info"
+    }
+    initial_queue_size: 120001
+    max_queue_growth: 4
+  }
+}
@@ -0,0 +1,13 @@
+<Project Sdk="Microsoft.NET.Sdk">
+
+  <PropertyGroup>
+    <TargetFramework>net7.0</TargetFramework>
+    <ImplicitUsings>enable</ImplicitUsings>
+    <Nullable>enable</Nullable>
+  </PropertyGroup>
+
+  <ItemGroup>
+    <PackageReference Include="Microsoft.Extensions.Hosting" Version="7.0.1" />
+  </ItemGroup>
+
+</Project>
@@ -0,0 +1,59 @@
+namespace Bts.Extensions.Hosting;
+
+using Microsoft.Extensions.Configuration;
+using Microsoft.Extensions.Hosting;
+
+using System.Diagnostics;
+
+public static class HostBuilderExtensions {
+   public static IHostBuilder CreateBtsBuilder(string[] args) {
+      // Obtain calling assembly name here, rather than in the lambda where it is actually needed,
+      // because want the name of the assembly that called this function, not the assembly that will eventually invoke
+      // the lambda...that will be the Microsoft.Extensions.Hosting assembly.
+      // Get the name of the assembly that called this function. That will act as the "appsettings"
+      // json file prefix
+      var exeName = GetCallingAssemblyName();
+      return Host.CreateDefaultBuilder(args)
+         .ConfigureAppConfiguration((context, configBuilder) =>
+            ConfigureApp(configBuilder, context.HostingEnvironment.EnvironmentName, exeName));
+   }
+
+   public static void ConfigureApp(IConfigurationBuilder configBuilder, string? environment = null, string? prefix = null) {
+      var baseFileName = prefix is null ? "appsettings" : $"{prefix}.appsettings";
+      AddJsonFile($"{baseFileName}.json");
+      if (environment is not null) {
+         AddJsonFile($"{baseFileName}.{environment}.json");
+      }
+
+      var cfg = configBuilder.Build().GetSection("config")?.Value;
+      if (!SetBasePath(cfg)) {
+         AddJsonFile(cfg);
+      }
+
+      bool SetBasePath(string? path) {
+         if (!Directory.Exists(path)) {
+            return false;
+         }
+         configBuilder.SetBasePath(path);
+         return true;
+      }
+
+      bool AddJsonFile(string? path) {
+         if (!File.Exists(path)) {
+            return false;
+         }
+         configBuilder.AddJsonFile(path);
+         return true;
+      }
+   }
+
+   public static string GetCallingAssemblyName() =>
+      new StackTrace()
+         .GetFrame(2)
+         ?.GetMethod()
+         ?.Module
+         ?.Assembly
+         ?.GetName()
+         ?.Name
+         ?? throw new InvalidOperationException("Failed to determine name of calling assembly (calling method is null)");
+}
@@ -0,0 +1,9 @@
+<Project Sdk="Microsoft.NET.Sdk">
+
+  <PropertyGroup>
+    <TargetFramework>net7.0</TargetFramework>
+    <ImplicitUsings>enable</ImplicitUsings>
+    <Nullable>enable</Nullable>
+  </PropertyGroup>
+
+</Project>
@@ -0,0 +1,28 @@
+namespace Bts.Net;
+
+using System.Linq;
+using System.Net;
+
+/// <summary>
+/// Provides facilities to compare hosts for equality.
+/// <para>
+/// This class doesn't implement IEqualityComparer because a consistent hash code cannot be
+/// guarenteed for equivalent host strings. The equals impl relies on Dns.GetHostAddresses, which
+/// is non-determinstic (implementations that depends on mutable properties face a similar problem).
+/// </para>
+/// </summary>
+public class HostComparer {
+   public static bool Equals(string? x, string? y) {
+      if (x is null) {
+         return y is null;
+      }
+      if (y is null) {
+         return false;
+      }
+      return Dns.GetHostAddresses(x).Intersect(
+         Dns.GetHostAddresses(y), IpAddressComparer.Instance).Any();
+   }
+
+   public static bool Equals(IPHostEntry x, IPHostEntry y) =>
+      x.AddressList.Intersect(y.AddressList, IpAddressComparer.Instance).Any();
+}
@@ -0,0 +1,30 @@
+namespace Bts.Net;
+
+using System.Diagnostics.CodeAnalysis;
+using System.Net;
+
+/// <summary>
+/// Compares ip addresses. Unlike the default comparer, v4 and equivalent v6 addresses are
+/// considered equal.
+/// </summary>
+public class IpAddressComparer : IEqualityComparer<IPAddress> {
+   public static IEqualityComparer<IPAddress> Instance { get; } = new IpAddressComparer();
+
+   public bool Equals(IPAddress? x, IPAddress? y) {
+      if (x is null) {
+         return y is null;
+      }
+      if (y is null) {
+         return false;
+      }
+      return x.Equals(y) ||
+         (x.IsIPv4MappedToIPv6 && x.MapToIPv4().Equals(y)) ||
+         (y.IsIPv4MappedToIPv6 && y.MapToIPv4().Equals(x));
+   }
+
+   public int GetHashCode([DisallowNull] IPAddress address) =>
+      address.IsIPv4MappedToIPv6
+         // Return consistent hash code for v4 and equivalent v6
+         ? address.MapToIPv4().GetHashCode()
+         : address.GetHashCode();
+}
@@ -0,0 +1,33 @@
+namespace Bts.Net;
+
+using System;
+using System.Net;
+using System.Text.Json;
+using System.Text.Json.Serialization;
+
+public class IpHostEntryJsonConverter : JsonConverter<IPHostEntry> {
+   public override IPHostEntry? Read(ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options) {
+      throw new NotImplementedException();
+   }
+
+   public override void Write(Utf8JsonWriter writer, IPHostEntry value, JsonSerializerOptions options) {
+      writer.WriteStartObject();
+      // HostName
+      writer.WriteString(nameof(value.HostName), value.HostName);
+      // Aliases
+      writer.WritePropertyName("Aliases");
+      writer.WriteStartArray();
+      foreach (var alias in value.Aliases) {
+         writer.WriteStringValue(alias);
+      }
+      writer.WriteEndArray();
+      // Addresses
+      writer.WritePropertyName("Addresses");
+      writer.WriteStartArray();
+      foreach (var address in value.AddressList) {
+         writer.WriteStringValue(address.ToString());
+      }
+      writer.WriteEndArray();
+      writer.WriteEndObject();
+   }
+}
@@ -0,0 +1,39 @@
+namespace Bts.Net;
+
+using System;
+using System.IO;
+using System.Net;
+using System.Text;
+using System.Text.Json;
+
+/// <summary>
+/// This class seriailzes IpHostEntry to json (string). A common buffer is used to reduce memory
+/// allocations on repeated calls to <see cref="Serialize"/>. For that reason, this class is NOT
+/// thread safe.
+/// </summary>
+public class IpHostEntryJsonSerializer {
+   readonly MemoryStream buffer = new();
+   readonly IpHostEntryJsonConverter converter;
+
+   public IpHostEntryJsonSerializer(IpHostEntryJsonConverter? converter = null) =>
+      this.converter = converter ?? new() { };
+
+   public string? Serialize(IPHostEntry dp, JsonSerializerOptions? opts = null) {
+      opts ??= JsonSerializerOptions.Default;
+      buffer.Position = 0;
+      buffer.SetLength(0);
+      using (var jsonWriter = new Utf8JsonWriter(buffer)) {
+         converter.Write(jsonWriter, dp, opts);
+      }
+      var bytes = buffer.GetBuffer();
+      var span = new ArraySegment<byte>(bytes, 0, (int)buffer.Length);
+      return Encoding.UTF8.GetString(span);
+   }
+
+   public void Serialize(IPHostEntry dp, Utf8JsonWriter jsonWriter, JsonSerializerOptions? opts = null) {
+      opts ??= JsonSerializerOptions.Default;
+      buffer.Position = 0;
+      buffer.SetLength(0);
+      converter.Write(jsonWriter, dp, opts);
+   }
+}
@@ -0,0 +1,94 @@
+namespace Bts.Net;
+
+using System.Diagnostics.CodeAnalysis;
+using System.Net;
+using System.Text.RegularExpressions;
+
+public static partial class IpUtils {
+   static readonly Regex ipv4Regex = new(string.Join('.', @"(((0|1)\d)|(2\d)|\d)?\d"));
+
+   // public static void ParseUri(string str, out string? scheme, out string? host, out string? port, out string? path, out string? queryParams) {
+   //    if (Uri.TryCreate(str, UriKind.Absolute, out var uri)) {
+   //    }
+   // }
+
+   // public static bool AreEndpointsEqual(Uri? x, Uri? y) {
+   //    if (x is null) {
+   //       return y is null;
+   //    }
+   //    if (y is null) {
+   //       return false;
+   //    }
+   //    if (x.IsAbsoluteUri != y.IsAbsoluteUri) {
+   //       return false;
+   //    }
+   //    if (x.IsAbsoluteUri) {
+   //       if (x.Scheme != y.Scheme) {
+   //          return false;
+   //       }
+   //    }
+   //    string? xHost = null;
+   // }
+
+   // static string ResolveHostIpv4(Uri uri) {
+   //    // switch (uri.HostNameType) {
+   //    //    case UriHostNameType.IPv4:
+   //    //       return uri.Host;
+   //    //    case UriHostNameType.Dns:
+   //    //       break;
+   //    //    case UriHostNameType.IPv6:
+   //    //       var addresses = Dns.GetHostAddresses(uri.Host);
+   //    //       foreach (var address in addresses) {
+   //    //          if (address.IsIPv4MappedToIPv6) {
+   //    //             return address.MapToIPv4().Address;
+   //    //          }
+   //    //       }
+   //    //       break;
+   //    // }
+   //    var hostAddresses = Dns.GetHostAddresses(uri.Host);
+   //    foreach (IPAddress address in hostAddresses) {
+   //       if (address.Equals(remoteEndPoint.Address) ||
+   //           (address.IsIPv4MappedToIPv6 && address.MapToIPv4().Equals(remoteEndPoint.Address)) ||
+   //           (remoteEndPoint.Address.IsIPv4MappedToIPv6 && remoteEndPoint.Address.MapToIPv4().Equals(address))) {
+   //          isMatch = true;
+   //          break;
+   //       }
+   //    }
+   // }
+
+   // public static bool AreEndpointsEqual(string? x, string? y) {
+   //    if (x == y) {
+   //       return true;
+   //    }
+   //    if (x is null) {
+   //       return y is null;
+   //    }
+   //    if (y is null) {
+   //       return false;
+   //    }
+   //    var xUri = new Uri(x);
+   //    var yUri = new Uri(y);
+   //    // return xUri.Equals(yUri);
+   // }
+
+   public static bool IsIpv4(string host) {
+      host = removeSchemaFromUrl(host);
+      host = removePortFromUrl(host);
+      return ipv4Regex.IsMatch(host);
+   }
+
+   static string removeSchemaFromUrl(string url) {
+      var i = url.IndexOf("://");
+      return i < 0 ? url : url.Substring(i + 3);
+   }
+
+   static string removePortFromUrl(string url) {
+      var match = MyRegex().Match(url);
+      return match.Success
+         ? string.Concat(url.AsSpan(0, match.Index), url.AsSpan(match.Index + match.Length))
+         : url;
+   }
+
+   [GeneratedRegex(":\\d{1,4}")]
+   private static partial Regex MyRegex();
+}
@@ -7,14 +7,11 @@
  </PropertyGroup>

  <ItemGroup>
-    <PackageReference Include="Google.Protobuf" Version="3.24.3" />
    <PackageReference Include="YaccLexTools" Version="1.2.2" />
  </ItemGroup>

  <ItemGroup>
-    <ProjectReference Include="..\DataStreamCore\DataStreamCore.csproj" />
    <ProjectReference Include="..\ExpressionCore\ExpressionCore.csproj" />
-    <ProjectReference Include="..\..\client\spark_core\spark_core.csproj" />
  </ItemGroup>

  <ItemGroup Label="ColumnMapExpression">
@@ -1,16 +1,24 @@
+using Bts.DataStream;
+using Bts.DataStreamCore;
+using Bts.DataStreamExpressions;
+using Bts.PubSub;
+
+using bts;
+
+using Microsoft.Extensions.Hosting;
+
+using Spark.Core.Disposables;
+
 using System;
 using System.Collections.Generic;
-
-using Bts.DataStreamExpressions;
-using BCG.Utilities.General;
-using bts;
-using Bts.PubSub;
-using Bts.DataStreamCore;
+using System.Reactive.Linq;
+using System.Threading;
+using System.Threading.Tasks;

 namespace BCG {
   // listens to the requests posted on "cloud_streams" request table, and 
   // if unfulfilled starts a ViewPubsliher
-   public class RemoteQueryManager : IWorker {
+   public class RemoteQueryManager : IHostedService {
      public class StreamPubUnit {
         public string name;
         public string stream_expression;
@@ -41,25 +49,46 @@ namespace BCG {
      }
      private IDictionary<string, StreamPubUnit>
         _stream_pub_map = new Dictionary<string, StreamPubUnit>();
-      private ViewStream _stream = new TableStream("cloud_streams");
      readonly IPubTableBroker _broker;
      readonly IManagedSubscriber _sub;
+      readonly object syncRoot = new();
+      IDisposable? cloudStreamsSub;
+      IDisposable? remoteRequestsSub;
+      bool isDisposed;


      public RemoteQueryManager(IManagedSubscriber managedSubscriber, IPubTableBroker pubTableBroker) {
         _sub = managedSubscriber;
         _broker = pubTableBroker;
-         _stream.onChange += _on_str;
-         _sub.Subscribe("RemoteRequest", (string tag, byte[] data) => {
-            RemoteRequest req = RemoteRequest.construct_from_bytes(data);
-            _spawn_remote_stream(req.published_table_name, req.stream_expression, false);
-         });
      }
-      private void _on_str(object obj)
-      {
-         if (obj is not DataPacket dp) {
-            return;
+
+      public Task StartAsync(CancellationToken cancellationToken) {
+         lock (syncRoot) {
+            Log.assert(!isDisposed);
+            Log.isnull(cloudStreamsSub);
+            Log.isnull(remoteRequestsSub);
+            cloudStreamsSub = new TableStream("cloud_streams").AsObservable().Subscribe(_on_str);
+            remoteRequestsSub = _sub.Observe("RemoteRequest").Subscribe(data => {
+               RemoteRequest req = RemoteRequest.construct_from_bytes(data);
+               _spawn_remote_stream(req.published_table_name, req.stream_expression, false);
+            });
         }
+         return Task.CompletedTask;
+      }
+
+      public Task StopAsync(CancellationToken cancellationToken) {
+         lock (syncRoot) {
+            if (!isDisposed) {
+               isDisposed = true;
+               Disposable.dispose(ref cloudStreamsSub);
+               Disposable.dispose(ref remoteRequestsSub);
+            }
+         }
+         return Task.CompletedTask;
+      }
+
+      private void _on_str(DataPacket dp)
+      {
         foreach (IBlueRow r in dp.newRows)
         {
            string name = r.get<string>("name");
@@ -86,9 +115,5 @@ namespace BCG {
      private void _expose_to_ws(string name, ViewStream stream) {
         // TODO: john
      }
-
-      public void doWork() {
-
-      }
   }
 }
@@ -1,23 +1,24 @@
+using bts;
+
 using System;
 using System.Collections;

-namespace BCG
-{
+namespace BCG {
 	/// <summary>
 	/// Provides a stream that with distinct rows with respect to the columns
 	/// the user specifies in the argument
 	/// </summary>
-	[DataStreamUserRespresentation ("distinct")]
-	public class DistinctRowStream : ViewStream
-	{
-		public DistinctRowStream (ViewStream stream, params string[] cols)
-		{
-         ArrayList groupByCols = new ArrayList ();
-         foreach (string col in cols)
-            foreach (string c in col.Split (' '))
-               groupByCols.Add (c);
-         mySetRealSource (new GroupByStream (stream, 
-            new DistinctRowEvaluator (groupByCols.ToArray (typeof(string)) as string[]), false));
+	[DataStreamUserRespresentation("distinct")]
+	public class DistinctRowStream : ViewStream {
+		public DistinctRowStream(ViewStream stream, params string[] cols) {
+			var g_by_eval = new DistinctRowEvaluator(cols);
+			ViewStream r = new GroupByStream(stream, g_by_eval);
+			var proj_exp = String.Empty;
+			foreach (var c in cols)
+				proj_exp += $"{c} ";
+			proj_exp = proj_exp.Trim();
+			r = new NewProjectStream(r, proj_exp);
+			mySetRealSource(r);
 		}
 	}
 }
@@ -1,49 +0,0 @@
-// using System;
-// using System.Collections.Generic;
-// using System.Text;
-// using System.IO;
-
-// using System.Runtime.Serialization.Formatters.Binary;
-
-// namespace BCG
-// {
-//    public class DumpLoaderStream : ViewStream, ISideEffectStream
-//    {
-//       //wtf
-//       //public class Exception : System.Exception
-//       //{public Exception(string msg) : base(msg) { }}
-
-//       public DumpLoaderStream(string fileName, int secondsBetweenPolls)
-//       {
-//          mySetRealSource(
-//             new PollStream(() => new LoaderStream(fileName).rows, secondsBetweenPolls, 
-//                ViewStreamDatabase.instance));
-//       }
-
-//       public DumpLoaderStream(string fileName)
-//       {
-//         if (fileName == null)
-//            throw new LoaderStream.Exception("File name cannot be null");
-
-//         for (int i = 0; i < 5; i++)
-//            try
-//            {
-//               using (FileStream fs = new FileStream(fileName, FileMode.Open, FileAccess.Read))
-//               {
-//                  IClientDeserializer deser = new ClientUniversalDeserializer();
-//                  byte[] the_ops;
-//                  Array the_rows;
-//                  deser.deserialize(new Stream[] { fs }, out the_ops, out the_rows);
-
-//                  mySetRealSource(new BlueRowCollectionStaticStream((IBlueRowCollection)the_rows.GetValue(0)));
-//               }
-//               break;
-//            }
-//            catch (IOException)
-//            {
-//               System.Threading.Thread.Sleep(50);
-//            }
-
-//       }
-//    }
-// }
@@ -1,6 +1,7 @@
 using BCG;

 namespace bts {
+   // This is the other side of the SaverStream, its "loads" the stream and fires once
   [DataStreamUserRespresentation("loader")]
   public class LoaderStream : ViewStream {

@@ -5,10 +5,11 @@ using System.Text;
 using System.IO;
 using Google.Protobuf;
 using System.Globalization;
-using bts;
+using BCG;

-namespace BCG {
-   // Writes all the udpates to a stream in a file.
+namespace bts {
+   // Writes all the udpates to a stream in a file or multiples files dependong on the rollover size (as opposed to Saver
+   // which just writes one update, as a create)
   // File naming:
   // token_epoch_yyyyMMdd_HHmmss
   // File location: directory(passed)\token\
@@ -94,10 +95,10 @@ namespace BCG {
      void _on_change(object new_value) {
         DataPacket dp = new_value as DataPacket;
         byte[] serialized_update = null;
-         bool _write_create = _serializer == null || _n_updates_in_current_file == 0;
+         bool first_write = _serializer == null || _n_updates_in_current_file == 0;
         _serializer = _serializer ?? new UniversalStreamIOSerializer();
-         if (_write_create) {
-            // new files always starts with create
+         if (first_write) {
+            // first file always starts with create
            serialized_update = _serializer.serialize(new bts.StreamIOMessage(new bts.StreamIOMessage.CreateMessage(dp.newRows)));
         } else
            serialized_update = _serializer.serialize(new bts.StreamIOMessage(new bts.StreamIOMessage.MultipleRemoveAddMessage(dp.deletedRows, dp.addedRows)));
@@ -105,8 +106,8 @@ namespace BCG {
         long utc_epoch = epoch_now();

         //time, length of data, data
-         bw.Write(utc_epoch);
-         bw.Write(serialized_update.Length);
+         //bw.Write(utc_epoch);
+         //bw.Write(serialized_update.Length);
         bw.Write(serialized_update);
         _out_fs.Flush();
         myFireEvent(dp);
@@ -1,190 +0,0 @@
-// using System;
-// using System.Collections.Generic;
-// using System.Linq;
-// using System.Text;
-// using System.IO;
-// using System.Runtime.Serialization.Formatters.Binary;
-// using BCG.Utilities.General;
-// using bts.collections;
-// using System.Reflection.Metadata;
-
-
-// namespace BCG {
-
-//    public class PlaybackStream : ViewStream, IWorker {
-//       private readonly PlaybackType _type;
-//       private BlueTable _current_table;
-//       private IClientDeserializer _deserilializer = new ClientUniversalDeserializer();
-//       // these are the packets already downloaded from the current file
-//       private List<PlaybackPacket> _packets = new List<PlaybackPacket>();
-//       //private IEnumerable<FileStream> _file_streams = new List<FileStream>();
-//       private IEnumerable<string> _file_names;
-//       private Tuple<int, FileStream> _current_fs = new Tuple<int, FileStream>(-1, null);
-//       private DateTime _playback_t0, _playback_t1;
-//       private bool _all_files_read = false;
-//       SnapshotData _snapshot_data = null; // only initialized when in snapshot mode
-
-//       public PlaybackStream(string datetime_start, string datetime_end, string directory_name, string token) {
-//          _type = PlaybackType.Streaming;
-//          _playback_t0 = PersistorStream.parse_datetime(datetime_start);
-//          _playback_t1 = PersistorStream.parse_datetime(datetime_end);
-//          string token_directory_name = $"{directory_name}\\{token}";
-//          bts.Log.Gen.info($"PlaybackStream t0: {_playback_t0} t1: {_playback_t1} dir: {token_directory_name}");
-//          List<string> files_in_range = new List<string>();
-//          if (Directory.Exists(token_directory_name)) {
-//             _file_names =
-//          Directory.GetFiles(token_directory_name).Where(f => {
-//             DateTime file_time = PersistorStream.parse_datetime_from_filename(f);
-//             return file_time > _playback_t0 && file_time < _playback_t1;
-//          });
-//          }
-//       }
-
-//       protected override void myOnFirstSubscriber() {
-//          FileStream fs = new FileStream(_snapshot_data.snapshot_file, FileMode.Open, FileAccess.Read);
-//          _snapshot_data.packets = StreamPersistorUtils.load_packets_from_file(fs);
-//          PlaybackPacket snapshot_packet = StreamPersistorUtils.find_packet_at_time(_snapshot_data.packets, _snapshot_data.snapshot_time);
-//          if (snapshot_packet != null) {
-//             foreach (PlaybackPacket p in _snapshot_data.packets) {
-//                byte[] ops;
-//                Array row_array;
-//                _deserilializer.deserialize(new Stream[] { new MemoryStream(p.data) }, out ops, out row_array);
-//                for (int i = 0; i < ops.Length; i++) {
-//                   IBlueRowCollection[] rows_args = row_array.GetValue(i) as IBlueRowCollection[];
-//                   byte op = ops[i];
-//                   if (op == StreamOpCode.create) {
-//                      IBlueRowCollection full_table = rows_args[0];
-//                      _current_table = new BlueTable(full_table);
-//                   } else if (op == StreamOpCode.multipleRemoveAdd) {
-//                      //IBlueRowCollection[] del_added = rows_args.GetValue(0) as IBlueRowCollection[];
-//                      IBlueRowCollection dels = rows_args.GetValue(0) as IBlueRowCollection;
-//                      IBlueRowCollection adds = rows_args.GetValue(1) as IBlueRowCollection;
-//                      BlueDataUtility.absorbChanges(_current_table, dels, adds);
-//                   } else {
-//                      bts.Log.assert(false, $"Oprator: {op} is not expected");
-//                   }
-//                   if (p.idx == snapshot_packet.idx) {
-//                      myFireEvent(new DataPacket(_current_table, _current_table, new BlueTable(_current_table.schema)));
-//                      break;
-//                   }
-//                }
-//             }
-//          }
-//       }
-
-
-//       private void _load_packets_from_current_file() {
-//          int curr_idx = _current_fs.Item1;
-//          FileStream curr_fs = _current_fs.Item2;
-//          _packets = new List<PlaybackPacket>();
-//          _packets.AddRange(StreamPersistorUtils.load_packets_from_file(curr_fs));
-//       }
-
-//       private bool current_file_all_replayed {
-//          get {
-//             bool rv = true;
-//             _packets.ForEach(p => rv = rv && p.replayed);
-//             return rv;
-//          }
-//       }
-
-//       protected override void myOnLastSubscriber() {
-//          var fs = _current_fs.Item2;
-//          if (fs != null) {
-//             fs.Close();
-//          }
-//          ViewStreamDatabase.instance.removeWorker(this);
-//       }
-
-//       // public event PlaybackStreamProgress onProgress;
-//       // public event Action onFileExhausted;
-//       private PlaybackPacket _locate_packet_to_replay() {
-//          foreach (var p in _packets) {
-//             // returns the first non replayed packet
-//             if (!p.replayed)
-//                return p;
-//          }
-//          //else
-//          {
-//             double secondsElapsed = new BinaryReader(myFileReader).ReadDouble();
-//             myNextPacketTime = myPlaybackStart + new TimeSpan(0, 0, 0, 0, (int)(1000 * secondsElapsed / myPlaybackSpeed));
-//             if (onProgress != null)
-//                onProgress(myFileReader.Position, myFileReader.Length);
-//          }
-//       }
-
-//       #region IWorker Members
-
-//       public void doWork() {
-//          if (_all_files_read)
-//             return;
-//          if (current_file_all_replayed) {
-//             _load_next_file();
-//          }
-//          PlaybackPacket curr_packet = _locate_packet_to_replay();
-//          if (curr_packet.idx == 0) { // first packet in the file, fire it rightaway 
-//             byte[] ops;
-//             Array row_array;
-//             // what a stupid interface!
-//             _deserilializer.deserialize(new Stream[] { new MemoryStream(curr_packet.data) }, out ops, out row_array);
-//             _fire_packet(ops, row_array);
-//             curr_packet.replay_timestamp = PersistorStream.epoch_now();
-//          } else {
-//             PlaybackPacket prev_packet = _get_previous_packet(curr_packet);
-//             // this is the time we need to fire (replay) this packet
-//             long time_to_fire = prev_packet.replay_timestamp + curr_packet.offset_from_previous;
-//             long now_epoch = PersistorStream.epoch_now();
-//             // fire when we are past the 
-//             if (now_epoch >= time_to_fire) {
-//                byte[] ops;
-//                Array row_array;
-//                _deserilializer.deserialize(new Stream[] { new MemoryStream(curr_packet.data) }, out ops, out row_array);
-//                _fire_packet(ops, row_array);
-//                curr_packet.replay_timestamp = PersistorStream.epoch_now();
-//             }
-//          }
-//       }
-
-//       private void _fire_packet(byte[] ops, Array row_array) {
-//          for (int i = 0; i < ops.Length; i++) {
-//             IBlueRowCollection[] rows_args = row_array.GetValue(i) as IBlueRowCollection[];
-//             byte op = ops[i];
-//             if (op == StreamOpCode.create) {
-//                IBlueRowCollection full_table = rows_args[0];
-//                _current_table = new BlueTable(full_table);
-//                //_current_table.addRows(full_table);
-//                myFireEvent(new DataPacket(full_table, full_table, myLastValue == null ? new BlueTable(full_table.schema) :
-//                                           (myLastValue as DataPacket).newRows));
-//             } else if (op == StreamOpCode.multipleRemoveAdd) {
-//                //IBlueRowCollection[] del_added = rows_args.GetValue(0) as IBlueRowCollection[];
-//                IBlueRowCollection dels = rows_args.GetValue(0) as IBlueRowCollection;
-//                IBlueRowCollection adds = rows_args.GetValue(1) as IBlueRowCollection;
-//                BlueDataUtility.absorbChanges(_current_table, dels, adds);
-//                myFireEvent(new DataPacket(_current_table, adds, dels));
-//             } else {
-//                bts.Log.assert(false, $"Oprator: {op} is not expected");
-//             }
-
-//          }
-//       }
-
-//       private void _load_next_file() {
-//          var new_idx = _current_fs.Item1 + 1;
-//          if (new_idx >= _file_names.Count()) {
-//             _all_files_read = true;
-//             bts.Log.Gen.info("All files read");
-//             return;
-//          }
-//          var prev_fs = _current_fs.Item2;
-//          if (prev_fs != null)
-//             prev_fs.Close();
-//          var new_filename = _file_names.ElementAt(new_idx);
-//          bts.Log.Gen.info("PlaybackStream_load_next_file new_filename=: {new_filename}");
-//          var new_fs = new FileStream(new_filename, FileMode.Open, FileAccess.Read);
-//          _current_fs = new Tuple<int, FileStream>(new_idx, new_fs);
-//          _load_packets_from_current_file();
-//       }
-
-//       #endregion
-//    }
-// }
@@ -1,68 +0,0 @@
-// using System;
-// using System.Collections.Generic;
-// using System.Linq;
-// using System.Text;
-// using System.IO;
-
-// namespace BCG
-// {
-//    public class RecorderStream : ViewStream
-//    {
-//       private ViewStream mySource;
-//       private DateTime myStartTime;
-//       private FileStream myOutFile;
-//       private string myFileName;
-//       private IBlueDataSerializer mySerializer;
-
-//       public RecorderStream(ViewStream src, string recordFile)
-//       {
-//          mySource = src;
-//          myFileName = recordFile;
-
-//          myFileName = myFileName.Replace("%DATE%", DateTime.Now.ToString("yyyyMMdd_HHmmss")); 
-//       }
-
-//       protected override void myOnFirstSubscriber()
-//       {
-//          myStartTime = DateTime.Now;         
-//          mySource.onChange += new ChangeDelegate(mySource_onChange);
-//          myOutFile = new FileStream(myFileName, FileMode.OpenOrCreate);
-//       }
-
-//       protected override void myOnLastSubscriber()
-//       {
-//          mySource.onChange -= mySource_onChange;
-//          myOutFile.Close();
-//       }
-
-//       void mySource_onChange(object newValue)
-//       {
-//          DataPacket dp = (DataPacket)newValue;         
-
-//          BlueTable toWrite = new BlueTable(dp.schema);
-
-//          if (mySerializer == null)
-//          {
-//             mySerializer = new UniversalSerializer(dp.schema);
-//          }
-//          foreach (IBlueRow r in dp.newRows)
-//             toWrite.addRow(r);
-
-//          byte[] result;
-
-//          if (myLastValue == null)
-//             mySerializer.serialize(StreamOpCode.create, new IBlueRowCollection[] { dp.newRows }, out result);
-//          else
-//             mySerializer.serialize(StreamOpCode.multipleRemoveAdd, new IBlueRowCollection[] { dp.deletedRows, dp.addedRows }, out result);
-
-//          BinaryWriter bw = new BinaryWriter(myOutFile);
-//          //write down what time this happened
-//          bw.Write((DateTime.Now - myStartTime).TotalSeconds);
-//          bw.Write(result.Length);
-//          bw.Write(result);         
-//          myOutFile.Flush();
-
-//          myFireEvent(dp);
-//       }
-//    }
-// }
@@ -5,7 +5,8 @@ using System.IO;
 using BCG;

 namespace bts {
-   // this saves the total state in the filr provided in the cons
+   // this saves the updates as a Create on the same file, each update calls -save, which
+   // reqrites the whole data to the packet. The other side of this stream is the Loader
   [DataStreamUserRespresentation("saver")]
   public class SaverStream : ViewStream {
      private ViewStream _stream;
@@ -31,7 +32,6 @@ namespace bts {
         StreamIOMessage.CreateMessage cr_msg = new StreamIOMessage.CreateMessage(dp.newRows);
         byte[] str_bytes = _serializer.serialize(new StreamIOMessage(cr_msg));
         StreamSerializationUtils.write_to_file(str_bytes, _filename);
-
      }
   }
 }
@@ -3,6 +3,7 @@ using System.Collections.Generic;
 using System.Linq;
 using System.Globalization;
 using System.IO;
+using bts;


 namespace BCG {
@@ -1,4 +1,7 @@
+using bts;
+
 using Bts.DataStream;
+using Bts.DataStream.Primitives.Filter;

 using mbts.config;

@@ -26,6 +29,7 @@ namespace BCG.StreamWorld.Addition
      {
         myBaseStream = vs;
         myFilter = filter;
+         vs.ErrorChanged += OnError;
         if (filter is IDynamicStreamFilter)
            (filter as IDynamicStreamFilter).onFilterChange += new FilterChangeDelegate(myHandleFilterChange);
      }
@@ -43,37 +47,50 @@ namespace BCG.StreamWorld.Addition
         myStateTable.clear();
      }

+      bool filterRow(IBlueRow r) {
+         try {
+            return myFilter.filter(r);
+         } catch (Exception ex) {
+            Log.Gen.error($"FilterStream delegate threw exception: {ex}");
+            OnError(new StreamFilterException(StreamId, StreamName, null, ex, r));
+         }
+         return false;
+      }
+
      private void myHandleChanges(object newValue)
      {
-         DataPacket dp = newValue as DataPacket;
-         if (myStateTable == null)
-         {
-            bts.Log.assert(myBaseTable == null);
-            myStateTable = new BlueTable(dp.schema);
-            myBaseTable = new BlueTable(dp.schema);
-         }
-         BlueTable dels = new BlueTable(dp.schema),
-                     adds = new BlueTable(dp.schema);
-         foreach (IBlueRow r in dp.deletedRows)
-         {
-            myBaseTable.deleteRow(r);
-            if (myStateTable.containsID((int)r["id"]))
+         try {
+            DataPacket dp = newValue as DataPacket;
+            if (myStateTable == null)
            {
-               dels.addRow(r);
-               myStateTable.deleteRow(r);
+               bts.Log.assert(myBaseTable == null);
+               myStateTable = new BlueTable(dp.schema);
+               myBaseTable = new BlueTable(dp.schema);
            }
-         }
-         foreach (IBlueRow r in dp.addedRows)
-         {
-            myBaseTable.addRow(r);
-            if (myFilter.filter(r))
+            BlueTable dels = new BlueTable(dp.schema),
+                        adds = new BlueTable(dp.schema);
+            foreach (IBlueRow r in dp.deletedRows)
            {
-               adds.addRow(r);
-               myStateTable.addRow(r);
+               myBaseTable.deleteRow(r);
+               if (myStateTable.containsID((int)r["id"]))
+               {
+                  dels.addRow(r);
+                  myStateTable.deleteRow(r);
+               }
            }
+            foreach (IBlueRow r in dp.addedRows) {
+               myBaseTable.addRow(r);
+               if (filterRow(r)) {
+                  adds.addRow(r);
+                  myStateTable.addRow(r);
+               }
+            }
+            if ((dels.Count > 0 || adds.Count > 0) || (myLastValue == null)) {
+               myFireEvent(new DataPacket(myStateTable, adds, dels));
+            }
+         } catch (Exception ex) {
+            OnError(ex);
         }
-         if ((dels.Count > 0 || adds.Count > 0) || (myLastValue == null))
-            myFireEvent(new DataPacket(myStateTable, adds, dels));
      }

      public IStreamFilter filter
@@ -85,9 +102,20 @@ namespace BCG.StreamWorld.Addition
            {
               IBlueRowCollection oldState = new BlueTable(myStateTable);
               myStateTable.clear();
-               foreach (IBlueRow r in myBaseTable)
-                  if (myFilter.filter(r))
-                     myStateTable.addRow(r);
+               // If the filter throws an exception, we can rethrow it as a StreamFilterException
+               // and include the 'currentRow' (for which the filter failed) to provide context.
+               IBlueRow? currentRow = null;
+               foreach (IBlueRow r in myBaseTable) {
+                  try {
+                     currentRow = r;
+                     if (filterRow(r)) {
+                        myStateTable.addRow(r);
+                     }
+                  } catch (Exception ex) {
+                     Log.Gen.error($"FilterStream delegate threw exception: {ex}");
+                     OnError(new StreamFilterException(StreamId, StreamName, null, ex, currentRow));
+                  }
+               }
               IBlueRowCollection dels, adds;
               BlueDataUtility.computeDifferences(myStateTable, oldState, out dels, out adds);
               myFireEvent(new DataPacket(myStateTable, adds, dels));
@@ -217,37 +245,37 @@ namespace BCG
      }
      #endregion

-      private void myHandleCallbacks(object newValue)
-      {
+      private void myHandleCallbacks(object newValue) {
         DataPacket retVal;
         DataPacket arrivedPacket = newValue as DataPacket;
         myLastReceivedRows = arrivedPacket.newRows;
-         if (myLastFiredRows != null)
-         {
-            retVal = new DataPacket(myLastFiredRows, new BlueTable(myLastFiredRows.schema),
-               new BlueTable(myLastFiredRows.schema));
-            foreach (IBlueRow r in arrivedPacket.deletedRows)
-               if (myLastFiredRows.containsID((int)r["id"]))
-               {
-                  myLastFiredRows.deleteRow(r);
-                  (retVal.deletedRows as BlueTable).addRow(r);
+         try {
+            if (myLastFiredRows != null) {
+               retVal = new DataPacket(myLastFiredRows, new BlueTable(myLastFiredRows.schema),
+                  new BlueTable(myLastFiredRows.schema));
+               foreach (IBlueRow r in arrivedPacket.deletedRows) {
+                  if (myLastFiredRows.containsID((int)r["id"])) {
+                     myLastFiredRows.deleteRow(r);
+                     (retVal.deletedRows as BlueTable).addRow(r);
+                  }
               }
-            foreach (IBlueRow r in arrivedPacket.addedRows)
-               if (myFilter.filter(r))
-               {
-                  myLastFiredRows.addRow(r);
-                  (retVal.addedRows as BlueTable).addRow(r);
+               foreach (IBlueRow r in arrivedPacket.addedRows) {
+                  if (filterRow(r)) {
+                     myLastFiredRows.addRow(r);
+                     (retVal.addedRows as BlueTable).addRow(r);
+                  }
               }
+            } else {
+               myLastFiredRows = new BlueTable(arrivedPacket.schema);
+               retVal = new DataPacket(myLastFiredRows, myLastFiredRows, new BlueTable(myLastFiredRows.schema));
+               foreach (IBlueRow r in arrivedPacket.newRows)
+                  if (filterRow(r))
+                     myLastFiredRows.addRow(r);
+            }
+            myFireEvent(retVal);
+         } catch (Exception ex) {
+            OnError(ex);
         }
-         else
-         {
-            myLastFiredRows = new BlueTable(arrivedPacket.schema);
-            retVal = new DataPacket(myLastFiredRows, myLastFiredRows, new BlueTable(myLastFiredRows.schema));
-            foreach (IBlueRow r in arrivedPacket.newRows)
-               if (myFilter.filter(r))
-                  myLastFiredRows.addRow(r);
-         }
-         myFireEvent(retVal);
      }

      public IStreamFilter filter
@@ -263,18 +291,23 @@ namespace BCG
               myFilter = value;
               if (numberOfSubscribers > 0)
               {
-                   if (myLastReceivedRows != null)
-                   {
-                       BlueTable nRows = new BlueTable(myLastReceivedRows.schema);
-                       foreach (IBlueRow r in myLastReceivedRows)
-                           if (myFilter.filter(r))
-                               nRows.addRow(r);
-                       IBlueRowCollection dRows = new BlueTable(nRows.schema),
-                          aRows = new BlueTable(nRows.schema);                      
-                       BlueDataUtility.computeDifferences(nRows, myLastFiredRows, out dRows, out aRows);
-                       myLastFiredRows = nRows;
-                       myFireEvent(new DataPacket(nRows, aRows, dRows));
-                   }
+                  if (myLastReceivedRows != null) {
+                     try {
+                        BlueTable nRows = new BlueTable(myLastReceivedRows.schema);
+                        foreach (IBlueRow r in myLastReceivedRows) {
+                           if (filterRow(r)) {
+                              nRows.addRow(r);
+                           }
+                        }
+                        IBlueRowCollection dRows = new BlueTable(nRows.schema),
+                           aRows = new BlueTable(nRows.schema);
+                        BlueDataUtility.computeDifferences(nRows, myLastFiredRows, out dRows, out aRows);
+                        myLastFiredRows = nRows;
+                        myFireEvent(new DataPacket(nRows, aRows, dRows));
+                     } catch (Exception ex) {
+                        OnError(ex);
+                     }
+                  }
               }
            }
         }
@@ -298,5 +331,13 @@ namespace BCG
      {
         filter = myFilter;
      }
+
+      bool filterRow(IBlueRow r) {
+         try {
+            return myFilter.filter(r);
+         } catch (Exception ex) {
+            throw new StreamFilterException(StreamId, StreamName, null, ex, r);
+         }
+      }
   }
 }
@@ -0,0 +1,19 @@
+#nullable enable
+namespace Bts.DataStream.Primitives.Filter;
+
+using BCG;
+
+using System;
+
+public class StreamFilterException : Exception {
+   public StreamFilterException(Guid streamId, string streamName, string? message, Exception? innerException, IBlueRow? badRow = null)
+      : base(message, innerException) {
+      StreamId = streamId;
+      StreamName = streamName;
+      BadRow = badRow;
+   }
+
+   public Guid StreamId { get; }
+   public string StreamName { get; }
+   public IBlueRow? BadRow { get; }
+}
@@ -90,7 +90,7 @@ namespace BCG {

         if (now > myNextFilter) {
            // fire event
-            onFilterChange();
+            onFilterChange?.Invoke();

            myNextFilter = now.Add(myFilterFrequency);
         }
@@ -43,6 +43,10 @@ namespace BCG
         _timer.Restart();
         IBlueRowCollection total_state = _groupby_func.evaluate(grouped_rows);
         _timer.Stop();
+         if (total_state is null) {
+            Log.Gen.warn("NonIncGroupByStream _groupby_func.evaluate returned null (ignoring last update).");
+            return;
+         }
         var applyElapsedMs = _timer.ElapsedMilliseconds;
         _n_total_fires++;         
         if (applyElapsedMs >= 50)
@@ -8,26 +8,33 @@ using System;
 using System.IO;
 using System.Linq;

-namespace bts {
-    public class PubSubTestingStream : ViewStream, IWorker {
-        public class PayloadMessage {
+namespace bts
+{
+    public class PubSubTestingStream : ViewStream, IWorker
+    {
+        public class PayloadMessage
+        {
            public int msg_id = 0;
            public string guid;
            public Int64 travel_epoch_ms = 0;
            public byte[] payload = null;
-            public PayloadMessage(int msg_id_args, Int64 send_epoch_args, int payload_size, byte payload_byte) {
+            public PayloadMessage(int msg_id_args, Int64 send_epoch_args, int payload_size, byte payload_byte)
+            {
                msg_id = msg_id_args;
                travel_epoch_ms = send_epoch_args;
                payload = new byte[payload_size];
-                for (int i = 0; i < payload_size; i++) {
+                for (int i = 0; i < payload_size; i++)
+                {
                    payload[i] = payload_byte;
                }
            }
            private PayloadMessage() { }
            public PayloadMessage(int msg_id, int payload_size, byte payload_byte) :
-            this(msg_id, PubSubTestingStream.epoch_now(), payload_size, payload_byte) { }
+            this(msg_id, PubSubTestingStream.epoch_now(), payload_size, payload_byte)
+            { }

-            public byte[] serialize() {
+            public byte[] serialize()
+            {
                var str = new MemoryStream();
                BinaryWriter bw = new BinaryWriter(str);
                bw.Write(msg_id);
@@ -40,7 +47,8 @@ namespace bts {
                return bytes;
            }

-            public static PayloadMessage deserialize(byte[] ser_bytes) {
+            public static PayloadMessage deserialize(byte[] ser_bytes)
+            {
                var str = new MemoryStream(ser_bytes);
                BinaryReader br = new BinaryReader(str);
                int msg_id = br.ReadInt32();
@@ -48,7 +56,8 @@ namespace bts {
                Int64 epoch = br.ReadInt64();
                int len = br.ReadInt32();
                var payload = br.ReadBytes(len);
-                return new PayloadMessage() {
+                return new PayloadMessage()
+                {
                    msg_id = msg_id,
                    guid = guid,
                    travel_epoch_ms = epoch,
@@ -73,7 +82,8 @@ namespace bts {
        private readonly int _payload_sz;
        private const byte _payload_byte = (byte)'A';
        private readonly string _tag_name;
-        public PubSubTestingStream(int publish_period_msecs, int max_pubs, int payload_size, string tag_name, ViewStreamDatabase vsd) {
+        public PubSubTestingStream(int publish_period_msecs, int max_pubs, int payload_size, string tag_name, ViewStreamDatabase vsd)
+        {
            _vsd = vsd;
            _payload_sz = payload_size;
            var schema = BlueDataUtility.form_schema("PubSubTest",
@@ -96,43 +106,53 @@ namespace bts {
        public PubSubTestingStream(int publish_period_seconds, int max_pubs, int msg_size, string tag_name)
        : this(publish_period_seconds, max_pubs, msg_size, tag_name, ViewStreamDatabase.instance) { }

-        protected override void myOnFirstSubscriber() {
+        protected override void myOnFirstSubscriber()
+        {
            _vsd.addWorker(this);
            System.Threading.Thread th = new System.Threading.Thread(_subscribe_thread_func);
            th.IsBackground = true;
            th.Start();
        }

-        protected override void myOnLastSubscriber() {
+        protected override void myOnLastSubscriber()
+        {
            _vsd.removeWorker(this);
            _continue_sub_thread = false;
        }

-        void _log_updates(IEnumerable<PubSubServerMessage> updates) {
+        void _log_updates(IEnumerable<PubSubServerMessage> updates)
+        {
            var idx = 0;
-            foreach (var msg in updates) {
+            foreach (var msg in updates)
+            {
                bts.Log.Gen.info($"PubSubTesStream._subscribe_thread_func idx: {idx} msg: {msg}");
                idx++;
            }
        }

-        private void _subscribe_thread_func() {
+        private void _subscribe_thread_func()
+        {
            // subscription is done in another thread
            SubscriberCore sub_core = new SubscriberCore();
            sub_core.subscribe(_tag_name);
-            while (_continue_sub_thread) {
+            while (_continue_sub_thread)
+            {
                // TODO recv_id
                var updates = sub_core.get_updates(TimeSpan.FromMilliseconds(250));
                if (updates == null || updates.Count() == 0)
                    continue;
                _log_updates(updates);
                _n_updates += updates.Count();
-                if (updates.Count() > 0) {
+                if (updates.Count() > 0)
+                {
                    var server_msg = updates.Last();
-                    if (server_msg.data != null && server_msg.data.Length > 0) {
+                    if (server_msg.data != null && server_msg.data.Length > 0)
+                    {
                        _last_recvd_msg = PayloadMessage.deserialize(server_msg.data);
                        _last_recvd_msg.travel_epoch_ms = epoch_now();
-                    } else {
+                    }
+                    else
+                    {
                        // why are we getting empty or null data ? (server is sending data regardless?)
                    }
                    _fire_dirty = true;
@@ -140,29 +160,36 @@ namespace bts {
            }
        }

-        private void _publish_next_packet() {
-            if (_last_sent_msg == null) {
-                _last_sent_msg = new PayloadMessage(0, _payload_sz, _payload_byte) {
+        private void _publish_next_packet()
+        {
+            if (_last_sent_msg == null)
+            {
+                _last_sent_msg = new PayloadMessage(0, _payload_sz, _payload_byte)
+                {
                    guid = Guid.NewGuid().ToString()
                };
-            } else {
+            }
+            else
+            {
                _last_sent_msg.msg_id++;
                _last_sent_msg.guid = Guid.NewGuid().ToString();
                _last_sent_msg.travel_epoch_ms = epoch_now();
            }
            var ser_bytes = _last_sent_msg.serialize();
            _n_pubs++;
-            _pub_core.publish(String.Empty, _tag_name, ser_bytes);
+            _pub_core.publish("Blob", _tag_name, ser_bytes);
            _fire_dirty = true;
        }

-        private void _fire_dp() {
+        private void _fire_dp()
+        {
            object last_sent_id = _last_sent_msg == null ? (object)DBNull.Value : (object)_last_sent_msg.msg_id;
            object last_recv_id = _last_recvd_msg == null ? (object)DBNull.Value : (object)_last_recvd_msg.msg_id;
            object echo_time = DBNull.Value;
            object last_sent_guid = _last_sent_msg == null ? (object)DBNull.Value : (object)_last_sent_msg.guid;
            object last_recv_guid = _last_recvd_msg == null ? (object)DBNull.Value : (object)_last_recvd_msg.guid;
-            if (_last_sent_msg != null && _last_recvd_msg != null) {
+            if (_last_sent_msg != null && _last_recvd_msg != null)
+            {
                echo_time = _last_recvd_msg.travel_epoch_ms - _last_sent_msg.travel_epoch_ms;
            }
            var match = last_sent_id.Equals(last_recv_id) && last_sent_guid.Equals(last_recv_guid);
@@ -174,21 +201,25 @@ namespace bts {
            myFireEvent(dp);
        }

-        public void doWork() {
+        public void doWork()
+        {
            if (DateTime.Now - _last_publish_time > _publish_period)
-                if (_n_pubs <= _max_pubs) {
+                if (_n_pubs <= _max_pubs)
+                {
                    _publish_next_packet();
                    _last_publish_time = DateTime.Now;
                }
            // publish 
            // TODO: send id
-            if (_fire_dirty) {
+            if (_fire_dirty)
+            {
                _fire_dirty = false;
                _fire_dp();
            }
        }

-        public static long epoch_now() {
+        public static long epoch_now()
+        {
            return DateTimeOffset.UtcNow.ToUnixTimeMilliseconds();
        }

@@ -10,17 +10,22 @@ using System.Runtime.CompilerServices;
 using bts.api;
 using Bts.PubSub.Routing;

-namespace bts {
+namespace bts
+{
   [DataStreamUserRespresentation("pss_echo")]
-   public class PubsubEchoTestStream : ViewStream, IWorker {
-      public class PayloadMessage {
+   public class PubsubEchoTestStream : ViewStream, IWorker
+   {
+      public class PayloadMessage
+      {
         public readonly string msg_id;
         public readonly Int64 timestamp = 0;
         public readonly byte[] payload = null;

-         public static byte[] create_data(int payload_size, byte payload_byte) {
+         public static byte[] create_data(int payload_size, byte payload_byte)
+         {
            var rv = new byte[payload_size];
-            for (int i = 0; i < payload_size; i++) {
+            for (int i = 0; i < payload_size; i++)
+            {
               rv[i] = payload_byte;
            }
            return rv;
@@ -36,14 +41,16 @@ namespace bts {
         //       payload[i] = payload_byte;
         //    }
         // }
-         public PayloadMessage(string guid_, Int64 timestamp_, byte[] payload_) {
+         public PayloadMessage(string guid_, Int64 timestamp_, byte[] payload_)
+         {
            msg_id = guid_;
            timestamp = timestamp_;
            payload = payload_;
         }

         private PayloadMessage() { }
-         public byte[] serialize() {
+         public byte[] serialize()
+         {
            var str = new MemoryStream();
            BinaryWriter bw = new BinaryWriter(str);
            bw.Write(msg_id);
@@ -55,7 +62,8 @@ namespace bts {
            return bytes;
         }

-         public static PayloadMessage deserialize(byte[] ser_bytes) {
+         public static PayloadMessage deserialize(byte[] ser_bytes)
+         {
            var str = new MemoryStream(ser_bytes);
            BinaryReader br = new BinaryReader(str);
            string guid = br.ReadString();
@@ -81,7 +89,8 @@ namespace bts {
      private readonly string _tag_name;
      private readonly int _payload_sz;
      private readonly IPubSubRouter _router;
-      private enum IOState {
+      private enum IOState
+      {
         SEND = 0,
         RECV = 1
      };
@@ -110,7 +119,8 @@ namespace bts {
      public PubsubEchoTestStream(string server_url, int publish_period_msecs, int payload_size, string tag_name)
      : this(server_url, TimeSpan.FromMilliseconds(publish_period_msecs), payload_size, tag_name, ViewStreamDatabase.instance) { }

-      protected override void myOnFirstSubscriber() {
+      protected override void myOnFirstSubscriber()
+      {
         _pub_core = new PublisherCore(_router);
         _sub_core = new SubscriberCore(_router);
         _sub_core.subscribe(_tag_name);
@@ -118,12 +128,14 @@ namespace bts {

      }

-      protected override void myOnLastSubscriber() {
+      protected override void myOnLastSubscriber()
+      {
         _vsd.removeWorker(this);
         _sub_core.unsubscribe(_tag_name);
      }

-      private void _fire_dp_after_send() {
+      private void _fire_dp_after_send()
+      {
         // as soon as the data is published, its reflected in the stream state
         Log.assert(_last_sent_msg != null);
         BlueRow r = new BlueRow(_table.schema, _table.Count, _last_sent_msg.msg_id, _last_sent_msg.timestamp, DBNull.Value, DBNull.Value,
@@ -133,7 +145,8 @@ namespace bts {
         myFireEvent(dp_out);
      }

-      private void _fire_dp_after_recv() {
+      private void _fire_dp_after_recv()
+      {
         Log.assert(_last_recvd_msg != null);
         // update the last publish row
         int id = _table.Count - 1;
@@ -147,11 +160,13 @@ namespace bts {
         myFireEvent(dp_out);
      }

-      private string _get_next_id() {
+      private string _get_next_id()
+      {
         return _table.Count.ToString();
      }

-      private void _publish_next_packet() {
+      private void _publish_next_packet()
+      {
         var bytes = PayloadMessage.create_data(_payload_sz, _payload_byte);
         _last_sent_msg = new PayloadMessage(_get_next_id(), epoch_now(), bytes);
         var ser_bytes = _last_sent_msg.serialize();
@@ -160,19 +175,25 @@ namespace bts {
         _last_publish_time = DateTime.Now;
      }

-      public void doWork() {
-         Func<IEnumerable<PubSubServerMessage>, PayloadMessage> find_packet = (updates) => {
-            foreach (var u in updates) {
+      public void doWork()
+      {
+         Func<IEnumerable<PubSubServerMessage>, PayloadMessage> find_packet = (updates) =>
+         {
+            foreach (var u in updates)
+            {
               var recv_msg = PayloadMessage.deserialize(u.data);
-               if (recv_msg.msg_id == _last_sent_msg.msg_id) {
+               if (recv_msg.msg_id == _last_sent_msg.msg_id)
+               {
                  return recv_msg;
               }
            }
            return null;
         };
-         switch (_state) {
+         switch (_state)
+         {
            case IOState.SEND:
-               if (DateTime.Now - _last_publish_time >= _publish_period) {
+               if (DateTime.Now - _last_publish_time >= _publish_period)
+               {
                  _publish_next_packet();
                  _last_publish_time = DateTime.Now;
                  _fire_dp_after_send();
@@ -184,7 +205,8 @@ namespace bts {
               var updates = _sub_core.get_updates(TimeSpan.Zero);
               updates = updates.Where(u => u.tag == _tag_name);
               var recv_msg = find_packet(updates);
-               if (recv_msg != null) {
+               if (recv_msg != null)
+               {
                  _last_recvd_msg = recv_msg;
                  _fire_dp_after_recv();
                  _state = IOState.SEND;
@@ -194,7 +216,8 @@ namespace bts {
         }
      }

-      public static long epoch_now() {
+      public static long epoch_now()
+      {
         return DateTimeOffset.UtcNow.ToUnixTimeMilliseconds();
      }

@@ -7,122 +7,127 @@ using MongoDB.Driver;
 using System;
 using System.Collections.Generic;
 using System.Diagnostics;
+using System.Threading.Tasks;
+using System.Threading;

 namespace bts {
-    [DataStreamUserRespresentation("mongodb_sql")]
-    public class MongoDBSQLStream : ViewStream, IWorker {
-        private ViewStreamDatabase _vsd;
-        private readonly string _connection_string, _db_name, _collection_name;
-        private MongoClient _client;
-        // make this a sorted table
-        private BlueTable _table;
+    // [DataStreamUserRespresentation("mongodb_sql")]
+    // public class MongoDBSQLStream : ViewStream, IWorker {
+    //     private ViewStreamDatabase _vsd;
+    //     private readonly string _connection_string, _db_name, _collection_name;
+    //     private MongoClient _client;
+    //     // make this a sorted table
+    //     private BlueTable _table;


-        // Access the database
-        private IMongoDatabase _database;
-        private IDictionary<string, int> _id_map = new Dictionary<string, int>();
-        // mongodb://192.168.1.80:27017
+    //     // Access the database
+    //     private IMongoDatabase _database;
+    //     private IDictionary<string, int> _id_map = new Dictionary<string, int>();
+    //     // mongodb://192.168.1.80:27017

-        public MongoDBSQLStream(string connection_string, string db_name, string collection_name)
-        : this(connection_string, db_name, collection_name, ViewStreamDatabase.instance) {
+    //     public MongoDBSQLStream(string connection_string, string db_name, string collection_name)
+    //     : this(connection_string, db_name, collection_name, ViewStreamDatabase.instance) {

-        }
-        public MongoDBSQLStream(string connection_string, string db_name, string collection_name, ViewStreamDatabase vsd) {
-            bts.Log.Gen.info($"MongoDBStream constructed connection_string: {connection_string} " +
-                                    $"db_name: {db_name}");
-            _connection_string = connection_string;
-            _db_name = db_name;
-            _collection_name = collection_name;
-            _vsd = vsd;
-            _client = new MongoClient(_connection_string);
-            _database = _client.GetDatabase(_db_name);
-        }
+    //     }
+    //     public MongoDBSQLStream(string connection_string, string db_name, string collection_name, ViewStreamDatabase vsd) {
+    //         bts.Log.Gen.info($"MongoDBStream constructed connection_string: {connection_string} " +
+    //                                 $"db_name: {db_name}");
+    //         _connection_string = connection_string;
+    //         _db_name = db_name;
+    //         _collection_name = collection_name;
+    //         _vsd = vsd;
+    //         _client = new MongoClient(_connection_string);
+    //         _database = _client.GetDatabase(_db_name);
+    //     }

-        protected override void myOnFirstSubscriber() {
-            MasterWorker.getInstance(false).addWorker(this, TimeSpan.FromSeconds(10));
-        }
+    //     protected override void myOnFirstSubscriber() {
+    //         MasterWorker.getInstance(false).addWorker(this, TimeSpan.FromSeconds(10));
+    //     }

-        protected override void myOnLastSubscriber() {
-            MasterWorker.getInstance(false).removeWorker(this);
-        }
+    //     protected override void myOnLastSubscriber() {
+    //         MasterWorker.getInstance(false).removeWorker(this);
+    //     }

-        public void doWork() {
-            _read_mogodb();
-        }
+    //     public void doWork() {
+    //         _read_mogodb();
+    //     }

-        private void _read_mogodb() {
-            var timer = Stopwatch.StartNew();
-            // Access the collection
-            IMongoCollection<BsonDocument> collection = _database.GetCollection<BsonDocument>(name: _collection_name);
+    //     private void _read_mogodb() {
+    //         var timer = Stopwatch.StartNew();
+    //         // Access the collection
+    //         IMongoCollection<BsonDocument> collection = _database.GetCollection<BsonDocument>(name: _collection_name);

-            // Fetch all documents from the collection
-            var documents = collection.Find(Builders<BsonDocument>.Filter.Empty).ToCursor();
-            _table = _table == null ? null : new BlueTable(_table.schema);
-            // Iterate through the documents and print values
-            foreach (var document in documents.ToEnumerable()) {
-                bts.Log.Gen.info($"document: {document} json: {document.ToJson()}");
-                if (_table == null) {
-                    _init_table(document);
-                }
-                _add_row(document);
-            }
-            timer.Stop();
-            Log.Gen.info($"MongoDBStream time to read {_collection_name} {timer.ElapsedMilliseconds}ms");
-            timer.Restart();
-            var dp = BlueDataUtility.build_dp_inc(_table, myLastValue == null ? null : myLastValue as DataPacket, true);
-            timer.Stop();
-            Log.Gen.info($"MongoDBStream time to compare tables :{timer.ElapsedMilliseconds}ms");
-            if (dp.addedRows.Count == 0 && dp.deletedRows.Count == 0)
-                return;
-            myFireEvent(dp);
-        }
+    //         // Fetch all documents from the collection
+    //         var documents = collection.Find(Builders<BsonDocument>.Filter.Empty).ToCursor();
+    //         _table = _table == null ? null : new BlueTable(_table.schema);
+    //         // Iterate through the documents and print values
+    //         foreach (var document in documents.ToEnumerable()) {
+    //             bts.Log.Gen.info($"document: {document} json: {document.ToJson()}");
+    //             if (_table == null) {
+    //                 _init_table(document);
+    //             }
+    //             _add_row(document);
+    //         }
+    //         timer.Stop();
+    //         Log.Gen.info($"MongoDBStream time to read {_collection_name} {timer.ElapsedMilliseconds}ms");
+    //         timer.Restart();
+    //         var dp = BlueDataUtility.build_dp_inc(_table, myLastValue == null ? null : myLastValue as DataPacket, true);
+    //         timer.Stop();
+    //         Log.Gen.info($"MongoDBStream time to compare tables :{timer.ElapsedMilliseconds}ms");
+    //         if (dp.addedRows.Count == 0 && dp.deletedRows.Count == 0)
+    //             return;
+    //         myFireEvent(dp);
+    //     }

-        private void _init_table(BsonDocument doc_row) {
-            bts.Log.assert(_table == null);
-            List<ColumnInfo> cis = new List<ColumnInfo>();
-            cis.Add((new ColumnInfo("id", 0, typeof(Int32))));
-            foreach (var n in doc_row.Names) {
-                cis.Add(new ColumnInfo(n, cis.Count, typeof(string)));
-            }
-            IBlueTableSchema schema = new BlueTableSchema(cis.ToArray(), _collection_name);
-            _table = new BlueTable(schema);
-        }
+    //     private void _init_table(BsonDocument doc_row) {
+    //         bts.Log.assert(_table == null);
+    //         List<ColumnInfo> cis = new List<ColumnInfo>();
+    //         cis.Add((new ColumnInfo("id", 0, typeof(Int32))));
+    //         foreach (var n in doc_row.Names) {
+    //             cis.Add(new ColumnInfo(n, cis.Count, typeof(string)));
+    //         }
+    //         IBlueTableSchema schema = new BlueTableSchema(cis.ToArray(), _collection_name);
+    //         _table = new BlueTable(schema);
+    //     }

-        private void _add_row(BsonDocument doc) {
-            var key_val = doc["_id"];
-            var key = key_val.ToString();
-            int id;
-            if (!_id_map.TryGetValue(key, out id)) {
-                id = _id_map.Count;
-                _id_map.Add(key, id);
-            }
-            MutableBlueRow nr = new MutableBlueRow(_table.schema);
-            nr.setValue("id", id);
-            foreach (var n in doc.Names) {
-                var val = doc[n].ToString();
-                nr.setValue(n, val);
-            }
-            _table.addRow(nr);
-        }
-    }
+    //     private void _add_row(BsonDocument doc) {
+    //         var key_val = doc["_id"];
+    //         var key = key_val.ToString();
+    //         int id;
+    //         if (!_id_map.TryGetValue(key, out id)) {
+    //             id = _id_map.Count;
+    //             _id_map.Add(key, id);
+    //         }
+    //         MutableBlueRow nr = new MutableBlueRow(_table.schema);
+    //         nr.setValue("id", id);
+    //         foreach (var n in doc.Names) {
+    //             var val = doc[n].ToString();
+    //             nr.setValue(n, val);
+    //         }
+    //         _table.addRow(nr);
+    //     }
+    // }

    [DataStreamUserRespresentation("mongodb")]
    public class MongoDBStream : ViewStream, IWorker {
        private ViewStreamDatabase _vsd;
        private readonly string _connection_string, _db_name, _collection_name;
+        private readonly TimeSpan _poll_period;
        private MongoClient _client;
        private BlueTable _table;
+        private SyncQueue<DataPacket> _table_queue = new SyncQueue<DataPacket>();
+        private BlockingQueue _work_queue = new BlockingQueue();

        // Access the database
        private IMongoDatabase _database;
        private IDictionary<string, int> _id_map = new Dictionary<string, int>();
        // mongodb://192.168.1.80:27017

-        public MongoDBStream(string connection_string, string db_name, string collection_name)
-        : this(connection_string, db_name, collection_name, ViewStreamDatabase.instance) {
+        public MongoDBStream(string connection_string, string db_name, string collection_name, string poll_period)
+        : this(connection_string, db_name, collection_name, TimeBasedFilter.parse_duration(poll_period), ViewStreamDatabase.instance) {

        }
-        public MongoDBStream(string connection_string, string db_name, string collection_name, ViewStreamDatabase vsd) {
+        public MongoDBStream(string connection_string, string db_name, string collection_name, TimeSpan poll_period, ViewStreamDatabase vsd) {
            bts.Log.Gen.info($"MongoDBStream constructed connection_string: {connection_string} " +
                                    $"db_name: {db_name}");
            _connection_string = connection_string;
@@ -130,19 +135,30 @@ namespace bts {
            _collection_name = collection_name;
            _vsd = vsd;
            _client = new MongoClient(_connection_string);
+            _poll_period = poll_period;
            _database = _client.GetDatabase(_db_name);
        }

        protected override void myOnFirstSubscriber() {
-            MasterWorker.getInstance(false).addWorker(this, TimeSpan.FromSeconds(10));
+            _vsd.addWorker(this);
+            //MasterWorker.getInstance(false).addWorker(this, _poll_period);
        }

        protected override void myOnLastSubscriber() {
-            MasterWorker.getInstance(false).removeWorker(this);
+            _vsd.removeWorker(this);
+            //MasterWorker.getInstance(false).removeWorker(this);
        }

+        private DateTime _last_poll_time = DateTime.MinValue;
        public void doWork() {
-            _read_mogodb();
+            var now = DateTime.Now;
+            if (now - _last_poll_time > _poll_period) {
+                ThreadPool.QueueUserWorkItem(_ => _read_mogodb());
+                _last_poll_time = DateTime.Now;
+            }
+            var dp = _table_queue.dequeue(0);
+            if (dp != null)
+                myFireEvent(dp);
        }

        private void _read_mogodb() {
@@ -166,10 +182,11 @@ namespace bts {
            timer.Restart();
            var dp = BlueDataUtility.build_dp_inc(_table, myLastValue == null ? null : myLastValue as DataPacket, true);
            timer.Stop();
-            Log.Gen.info($"MongoDBStream time to compare tables: {timer.ElapsedMilliseconds}ms");;
+            Log.Gen.info($"MongoDBStream time to compare tables: {timer.ElapsedMilliseconds}ms"); ;
            if (dp.addedRows.Count == 0 && dp.deletedRows.Count == 0)
                return;
-            myFireEvent(dp);
+            _table_queue.enqueue(dp);
+            //myFireEvent(dp);
        }

        private void _init_table(BsonDocument doc_row) {
@@ -208,46 +225,46 @@ namespace bts {
        }
    }

-    [DataStreamUserRespresentation("mongodb_flat")]
-    public class MongoDBFlatStream : ViewStream {
-        public MongoDBFlatStream(string connection_string, string db_name, string collection_name, string timestamp_col, string ttl_duration)
-        :
-        this(ViewStreamDatabase.instance, connection_string, db_name, collection_name, timestamp_col, TimeBasedFilter.parse_duration(ttl_duration)) { }
+    // [DataStreamUserRespresentation("mongodb_flat")]
+    // public class MongoDBFlatStream : ViewStream {
+    //     public MongoDBFlatStream(string connection_string, string db_name, string collection_name, string timestamp_col, string ttl_duration)
+    //     :
+    //     this(ViewStreamDatabase.instance, connection_string, db_name, collection_name, timestamp_col, TimeBasedFilter.parse_duration(ttl_duration)) { }

-        public MongoDBFlatStream(ViewStreamDatabase vsd, string connection_string, string db_name, string collection_name, string timestamp_col, TimeSpan ttl_duration) {
-            ViewStream rs = new MongoDBStream(connection_string, db_name, collection_name, vsd);
-            rs = new KeyValueToRelationalStream(rs, "doc_id", "key", "val");
-            rs = new DateStandardizationStream(rs, timestamp_col, DateConversion.ISO8601);
-            //rs = new TimeBasedFilterStream(vsd, rs, timestamp_col, ttl_duration);
-            mySetRealSource(rs);
-        }
-    }
+    //     public MongoDBFlatStream(ViewStreamDatabase vsd, string connection_string, string db_name, string collection_name, string timestamp_col, TimeSpan ttl_duration) {
+    //         ViewStream rs = new MongoDBStream(connection_string, db_name, collection_name, vsd);
+    //         rs = new KeyValueToRelationalStream(rs, "doc_id", "key", "val");
+    //         rs = new DateStandardizationStream(rs, timestamp_col, DateConversion.ISO8601);
+    //         //rs = new TimeBasedFilterStream(vsd, rs, timestamp_col, ttl_duration);
+    //         mySetRealSource(rs);
+    //     }
+    // }



-    [DataStreamUserRespresentation("newsreplay")]
-    public class NewsReplayStream : ViewStream {
-        public NewsReplayStream(string connection_string, string db_name, string collection_name,
-                                int buffer_sz, string fire_period) :
-        this(ViewStreamDatabase.instance, connection_string, db_name, collection_name, buffer_sz, TimeBasedFilter.parse_duration(fire_period)) { }
+    // [DataStreamUserRespresentation("newsreplay")]
+    // public class NewsReplayStream : ViewStream {
+    //     public NewsReplayStream(string connection_string, string db_name, string collection_name,
+    //                             int buffer_sz, string fire_period) :
+    //     this(ViewStreamDatabase.instance, connection_string, db_name, collection_name, buffer_sz, TimeBasedFilter.parse_duration(fire_period)) { }

-        public NewsReplayStream(ViewStreamDatabase vsd, string connection_string, string db_name, string collection_name,
-                                int buffer_sz, TimeSpan fire_period) {
-            ViewStream r;
+    //     public NewsReplayStream(ViewStreamDatabase vsd, string connection_string, string db_name, string collection_name,
+    //                             int buffer_sz, TimeSpan fire_period) {
+    //         ViewStream r;

-            r = new MongoDBStream(connection_string, db_name, collection_name, vsd);
-            r = new KeyValueToRelationalStream(r,
-                                                "doc_id",
-                                                "key",
-                                                "value");
-            r = new DateStandardizationStream(r,
-                                               "created_at",
-                                               "ISO8601");
-            r = new SnapshotReplayStream(vsd, r, "created_at", buffer_sz, fire_period);
+    //         r = new MongoDBStream(connection_string, db_name, collection_name, vsd);
+    //         r = new KeyValueToRelationalStream(r,
+    //                                             "doc_id",
+    //                                             "key",
+    //                                             "value");
+    //         r = new DateStandardizationStream(r,
+    //                                            "created_at",
+    //                                            "ISO8601");
+    //         r = new SnapshotReplayStream(vsd, r, "created_at", buffer_sz, fire_period);

-            mySetRealSource(r);
-        }
-    }
+    //         mySetRealSource(r);
+    //     }
+    // }


 }
@@ -6,6 +6,7 @@ namespace bts {
   using System;
   using System.Collections;
   using System.Collections.Generic;
+   using System.Linq;

   public class ProjectData {
      // schema from the base stream
@@ -147,12 +148,23 @@ namespace bts {
               mapped_to_orig_map.Add(mapped_col, orig_col);
               orig_to_mapped.Add(orig_col, mapped_col);
            } else {
-               Log.Gen.error($"NewProjectStream unexpecetd token: {p_tok}");
+               if (input_schema.columnNames.Contains(p_tok)) {
+                  string orig_col = p_tok;
+                  string mapped_col = orig_col;
+                  orig_to_mapped.Add(orig_col, mapped_col);
+               }
            }
         }
         // make the new schema
         List<ColumnInfo> new_cols = new List<ColumnInfo>();
         foreach (ColumnInfo ci in input_schema.columnInfo) {
+            if (ci.colName == "id") {
+               // if there's a hidden col, other columns will show by default, but if cols are being
+               // projected epxlicitely, than others wil be hidden
+               ColumnInfo nc = new ColumnInfo(ci.colName, new_cols.Count, ci.colType);
+               new_cols.Add(nc);
+               continue;
+            }
            if (removed_cols.Contains(ci.colName)) {
               continue;
            } else if (orig_to_mapped.ContainsKey(ci.colName)) {
@@ -160,8 +172,12 @@ namespace bts {
               ColumnInfo nc = new ColumnInfo(mapped_name, new_cols.Count, ci.colType);
               new_cols.Add(nc);
            } else {
-               ColumnInfo nc = new ColumnInfo(ci.colName, new_cols.Count, ci.colType);
-               new_cols.Add(nc);
+               if (ci.colName == "id" || removed_cols.Count > 0) {
+                  // if there's a hidden col, other columns will show by default, but if cols are being
+                  // projected epxlicitely, than others wil be hidden
+                  ColumnInfo nc = new ColumnInfo(ci.colName, new_cols.Count, ci.colType);
+                  new_cols.Add(nc);
+               }
            }
         }
         IBlueTableSchema new_schema = new BlueTableSchema(input_schema.tableName, new_cols.ToArray());
@@ -1,11 +1,11 @@
 namespace Bts.DataStream.Services;

 using BCG;
-using BCG.Utilities.General;

 using Microsoft.Extensions.DependencyInjection;

 public static class DataStreamServices {
   public static IServiceCollection AddRemoteQueryManager(this IServiceCollection services) =>
-      services.AddSingleton<IWorker, RemoteQueryManager>();
+      services.AddSingleton<RemoteQueryManager>()
+         .AddHostedService(sp => sp.GetRequiredService<RemoteQueryManager>());
 }
@@ -1783,6 +1783,18 @@ namespace BCG {
         #endregion
      }

+      public static IBlueRow find_max(IEnumerable<IBlueRow> rows, params string[] cols_to_compare) {
+         if (rows == null || rows.Count() == 0)
+            return null;
+         IBlueRow max_row = rows.ElementAt(0);
+         foreach (IBlueRow r in rows) {
+            int c = compare_rows(r, max_row, cols_to_compare);
+            if (c > 0)
+               max_row = r;
+         }
+         return max_row;
+      }
+
      public static int compare_rows(IBlueRow r1, IBlueRow r2, Dictionary<string, bool> sort_map) {
         foreach (KeyValuePair<string, bool> p in sort_map) {
            int comp = compare_send_dontcare_to_last(r1, p.Key, r2, p.Key, p.Value);
@@ -115,10 +115,16 @@ public class DataPacketJsonConverter : JsonConverter<DataPacket> {
            return NullCheckWrapper(v => writer.WriteNumberValue((ushort)v));
         }
         if (t == typeof(double)) {
-            return NullCheckWrapper(v => JsonSerializer.Serialize(writer, (double)v, options));
+            return NullCheckWrapper(v => {
+               var dbl = Equals(v, DBNull.Value) ? double.NaN : (double)v;
+               JsonSerializer.Serialize(writer, dbl, options);
+            });
         }
         if (t == typeof(float)) {
-            return NullCheckWrapper(v => writer.WriteNumberValue((float)v));
+            return NullCheckWrapper(v => {
+               var flt = Equals(v, DBNull.Value) ? float.NaN : (float)v;
+               writer.WriteNumberValue(flt);
+            });
         }
         if (t == typeof(bool)) {
            return NullCheckWrapper(v => writer.WriteBooleanValue((bool)v));
@@ -32,6 +32,9 @@ namespace BCG
      public DataStream (DataStream realSource)
      {
         myRealSource = realSource;
+         if (myRealSource is not null) {
+            myRealSource.ErrorChanged += OnError;
+         }
         myOnChangeDelegate = null;
         myPendingAdds = new Stack();
      }
@@ -48,6 +51,8 @@ namespace BCG
         }
      }

+      public event Action<Exception> ErrorChanged;
+
      public delegate void ChangeDelegate(object newValue);
      public event ChangeDelegate onChange
      {
@@ -120,8 +125,15 @@ namespace BCG
      protected virtual object myGetValueForInitialSubscription()
      {return myLastValue;}

-      protected void mySetRealSource (DataStream src)
-      {myRealSource = src;}
+      protected void mySetRealSource (DataStream src) {
+         if (myRealSource is not null) {
+            myRealSource.ErrorChanged -= OnError;
+         }
+         myRealSource = src;
+         if (myRealSource is not null) {
+            myRealSource.ErrorChanged += OnError;
+         }
+      }

      protected virtual void myOnFirstSubscriber() {myFirstSubImplemented = false;}
      protected virtual void myOnLastSubscriber () {myLastSubImplemented = false;}
@@ -143,6 +155,11 @@ namespace BCG
         }
      }

+      protected void OnError(Exception error) {
+         Log.Gen.error($"{GetType()} OnError: {error}");
+         ErrorChanged?.Invoke(error);
+      }
+
      public Guid ID
      {
         get
@@ -8,7 +8,7 @@

  <ItemGroup>
    <PackageReference Include="Google.Protobuf" Version="3.24.3" />
-    <PackageReference Include="Microsoft.Extensions.Hosting" Version="8.0.0" />
+    <PackageReference Include="Microsoft.Extensions.Hosting" Version="7.0.1" />
  </ItemGroup>

  <ItemGroup>
				`@@ -0,0 +1 @@`
				`VITE_DATASTREAM_WEBSOCKET_URL=wss://localhost:443`