CLEANUP: BYBIT Symbols.conf.

Originally, this spark.conf was hardcoded into compose.yaml, and copied
into the sparkd container on each deploy. It hasn't been used since
config profiles were introduced.

.config/dotnet-tools.json

@@ -9,7 +9,7 @@
       ]
     },
     "fable": {
-      "version": "4.0.6",
+      "version": "4.24.0",
       "commands": [
         "fable"
       ]
@@ -27,4 +27,4 @@
       ]
     }
   }
-}
+}

.devcontainer/Dockerfile

@@ -65,7 +65,7 @@ RUN \
    && apt-get install -y cmake
 
 # Dotnet
-RUN apt-get update && apt-get install -y dotnet7
+RUN apt-get update && apt-get install -y dotnet8
 
 # Node
 ARG node_major=18
@@ -90,6 +90,12 @@ RUN apt-get update && apt-get install -y libpcap0.8-dev
 # Databento (deps)
 RUN apt-get update && apt-get -y install libzstd-dev
 
+# Docker in Docker
+RUN apt-get update && apt-get install -y \
+    docker.io \
+    docker-compose \
+    docker-compose-v2
+
 # Set up default executables (update-alternatives)
 RUN \
    update-alternatives --install /usr/bin/clang clang /usr/bin/clang-$clang_major 100; \

.devcontainer/devcontainer.json

@@ -1,15 +1,10 @@
 {
   "dockerFile": "Dockerfile",
-  // Creates local dir to mount to HOME in docker 
-  "initializeCommand": "mkdir -p ${localEnv:HOME}/.bts/devcontainer-mounts/${devcontainerId}",
   "hostRequirements": {
-    "memory": "4gb",
-    "storage": "6gb"
+    "memory": "8gb",
+    "storage": "10gb"
   },
-  "postCreateCommand": "/home/bts/spark/docker/sparkd/dev/devcontainer_post_create.sh",
   "containerUser": "bts",
-  "workspaceMount": "source=${localWorkspaceFolder},target=/home/bts/spark,type=bind",
-  "workspaceFolder": "/home/bts/spark",
   "updateRemoteUserUID": true,
   "shutdownAction": "none",
   // Note: If running multiple containers on the same host, each must use a
@@ -17,34 +12,20 @@
   // example, if 8000-8003 is taken, go with 9000-9003, then 10000-10003, etc.
   "runArgs": [
     "--shm-size",
-    "2gb",
-    "-p",
-    "8000:8000",
-    "-p",
-    "8001:8001",
-    "-p",
-    "8002:8002",
-    "-p",
-    "8003:8003",
-    "-p",
-    "12345:12345"
+    "8gb"
   ],
   "mounts": [
-    "source=${localEnv:HOME}/.bts/devcontainer-mounts/${devcontainerId},target=/home/bts,type=bind"
+    "source=/var/run/docker.sock,target=/var/run/docker.sock,type=bind"
   ],
   "customizations": {
     "vscode": {
-      "settings": {
-        "clangd.arguments": [
-          "--compile-commands-dir=${workspaceFolder}/build/clang/debug"
-        ]
-      },
       "extensions": [
         "eamodio.gitlens",
         "llvm-vs-code-extensions.vscode-clangd",
         "ms-vscode.cmake-tools",
-        "ms-vscode.cpptools"
+        "ms-vscode.cpptools",
+        "xaver.clang-format"
       ]
     }
   }
-}
+}

.env

@@ -0,0 +1,48 @@
+# This file defines the global default configuration for docker deployments.
+# It is tracked in source control. For temporary modifications, please create
+# a separate file called ".env.local" (it's in .gitignore, so won't be tracked).
+# If a variable is defined in both files, the value in .env.local will be used.
+#
+# About the deployment procedure:
+#   We use Docker Compose to build images and launch containers for various services
+#   that make up our software suite:
+#     - sparkd
+#     - TODO: pubsub
+#     - TODO: md-publisher
+#     - TODO: webapp (DSQA on the web)
+#     - TODO: api-server (websocket based)
+#   From inside a devcontainer, you can launch all services in separate containers
+#   on your machine by running:
+#
+#     tools/deploy.sh
+#
+#   This script builds release versions for each service, packaged (generally) in
+#   self-contained directories. Next, it runs a few "docker compose" commands to
+#   build docker images loaded with the self-contained binaries from the previous
+#   step, and start containers with those images to run each service. To view the
+#   commands that deploy.sh runs internally (without actually running them), run:
+#
+#     tools/deploy.sh --dry-run
+#
+#   For additional information/help, run:
+#
+#     tools/deploy.sh --help
+#
+
+### Sparkd ###
+# The internal image for sparkd to use. For more info, view:
+#   src/server/sparkd/Dockerfile
+SPARKD__IMAGE=sparkd
+# The config dir for this service to use
+SPARKD__CONFIG_DIR=src/server/sparkd/configurations/dev
+# Mount a specific volume or dir to "~/sparkdata/data" inside the docker.
+# Defaults to a volume named "sparkd_<config_port>" (e.g., sparkd_8000).
+# SPARKD__DATA_DIR=/home/bts/sparkdata/data
+SPARKD__SOURCE_DIR=install/sparkd-release-gcc
+SPARKD__RUNTIME_CONF_DIR=/home/ubuntu/8000/sparkdata/conf/
+SPARKD__CONFIG_PORT=8000
+SPARKD__QUICKBUS_PORT=8001
+SPARKD__SIM_FIX_PORT=8002
+SPARKD__SHM_SIZE=4gb
+
+# TODO: add config for other services (pubsub, md-publisher, webapp, api-server, etc.)

.gitignore

@@ -59,3 +59,5 @@ trace_data/
 .devcontainer/
 *.Generated.cs
 *.lst
+.env.local
+.secrets

.gitmodules

@@ -1,6 +1,6 @@
 [submodule "externals"]
 	path = externals
-	url = git@bitbucket.org:bluetradesys/externals.git
+	url = https://bitbucket.org/bluetradesys/externals.git
 [submodule "vcpkg"]
 	path = vcpkg
 	url = https://github.com/microsoft/vcpkg.git

.vscode/launch.json

@@ -1,6 +1,14 @@
 {
   "version": "0.2.0",
   "configurations": [
+    {
+      "name": "Python Debugger: Current File with Arguments",
+      "type": "debugpy",
+      "request": "launch",
+      "program": "${file}",
+      "console": "integratedTerminal",
+      "args": "${command:pickArgs}"
+    },
     {
       "name": ".NET Core Attach",
       "type": "coreclr",
@@ -22,7 +30,7 @@
         "--config",
         "${workspaceFolder}/src/DataStreamSuite/DataStreamPublisher/Configurations/${input:dataStreamPublisherConfig}.json"
       ],
-      "cwd": "build/Debug/bin/net7.0",
+      "cwd": "build/Debug/bin/net8.0",
       "stopAtEntry": false,
       "console": "integratedTerminal"
     },
@@ -31,7 +39,39 @@
       "type": "cppdbg",
       "request": "launch",
       "program": "/home/bts/spark/build/sparkd-debug-clang/server/sparkd",
-      "args": ["run"],
+      "args": [
+        "run"
+      ],
+      "stopAtEntry": false,
+      "cwd": "/home/bts/spark/build/sparkd-debug-clang/server",
+      "environment": [],
+      "externalConsole": false,
+      "MIMode": "gdb",
+    },
+    {
+      "name": "spreader tests2",
+      "type": "cppdbg",
+      "request": "launch",
+      "program": "/home/bts/spark/build/sparkd-debug-clang/server/sparkd",
+      "args": [
+        "test",
+        "[spreader2]"
+      ],
+      "stopAtEntry": false,
+      "cwd": "/home/bts/spark/build/sparkd-debug-clang/server",
+      "environment": [],
+      "externalConsole": false,
+      "MIMode": "gdb",
+    },
+    {
+      "name": "spreader tests",
+      "type": "cppdbg",
+      "request": "launch",
+      "program": "/home/bts/spark/build/sparkd-debug-clang/server/sparkd",
+      "args": [
+        "test",
+        "[spreader]"
+      ],
       "stopAtEntry": false,
       "cwd": "/home/bts/spark/build/sparkd-debug-clang/server",
       "environment": [],
@@ -50,7 +90,7 @@
           "--config",
           "${workspaceFolder}/src/DataStreamSuite/DataStreamQueryAnalyzer/Configurations/DataStreamQueryAnalyzer.json"
         ],
-        "cwd": "build/Debug/bin/net7.0-windows",
+        "cwd": "build/Debug/bin/net8.0-windows",
         "env": {
           "PYTHON_EXE": "${workspaceFolder}/.venv/Scripts/python.exe"
         },
@@ -74,7 +114,7 @@
         "--config",
         "${workspaceFolder}/src/DataStreamSuite/PubSubServer/Configurations/${input:pubSubServerConfig}.json"
       ],
-      "cwd": "build/Debug/bin/net7.0",
+      "cwd": "build/Debug/bin/net8.0",
       "stopAtEntry": false,
       "console": "integratedTerminal"
     },
@@ -86,7 +126,19 @@
       "windows": {
         "preLaunchTask": "Build Spark (Debug)",
         "program": "Spark.exe",
-        "cwd": "build/Debug/bin/net7.0-windows",
+        "cwd": "build/Debug/bin/net8.0-windows",
+      },
+      "stopAtEntry": false,
+      "console": "integratedTerminal"
+    },
+    {
+      "name": "Spark (Skip Build)",
+      "type": "coreclr",
+      "request": "launch",
+      "program": "",
+      "windows": {
+        "program": "Spark.exe",
+        "cwd": "build/Debug/bin/net8.0-windows",
       },
       "stopAtEntry": false,
       "console": "integratedTerminal"
@@ -103,27 +155,100 @@
           "--sparkd",
           "${input:sparkConfigServer}"
         ],
-        "cwd": "build/Debug/bin/net7.0-windows",
+        "cwd": "build/Debug/bin/net8.0-windows",
+      },
+      "stopAtEntry": false,
+      "console": "integratedTerminal"
+    },
+    {
+      "name": "Spark Test",
+      "type": "coreclr",
+      "request": "launch",
+      "program": "",
+      "windows": {
+        "preLaunchTask": "Build Spark (Debug)",
+        "program": "Spark.Console.exe",
+        "cwd": "build/Debug/bin/net8.0",
+        "args": [
+          "test",
+          "${input:sparkTest}"
+        ]
+      },
+      "stopAtEntry": false,
+      "console": "integratedTerminal"
+    },
+    {
+      "name": "Spark Test (Skip Build)",
+      "type": "coreclr",
+      "request": "launch",
+      "program": "",
+      "windows": {
+        "program": "Spark.Console.exe",
+        "cwd": "build/Debug/bin/net8.0",
+        "args": [
+          "test",
+          "${input:sparkTest}"
+        ]
       },
       "stopAtEntry": false,
       "console": "integratedTerminal"
     },
     {
       "name": "Sparkd",
-      "type": "coreclr",
+      "type": "cppdbg",
       "request": "launch",
-      "preLaunchTask": "Build Sparkd (Debug | Clang)",
       "program": "",
       "linux": {
-        "program": "sparkd",
+        "program": "${workspaceFolder}/build/sparkd-debug-clang/src/app_sparkd/sparkd",
       },
       "args": [
         "run"
       ],
-      "cwd": "build/sparkd-debug-clang/server",
+      "cwd": "${workspaceFolder}/build/sparkd-debug-clang/src/app_sparkd",
       "stopAtEntry": false,
-      "console": "integratedTerminal"
-    }
+      "externalConsole": false,
+      "MIMode": "gdb",
+      "setupCommands": [
+        {
+          "description": "Enable pretty-printing for gdb",
+          "text": "-enable-pretty-printing",
+          "ignoreFailures": true
+        },
+        {
+          "description": "Set Disassembly Flavor to Intel",
+          "text": "-gdb-set disassembly-flavor intel",
+          "ignoreFailures": true
+        }
+      ]
+    },
+    {
+      "name": "Sparkd (Skip Build)",
+      "type": "cppdbg",
+      "request": "launch",
+      "program": "",
+      "linux": {
+        "program": "${workspaceFolder}/build/sparkd-debug-clang/src/app_sparkd/sparkd",
+      },
+      "args": [
+        "run"
+      ],
+      "cwd": "${workspaceFolder}/build/sparkd-debug-clang/src/app_sparkd",
+      "stopAtEntry": false,
+      "externalConsole": false,
+      "MIMode": "gdb",
+      "setupCommands": [
+        {
+          "description": "Enable pretty-printing for gdb",
+          "text": "-enable-pretty-printing",
+          "ignoreFailures": true
+        },
+        {
+          "description": "Set Disassembly Flavor to Intel",
+          "text": "-gdb-set disassembly-flavor intel",
+          "ignoreFailures": true
+        }
+      ]
+    },
   ],
   "inputs": [
     {
@@ -154,5 +279,15 @@
       "description": "Config Server URL",
       "default": "http://localhost:8000/config",
     },
+    {
+      "id": "sparkTest",
+      "type": "promptString",
+      "description": "Test to run",
+    },
+    {
+      "id": "sparkdTest",
+      "type": "promptString",
+      "description": "Test to run",
+    },
   ]
-}
+}

.vscode/tasks.json

@@ -641,6 +641,21 @@
         "color": "terminal.ansiGreen",
       }
     },
+    {
+      "label": "Run Spark (Skip Build)",
+      "type": "process",
+      "command": "",
+      "windows": {
+        "command": "Spark.exe",
+        "options": {
+          "cwd": "${workspaceFolder}/build/${input:buildConfiguration}/bin/net7.0-windows"
+        },
+      },
+      "icon": {
+        "id": "run",
+        "color": "terminal.ansiGreen",
+      }
+    },
     {
       "label": "Run Spark Config Query",
       "dependsOn": "Build Spark",
@@ -684,6 +699,24 @@
         "color": "terminal.ansiGreen",
       },
     },
+    {
+      "label": "Run Sparkd (Skip Build)",
+      "type": "process",
+      "command": "",
+      "linux": {
+        "command": "sparkd",
+      },
+      "args": [
+        "run"
+      ],
+      "options": {
+        "cwd": "${workspaceFolder}/build/${input:sparkdCmakePreset}/server"
+      },
+      "icon": {
+        "id": "run",
+        "color": "terminal.ansiGreen",
+      },
+    },
     {
       "label": "Run Sparkd Config Query",
       "dependsOn": "Build Sparkd",
@@ -705,6 +738,65 @@
         "color": "terminal.ansiGreen",
       },
     },
+    {
+      "label": "Test Spark (Debug)",
+      "dependsOn": "Build Spark (Debug)",
+      "type": "process",
+      "icon": {
+        "id": "coverage",
+        "color": "terminal.ansiYellow",
+      },
+      "command": "",
+      "windows": {
+        "command": "Spark.Console.exe",
+        "args": [
+          "test",
+          "${input:sparkTest}"
+        ],
+        "options": {
+          "cwd": "${workspaceFolder}/build/Debug/bin/net7.0",
+        }
+      },
+    },
+    {
+      "label": "Test Sparkd",
+      "dependsOn": "Build Sparkd",
+      "type": "process",
+      "command": "",
+      "linux": {
+        "command": "sparkd",
+      },
+      "args": [
+        "test",
+        "${input:sparkdTest}",
+      ],
+      "options": {
+        "cwd": "${workspaceFolder}/build/${input:sparkdCmakePreset}/server"
+      },
+      "icon": {
+        "id": "coverage",
+        "color": "terminal.ansiYellow",
+      },
+    },
+    {
+      "label": "Test Sparkd (Skip Build)",
+      "type": "process",
+      "command": "",
+      "linux": {
+        "command": "sparkd",
+      },
+      "args": [
+        "test",
+        "${input:sparkdTest}",
+      ],
+      "options": {
+        "cwd": "${workspaceFolder}/build/${input:sparkdCmakePreset}/server"
+      },
+      "icon": {
+        "id": "coverage",
+        "color": "terminal.ansiYellow",
+      },
+    },
     {
       "label": "View DataStreamQueryAnalyzer Logs (Last Run)",
       "type": "process",
@@ -823,6 +915,16 @@
       "id": "sparkdConfigQuery",
       "type": "promptString",
       "description": "Config Type(s)"
-    }
+    },
+    {
+      "id": "sparkTest",
+      "type": "promptString",
+      "description": "Test to run",
+    },
+    {
+      "id": "sparkdTest",
+      "type": "promptString",
+      "description": "Test to run",
+    },
   ]
 }

CMakeLists.txt

@@ -1,8 +1,10 @@
 cmake_minimum_required(VERSION 3.12)
 project(spark)
 include(GNUInstallDirs)
+include(${CMAKE_SOURCE_DIR}/cmake/GitVersion.cmake)
 set(CMAKE_CXX_STANDARD 20)
 
+get_version_from_git()
 if(WIN32)
   set(platform "windows")
 else()
@@ -20,9 +22,9 @@ set(
 # can run simultaneously without fighting over the same shm files (e.g., sparkd,
 # api server, etc.).
 set(SHARED_MEMORY_PREFIX "bts" CACHE STRING "Prefix to use when creating shared memory files")
-set(SPARK_BIN "${EXTERNALS_ROOT}/bin")
-set(SPARK_INCLUDE "${EXTERNALS_ROOT}/include")
-set(SPARK_LIB "${EXTERNALS_ROOT}/lib/$<CONFIG>")
+set(EXTERNALS_BIN "${EXTERNALS_ROOT}/bin")
+set(EXTERNALS_INCLUDE "${EXTERNALS_ROOT}/include")
+set(EXTERNALS_LIB "${EXTERNALS_ROOT}/lib/$<CONFIG>")
 set(BUILD_DOCS OFF CACHE BOOL "Build documentation")
 set(BUILD_SDK OFF CACHE BOOL "Build the sdk")
 set(EXPORT_MD_CONSUMER OFF CACHE BOOL "Export (extern) C functions for MD consumer interop")
@@ -34,8 +36,8 @@ else()
   set(BUILD_ADMIN_WEB_APP ON CACHE BOOL "Build the Admin WebApp")
 endif()
 
-include_directories(BEFORE SYSTEM ${SPARK_INCLUDE} ${EXTERNALS_ROOT}/include/zlib)
-link_directories(${SPARK_LIB})
+include_directories(BEFORE SYSTEM ${EXTERNALS_INCLUDE} ${EXTERNALS_ROOT}/include/zlib)
+link_directories(${EXTERNALS_LIB})
 
 add_compile_definitions(SHARED_MEMORY_PREFIX="${SHARED_MEMORY_PREFIX}")
 add_compile_definitions(APP_DATA_DIR="${APP_DATA_DIR}")
@@ -45,12 +47,8 @@ endif()
 add_compile_definitions(EXPORT_MD_CONSUMER="${EXPORT_MD_CONSUMER}")
 if(WIN32)
   # subdirs
-  add_subdirectory(src/lib)
+  add_subdirectory(src/lib_spark)
 else()
-  # make sparkd look for libspark.so in its directory
-  set(CMAKE_BUILD_WITH_INSTALL_RPATH TRUE)
-  set(CMAKE_INSTALL_RPATH ".")
-
   if(BUILD_SPARKD)
     # Note: All macros that begin w/ 'CCAPI' are ccapi's macros (defined and/or
     # read by their code). We override some below.
@@ -88,9 +86,15 @@ else()
     add_compile_definitions(BINANCE_US_PRICE_STATS_PATH="/api/v3/ticker/24hr")
 
     # ** Binance USDS Futures **
-    add_compile_definitions(CCAPI_BINANCE_USDS_FUTURES_URL_REST_BASE="https://dapi.binance.com")
-    add_compile_definitions(BINANCE_USDS_FUTURES_INSTRUMENTS_PATH="/dapi/v1/exchangeInfo")
-    add_compile_definitions(BINANCE_USDS_FUTURES_PRICE_STATS_PATH="/dapi/v1/ticker/24hr")
+    add_compile_definitions(CCAPI_BINANCE_USDS_FUTURES_URL_REST_BASE="https://fapi.binance.com")
+    add_compile_definitions(BINANCE_USDS_FUTURES_INSTRUMENTS_PATH="/fapi/v1/exchangeInfo")
+    add_compile_definitions(BINANCE_USDS_FUTURES_PRICE_STATS_PATH="/fapi/v1/ticker/24hr")
+    add_compile_definitions(BINANCE_USDS_FUTURES_USER_DATA_PATH="/api/v3/userDataStream")
+
+    # ** Binance Coin Futures **
+    add_compile_definitions(CCAPI_BINANCE_COIN_FUTURES_URL_REST_BASE="https://dapi.binance.com")
+    add_compile_definitions(BINANCE_COIN_FUTURES_INSTRUMENTS_PATH="/dapi/v1/exchangeInfo")
+    add_compile_definitions(BINANCE_COIN_FUTURES_PRICE_STATS_PATH="/dapi/v1/ticker/24hr")
 
     # ** Deribit **
     add_compile_definitions(DERIBIT_INSTRUMENT_PATH="/api/v2/public/get_instrument")
@@ -110,10 +114,14 @@ else()
 
   # subdirs
   if (BUILD_SPARKD)
-    add_subdirectory(src/server/sparkd)
+    add_subdirectory(src/lib_onixs_fix)
+    add_subdirectory(src/lib_binance_execution)
+    add_subdirectory(src/lib_okx_execution)
+    add_subdirectory(src/lib_python_md)
+    add_subdirectory(src/app_sparkd)
     add_subdirectory(src/data_files)
   endif()
-  add_subdirectory(src/lib)
+  add_subdirectory(src/lib_spark)
   if (BUILD_ADMIN_WEB_APP)
     add_subdirectory(src/Spark.Admin.WebApp)
   endif()
@@ -123,37 +131,4 @@ else()
   if(BUILD_SDK)
     add_subdirectory(src/sdk)
   endif()
-
-  if(BUILD_SPARKD)
-    add_custom_command(
-      OUTPUT ${CMAKE_BINARY_DIR}/server
-      COMMAND mkdir -p ${CMAKE_BINARY_DIR}/server
-      COMMAND ln ARGS -sf ${CMAKE_BINARY_DIR}/src/server/sparkd/sparkd ${CMAKE_BINARY_DIR}/server
-      COMMAND ln ARGS -sf ${CMAKE_BINARY_DIR}/data_files ${CMAKE_BINARY_DIR}/server
-      COMMAND ln ARGS -sf ${CMAKE_BINARY_DIR}/src/lib/libspark.so ${CMAKE_BINARY_DIR}/server
-      COMMAND ln ARGS -sf ${EXTERNALS_ROOT}/lib/${CMAKE_BUILD_TYPE}/libonixs-fix-engine.so.4 ${CMAKE_BINARY_DIR}/server
-      
-      COMMAND
-        ln ARGS -sf
-        ${EXTERNALS_ROOT}/lib/Release/libOnixS.IceImpactMulticastPriceFeedHandlerCpp.so.3.16.0.0
-        ${CMAKE_BINARY_DIR}/server
-      COMMAND ln ARGS -sf ${EXTERNALS_ROOT}/lib/Release/libACE-5.7.5.so ${CMAKE_BINARY_DIR}/server
-      COMMAND mkdir ARGS -p ${CMAKE_BINARY_DIR}/server/plugins
-      COMMENT "Creating server runtime layout"
-      DEPENDS sparkd data_files_dir)
-
-    add_custom_target(server_layout ALL DEPENDS ${CMAKE_BINARY_DIR}/server)
-
-    install(
-      DIRECTORY "${CMAKE_BINARY_DIR}/server/plugins"
-      DESTINATION "${CMAKE_INSTALL_BINDIR}")
-    install(
-      FILES
-      "${SPARK_LIB}/libACE-5.7.5.so"
-      "${SPARK_LIB}/libOnixS.IceImpactMulticastPriceFeedHandlerCpp.so.3.16.0.0"
-      "${SPARK_LIB}/libonixs-fix-engine.so.4.10.1"
-      "${SPARK_LIB}/libonixs-fix-engine.so.4"
-      "${SPARK_LIB}/libonixs-fix-engine.so"
-      DESTINATION "${CMAKE_INSTALL_BINDIR}")
-  endif()
 endif()

CMakePresets.json

@@ -26,13 +26,13 @@
         "CMAKE_CXX_FLAGS_DEBUG": "$env{SPARK_CMAKE_CXX_FLAGS} /MTd",
         "CMAKE_CXX_FLAGS_RELEASE": "$env{SPARK_CMAKE_CXX_FLAGS} /MT",
         "CMAKE_CXX_STANDARD": "20",
+        "CMAKE_INSTALL_PREFIX": "${sourceDir}/install/${presetName}",
         "CMAKE_SHARED_LINKER_FLAGS_DEBUG": "/DEBUG",
         "CMAKE_SHARED_LINKER_FLAGS_RELEASE": "/OPT:REF /OPT:ICF /LTCG",
         "CMAKE_TOOLCHAIN_FILE": "${sourceDir}/vcpkg/scripts/buildsystems/vcpkg.cmake",
         "EXTERNALS_ROOT": "${sourceDir}/externals/windows",
-        "CONFIG_DEFAULTS_PATH": "config/profiles/default/*.conf",
-        "DOTNET_BINARY_DIR": "${sourceDir}/build/$<CONFIG>/bin/net7.0",
-        "DOTNET_WINDOWS_BINARY_DIR": "${sourceDir}/build/$<CONFIG>/bin/net7.0-windows",
+        "CONFIG_DEFAULTS_PATH": "spark/config/profiles/default/*.conf",
+        "DOTNET_BINARY_DIR": "${sourceDir}/build/$<CONFIG>/bin/net8.0;${sourceDir}/build/$<CONFIG>/bin/net8.0-windows",
         "EXPORT_MD_CONSUMER": {
           "type": "BOOL",
           "value": "ON"
@@ -57,12 +57,18 @@
         "APP_DATA_DIR_OVERRIDE": "${dollar}{SPARKD_DATA_DIR}",
         "CMAKE_BUILD_WITH_INSTALL_RPATH": {
           "type": "BOOL",
-          "value": "ON"
+          "value": "OFF"
         },
         "CMAKE_CXX_STANDARD": "20",
         "CMAKE_INSTALL_RPATH": ".",
         "CMAKE_INSTALL_PREFIX": "${sourceDir}/install/${presetName}",
+        "CMAKE_PREFIX_PATH": "${sourceDir}/cmake/;${sourceDir}/.venv/lib/python3.10/site-packages/pybind11/share/cmake/pybind11",
         "CMAKE_EXE_LINKER_FLAGS": "-Wno-undef",
+        "CMAKE_TOOLCHAIN_FILE": "${sourceDir}/vcpkg/scripts/buildsystems/vcpkg.cmake",
+        "DATABENTO_USE_EXTERNAL_JSON": {
+          "type": "BOOL",
+          "value": "ON"
+        },
         "EXTERNALS_ROOT": "${sourceDir}/externals/linux",
         "BUILD_SDK": {
           "type": "BOOL",
@@ -87,7 +93,7 @@
           "value": "ON"
         },
         "CCAPI_LOG_LEVEL": "Info",
-        "CONFIG_DEFAULTS_PATH": "config/profiles/md_dev/*.conf;config/profiles/default/*.conf"
+        "CONFIG_DEFAULTS_PATH": "spark/config/profiles/md_dev/*.conf;spark/config/profiles/default/*.conf"
       }
     },
 
@@ -98,7 +104,7 @@
       "cacheVariables": {
         "CMAKE_BUILD_TYPE": "Release",
         "CCAPI_LOG_LEVEL": "Warn",
-        "CONFIG_DEFAULTS_PATH": "config/profiles/md_prod/*.conf;config/profiles/default/*.conf"
+        "CONFIG_DEFAULTS_PATH": "spark/config/profiles/md_prod/*.conf;spark/config/profiles/default/*.conf"
       }
     },
 
@@ -180,8 +186,12 @@
         "CMAKE_CXX_FLAGS_DEBUG": "-g -fPIC -Wall -pedantic -Wextra -Werror -Wno-missing-braces -Wno-ignored-qualifiers --std=c++20 -fdiagnostics-color -DDEBUG",
         "CMAKE_CXX_FLAGS_RELEASE": "-fPIC -Wall -pedantic -Wextra -Werror -Wno-missing-braces -Wno-ignored-qualifiers --std=c++20 -fdiagnostics-color -fno-omit-frame-pointer -DNDEBUG -O3",
         "CMAKE_INSTALL_PREFIX": "${sourceDir}/install/${presetName}",
-        "CONFIG_DEFAULTS_PATH": "config/profiles/default/*.conf",
-        "DOTNET_BINARY_DIR": "${sourceDir}/build/$<CONFIG>/bin/net7.0",
+        "CMAKE_EXPORT_COMPILE_COMMANDS": {
+          "type": "BOOL",
+          "value": "ON"
+        },
+        "CONFIG_DEFAULTS_PATH": "spark/config/profiles/default/*.conf",
+        "DOTNET_BINARY_DIR": "${sourceDir}/build/$<CONFIG>/bin/net8.0",
         "EXPORT_MD_CONSUMER": {
           "type": "BOOL",
           "value": "ON"
@@ -221,7 +231,7 @@
       "cacheVariables": {
         "APP_DATA_DIR": "${dollar}{APPDATA}/bts/DataStream",
         "APP_DATA_DIR_OVERRIDE": "${dollar}{DATA_STREAM_DATA_DIR}",
-        "CONFIG_DEFAULTS_PATH": "config/profiles/default/*.conf;config/profiles/data_stream/*.conf",
+        "CONFIG_DEFAULTS_PATH": "spark/config/profiles/default/*.conf;spark/config/profiles/data_stream/*.conf",
         "SHARED_MEMORY_PREFIX": "data_stream"
       }
     },
@@ -256,8 +266,7 @@
         "CMAKE_CXX_COMPILER": "clang++",
         "CMAKE_CXX_FLAGS_DEBUG": "-g -fPIC -Wall -pedantic -Wextra -Werror -Wno-missing-braces -Wno-ignored-qualifiers --std=c++20 -fdiagnostics-color -DDEBUG",
         "CMAKE_CXX_FLAGS_RELEASE": "-fPIC -Wall -pedantic -Wextra -Werror -Wno-missing-braces -Wno-ignored-qualifiers --std=c++20 -fdiagnostics-color -fno-omit-frame-pointer -DNDEBUG -O3",
-        "CMAKE_INSTALL_PREFIX": "${sourceDir}/install/${presetName}",
-        "CONFIG_DEFAULTS_PATH": "config/profiles/default/*.conf;config/profiles/data_stream/*.conf",
+        "CONFIG_DEFAULTS_PATH": "spark/config/profiles/default/*.conf;spark/config/profiles/data_stream/*.conf",
         "DOTNET_BINARY_DIR": "${sourceDir}/build/$<CONFIG>/bin/net7.0",
         "EXPORT_MD_CONSUMER": {
           "type": "BOOL",
@@ -302,16 +311,7 @@
         "lhs": "${hostSystemName}",
         "rhs": "Linux"
       },
-      "hidden": true,
-      "targets": [
-        "data_files_dir",
-        "admin_web_app_bundle",
-        "install",
-        "libspark",
-        "rapidjson_project",
-        "server_layout",
-        "sparkd"
-      ]
+      "hidden": true
     },
 
     {

cmake/ExportToParent.cmake

@@ -0,0 +1,7 @@
+include_guard(GLOBAL)
+
+macro(export_to_parent)
+  foreach(var IN ITEMS ${ARGN})
+    set(${var} ${${var}} PARENT_SCOPE)
+  endforeach()
+endmacro()

cmake/GitVersion.cmake

@@ -0,0 +1,48 @@
+include_guard(GLOBAL)
+
+function(get_version_from_git)
+  find_package(Git QUIET)
+  if(NOT Git_FOUND)
+    message(WARNING "Git not found")
+    return()
+  endif()
+
+  execute_process(
+    COMMAND ${GIT_EXECUTABLE} describe --tags --always
+    WORKING_DIRECTORY ${CMAKE_SOURCE_DIR}
+    OUTPUT_VARIABLE GIT_TAG
+    OUTPUT_STRIP_TRAILING_WHITESPACE
+    RESULT_VARIABLE GIT_RESULT
+  )
+
+  if(NOT GIT_RESULT EQUAL 0)
+    message(WARNING "Failed to get git tag")
+    return()
+  endif()
+
+  execute_process(
+    COMMAND ${GIT_EXECUTABLE} rev-parse --short=7 HEAD
+    WORKING_DIRECTORY ${CMAKE_SOURCE_DIR}
+    OUTPUT_VARIABLE GIT_COMMIT_SHORT_HASH
+    OUTPUT_STRIP_TRAILING_WHITESPACE
+  )
+  set(PROJECT_VERSION_BUILD_METADATA ${GIT_COMMIT_SHORT_HASH})
+  set(PROJECT_VERSION_BUILD_METADATA ${GIT_COMMIT_SHORT_HASH} PARENT_SCOPE)
+
+  string(REGEX REPLACE "^v" "" CLEAN_TAG "${GIT_TAG}")
+  if(CLEAN_TAG MATCHES "^([0-9]+)\\.([0-9]+)\\.([0-9]+)(-.*)?$")
+    set(PROJECT_VERSION_MAJOR ${CMAKE_MATCH_1})
+    set(PROJECT_VERSION_MAJOR ${CMAKE_MATCH_1} PARENT_SCOPE)
+    set(PROJECT_VERSION_MINOR ${CMAKE_MATCH_2})
+    set(PROJECT_VERSION_MINOR ${CMAKE_MATCH_2} PARENT_SCOPE)
+    set(PROJECT_VERSION_PATCH ${CMAKE_MATCH_3})
+    set(PROJECT_VERSION_PATCH ${CMAKE_MATCH_3} PARENT_SCOPE)
+
+    set(PROJECT_VERSION "${CMAKE_MATCH_1}.${CMAKE_MATCH_2}.${CMAKE_MATCH_3}") 
+    set(PROJECT_VERSION "${CMAKE_MATCH_1}.${CMAKE_MATCH_2}.${CMAKE_MATCH_3}" PARENT_SCOPE)
+    set(PROJECT_VERSION_FULL "${CMAKE_MATCH_1}.${CMAKE_MATCH_2}.${CMAKE_MATCH_3}+${GIT_COMMIT_SHORT_HASH}")
+    set(PROJECT_VERSION_FULL "${CMAKE_MATCH_1}.${CMAKE_MATCH_2}.${CMAKE_MATCH_3}+${GIT_COMMIT_SHORT_HASH}" PARENT_SCOPE)
+  else()
+    message(WARNING "Tag '${CLEAN_TAG}' does not match semver format")
+  endif()
+endfunction()

cmake/InitLibrary.cmake

@@ -0,0 +1,142 @@
+include_guard(DIRECTORY)
+include(${CMAKE_SOURCE_DIR}/cmake/ExportToParent.cmake)
+include(${CMAKE_SOURCE_DIR}/cmake/InstallBasic.cmake)
+include(${CMAKE_SOURCE_DIR}/cmake/ProtobufGenerateCpp.cmake)
+include(${CMAKE_SOURCE_DIR}/cmake/SetFirstDefined.cmake)
+include(${CMAKE_SOURCE_DIR}/cmake/safe_dep_glob.cmake)
+
+function(init_library)
+  set(_options SHARED STATIC)
+  set(_one_value_args
+    NAME
+    SOURCE_DIR
+    INCLUDE_DIR
+  )
+  set(_multi_value_args)
+  cmake_parse_arguments(_ARGS "${_options}" "${_one_value_args}" "${_multi_value_args}" ${ARGN})
+  if (_ARGS_SHARED AND _ARGS_STATIC)
+    message(FATAL_ERROR "Invalid args passed to init_library: cannot specify STATIC and SHARED")
+  endif()
+  set_first_defined(_name _ARGS_NAME PROJECT_NAME)
+  set_first_defined(_source_dir _ARGS_SOURCE_DIR _name)
+  set_first_defined(_include_dir _ARGS_INCLUDE_DIR _source_dir)
+
+  # Cpp sources
+  file(
+    GLOB_RECURSE ${_name}_SOURCES
+    RELATIVE ${CMAKE_CURRENT_SOURCE_DIR}
+    ${_source_dir}/*.cpp
+  )
+  # Note: cpp files inside 'tests' dir(s) are considered tests.
+  # Remove tests from main sources, and add them to test sources
+  foreach(item ${${_name}_SOURCES})
+    if(${item} MATCHES "tests/.*\.cpp$")
+      list(APPEND ${_name}_TEST_SOURCES ${item})
+    endif()
+  endforeach()
+  foreach(item ${${_name}_TEST_SOURCES})
+    list(REMOVE_ITEM ${_name}_SOURCES ${item})
+  endforeach()
+
+  # Proto sources 
+  file(
+    GLOB_RECURSE ${_name}_PROTO_SOURCES
+    RELATIVE ${CMAKE_CURRENT_SOURCE_DIR}
+    ${_source_dir}/protobufs/*.proto
+  )
+  update_deps_file("${${_name}_PROTO_SOURCES};")
+  protobuf_generate_cpp(
+    CXX_SOURCES ${_name}_CXX_PROTO_SOURCES
+    CXX_HEADERS ${_name}_CXX_PROTO_HEADERS
+    CXX_OUT_DIR ${CMAKE_CURRENT_BINARY_DIR}
+    PROTO_INCLUDE_DIRS
+      ${EXTERNALS_INCLUDE}
+      ${CMAKE_CURRENT_SOURCE_DIR}
+    PROTO_SOURCES
+      ${${_name}_PROTO_SOURCES}
+  )
+
+  export_to_parent(
+    ${_name}_SOURCES
+    ${_name}_TEST_SOURCES
+    ${_name}_PROTO_SOURCES
+    ${_name}_CXX_PROTO_SOURCES
+    ${_name}_CXX_PROTO_HEADERS
+  )
+
+  # Add main lib
+  if (_ARGS_STATIC)
+    add_library(${_name} STATIC ${${_name}_SOURCES} ${${_name}_CXX_PROTO_SOURCES})
+  elseif(_ARGS_SHARED)
+    add_library(${_name} SHARED ${${_name}_SOURCES} ${${_name}_CXX_PROTO_SOURCES})
+  else()
+    add_library(${_name} ${${_name}_SOURCES} ${${_name}_CXX_PROTO_SOURCES})
+  endif()
+  target_include_directories(
+    ${_name}
+    SYSTEM
+    BEFORE
+    PUBLIC
+      $<BUILD_INTERFACE:${EXTERNALS_ROOT}/include>
+      $<BUILD_INTERFACE:${EXTERNALS_ROOT}/include/zlib>
+  )
+  target_include_directories(
+    ${_name}
+    PUBLIC
+      $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}>
+      $<INSTALL_INTERFACE:${CMAKE_INSTALL_INCLUDEDIR}>
+  )
+  # Include protos (if any)
+  if (${_name}_PROTO_SOURCES)
+    target_include_directories(
+      ${_name}
+      PUBLIC
+        $<BUILD_INTERFACE:${CMAKE_CURRENT_BINARY_DIR}>
+    )
+  endif()
+  target_link_directories(
+    ${_name}
+    PUBLIC
+      $<BUILD_INTERFACE:${CMAKE_CURRENT_BINARY_DIR}>
+      $<BUILD_INTERFACE:${EXTERNALS_ROOT}/lib/$<CONFIG>>
+      $<INSTALL_INTERFACE:${CMAKE_INSTALL_LIBDIR}>
+  )
+  # Install main lib
+  foreach(_hdr ${${_name}_CXX_PROTO_HEADERS})
+    file(RELATIVE_PATH _relative_path ${CMAKE_CURRENT_BINARY_DIR} ${_hdr})
+    install(FILES ${_hdr} DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}/${_relative_path})
+  endforeach()
+  install_basic(
+    NAME ${_name}
+    INCLUDE_DIR ${_include_dir}
+  )
+
+  # Add tests executable (if there are any)
+  if (${_name}_TEST_SOURCES AND NOT WIN32)
+    add_executable(${_name}-tests ${${_name}_TEST_SOURCES})
+
+    get_target_property(_lib_type ${_name} TYPE)
+    if (_lib_type STREQUAL "SHARED_LIBRARY")
+      # Set RPATH so the tests exe knows where to find the shared lib.
+      set_target_properties(
+        ${_name}-tests
+        PROPERTIES
+          BUILD_RPATH "$ORIGIN"
+          INSTALL_RPATH "$ORIGIN/../${CMAKE_INSTALL_LIBDIR}"
+      )
+    endif()
+
+    # Link main lib
+    target_link_directories(
+      ${_name}-tests
+      PRIVATE
+        $<BUILD_INTERFACE:${CMAKE_CURRENT_BINARY_DIR}>
+        $<INSTALL_INTERFACE:${CMAKE_INSTALL_LIBDIR}>
+    )
+    target_link_libraries(${_name}-tests PRIVATE ${_name})
+
+    # Catch2
+    find_package(Catch2 CONFIG REQUIRED COMPONENTS Catch2WithMain)
+    target_link_libraries(${_name}-tests PRIVATE Catch2::Catch2WithMain)
+  endif()
+endfunction()

cmake/InstallBasic.cmake

@@ -0,0 +1,39 @@
+include_guard(DIRECTORY)
+include(GNUInstallDirs)
+include(${CMAKE_SOURCE_DIR}/cmake/PackBasic.cmake)
+include(${CMAKE_SOURCE_DIR}/cmake/SetFirstDefined.cmake)
+
+function(install_basic)
+  set(_options)
+  set(_one_value_args NAME INCLUDE_DIR)
+  set(_multi_value_args)
+  cmake_parse_arguments(_ARGS "${_options}" "${_one_value_args}" "${_multi_value_args}" ${ARGN})
+  set_first_defined(_name _ARGS_NAME PROJECT_NAME)
+  set_first_defined(_include_dir _ARGS_INCLUDE_DIR _name)
+
+  file(
+    GLOB_RECURSE ${_name}_HEADERS
+    RELATIVE ${CMAKE_CURRENT_SOURCE_DIR}
+    ${_include_dir}/*.h)
+
+  target_sources(${_name}
+    PUBLIC
+      FILE_SET HEADERS FILES ${${_name}_HEADERS})
+
+  install(
+    TARGETS ${_name}
+    EXPORT ${_name}Targets
+    RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR}
+    LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR}
+    ARCHIVE DESTINATION ${CMAKE_INSTALL_LIBDIR}
+    FILE_SET HEADERS DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}
+  )
+
+  install(
+    EXPORT ${_name}Targets
+    FILE ${_name}Targets.cmake
+    NAMESPACE ${_name}::
+    DESTINATION ${CMAKE_INSTALL_LIBDIR}/cmake/${_name})
+
+  pack_basic()
+endfunction()

cmake/PackBasic.cmake

@@ -0,0 +1,44 @@
+include_guard(DIRECTORY)
+include(CMakePackageConfigHelpers)
+include(${CMAKE_SOURCE_DIR}/cmake/SetFirstDefined.cmake)
+
+function(pack_basic)
+  set(_options)
+  set(_one_value_args NAME)
+  set(_multi_value_args)
+  cmake_parse_arguments(_ARGS "${_options}" "${_one_value_args}" "${_multi_value_args}" ${ARGN})
+  set_first_defined(_name _ARGS_NAME PROJECT_NAME)
+
+  find_package(Git QUIET)
+
+  if(GIT_FOUND)
+    execute_process(
+      COMMAND ${GIT_EXECUTABLE} describe --tags --always --dirty
+      WORKING_DIRECTORY ${CMAKE_SOURCE_DIR}
+      OUTPUT_VARIABLE PROJECT_VERSION
+      OUTPUT_STRIP_TRAILING_WHITESPACE
+    )
+  endif()
+
+  if(NOT GIT_FOUND OR PROJECT_VERSION STREQUAL "")
+    set(PROJECT_VERSION "0.0.0")
+  endif()
+
+  write_basic_package_version_file(
+    "${CMAKE_CURRENT_BINARY_DIR}/${_name}ConfigVersion.cmake"
+    VERSION ${PROJECT_VERSION}
+    COMPATIBILITY AnyNewerVersion
+  )
+
+  configure_package_config_file(
+    "${CMAKE_CURRENT_SOURCE_DIR}/Config.cmake.in"
+    "${CMAKE_CURRENT_BINARY_DIR}/${_name}Config.cmake"
+    INSTALL_DESTINATION ${CMAKE_INSTALL_LIBDIR}/cmake/${_name}
+  )
+
+  install(
+    FILES
+      "${CMAKE_CURRENT_BINARY_DIR}/${_name}Config.cmake"
+      "${CMAKE_CURRENT_BINARY_DIR}/${_name}ConfigVersion.cmake"
+    DESTINATION ${CMAKE_INSTALL_LIBDIR}/cmake/${_name})
+endfunction()

cmake/Protobuf.cmake

@@ -1,12 +1,23 @@
-set(Protobuf_PROTOC_EXECUTABLE "${SPARK_BIN}/protoc")
-set(PROTOBUF_INCLUDE_DIRS "${SPARK_INCLUDE}")
+set(Protobuf_PROTOC_EXECUTABLE "${EXTERNALS_BIN}/protoc")
+set(PROTOBUF_INCLUDE_DIRS "${EXTERNALS_INCLUDE}")
 set(PROTOBUF_STATIC_LIB "libprotobuf")
+set(_protobuf_src_dir ${CMAKE_SOURCE_DIR}/src/lib_spark)
+set(_protobuf_bin_dir ${CMAKE_BINARY_DIR}/src/lib_spark)
 
-function(Protobuf_Generate_Cpp SRCS HDRS)
-  set(${SRCS})
-  set(${HDRS})
+function(Protobuf_Generate_Cpp)
+  set(_options)
+  set(_one_value_args CXX_SOURCES CXX_HEADERS CXX_OUT_DIR)
+  set(_multi_value_args PROTO_INCLUDE_DIRS PROTO_SOURCES)
+  cmake_parse_arguments(_ARGS "${_options}" "${_one_value_args}" "${_multi_value_args}" ${ARGN})
+  set(${_ARGS_CXX_SOURCES})
+  set(${_ARGS_CXX_HEADERS})
 
-  foreach(FIL ${ARGN})
+  set(_proto_include_cli_args)
+  foreach(_dir ${_ARGS_PROTO_INCLUDE_DIRS})
+    list(APPEND _proto_include_cli_args -I ${_dir})
+  endforeach()
+
+  foreach(FIL ${_ARGS_PROTO_SOURCES})
     get_filename_component(ABS_FIL ${FIL} ABSOLUTE)
     get_filename_component(FIL_WE ${FIL} NAME_WE)
     get_filename_component(FIL_DIR ${FIL} DIRECTORY)
@@ -14,48 +25,56 @@ function(Protobuf_Generate_Cpp SRCS HDRS)
       set(FIL_WE "${FIL_DIR}/${FIL_WE}")
     endif()
 
-    list(APPEND ${SRCS} "${CMAKE_CURRENT_BINARY_DIR}/${FIL_WE}.pb.cc")
-    list(APPEND ${HDRS} "${CMAKE_CURRENT_BINARY_DIR}/${FIL_WE}.pb.h")
+    list(APPEND ${_ARGS_CXX_SOURCES} "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.pb.cc")
+    list(APPEND ${_ARGS_CXX_HEADERS} "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.pb.h")
 
     add_custom_command(
-      OUTPUT "${CMAKE_CURRENT_BINARY_DIR}/${FIL_WE}.pb.cc"
-             "${CMAKE_CURRENT_BINARY_DIR}/${FIL_WE}.pb.h"
+      OUTPUT "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.pb.cc"
+             "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.pb.h"
       COMMAND
-        ${Protobuf_PROTOC_EXECUTABLE} ARGS --cpp_out=${CMAKE_BINARY_DIR}/src -I
-        ${PROTOBUF_INCLUDE_DIRS} -I ${CMAKE_SOURCE_DIR}/src ${ABS_FIL}
+        ${Protobuf_PROTOC_EXECUTABLE} ARGS --cpp_out=${_ARGS_CXX_OUT_DIR} ${_proto_include_cli_args} ${ABS_FIL}
       DEPENDS ${ABS_FIL}
       COMMENT "Running C++ protocol buffer compiler on ${FIL}"
       VERBATIM)
   endforeach()
 
-  set_source_files_properties(${${SRCS}} ${${HDRS}} PROPERTIES GENERATED TRUE)
+  set_source_files_properties(${${_ARGS_CXX_SOURCES}} ${${_ARGS_CXX_HEADERS}} PROPERTIES GENERATED TRUE)
 
   # remove some warnings from protobuf generated files
   if(WIN32)
-    set_source_files_properties(${${SRCS}} PROPERTIES COMPILE_FLAGS
+    set_source_files_properties(${${_ARGS_CXX_SOURCES}} PROPERTIES COMPILE_FLAGS
                                                       "/wd4309 /wd4838")
   else()
     set_source_files_properties(
-      ${${SRCS}}
+      ${${_ARGS_CXX_SOURCES}}
       PROPERTIES
         COMPILE_FLAGS
         "-Wno-unused-parameter -Wno-effc++ -Wno-pedantic -Wno-sign-compare -Wno-uninitialized"
     )
   endif()
 
-  set(${SRCS}
-      ${${SRCS}}
+  set(${_ARGS_CXX_SOURCES}
+      ${${_ARGS_CXX_SOURCES}}
       PARENT_SCOPE)
-  set(${HDRS}
-      ${${HDRS}}
+  set(${_ARGS_CXX_HEADERS}
+      ${${_ARGS_CXX_HEADERS}}
       PARENT_SCOPE)
 endfunction()
 
-function(Service_Compiler_Generate_Cpp SRCS HDRS)
-  set(${SRCS})
-  set(${HDRS})
+function(Service_Compiler_Generate_Cpp)
+  set(_options)
+  set(_one_value_args CXX_SOURCES CXX_HEADERS CXX_OUT_DIR)
+  set(_multi_value_args PROTO_INCLUDE_DIRS PROTO_SOURCES)
+  cmake_parse_arguments(_ARGS "${_options}" "${_one_value_args}" "${_multi_value_args}" ${ARGN})
+  set(${_ARGS_CXX_SOURCES})
+  set(${_ARGS_CXX_HEADERS})
 
-  foreach(FIL ${ARGN})
+  set(_proto_include_cli_args)
+  foreach(_dir ${_ARGS_PROTO_INCLUDE_DIRS})
+    list(APPEND _proto_include_cli_args -I ${_dir})
+  endforeach()
+
+  foreach(FIL ${_ARGS_PROTO_SOURCES})
     get_filename_component(ABS_FIL ${FIL} ABSOLUTE)
     get_filename_component(FIL_WE ${FIL} NAME_WE)
     get_filename_component(FIL_DIR ${FIL} DIRECTORY)
@@ -63,58 +82,58 @@ function(Service_Compiler_Generate_Cpp SRCS HDRS)
       set(FIL_WE "${FIL_DIR}/${FIL_WE}")
     endif()
 
-    list(APPEND ${SRCS} "${CMAKE_CURRENT_BINARY_DIR}/${FIL_WE}.pb.cc")
-    list(APPEND ${HDRS} "${CMAKE_CURRENT_BINARY_DIR}/${FIL_WE}.pb.h")
+    list(APPEND ${_ARGS_CXX_SOURCES} "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.pb.cc")
+    list(APPEND ${_ARGS_CXX_HEADERS} "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.pb.h")
 
-    list(APPEND ${SRCS} "${CMAKE_CURRENT_BINARY_DIR}/${FIL_WE}.client.pb.cc")
-    list(APPEND ${HDRS} "${CMAKE_CURRENT_BINARY_DIR}/${FIL_WE}.client.pb.h")
+    list(APPEND ${_ARGS_CXX_SOURCES} "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.client.pb.cc")
+    list(APPEND ${_ARGS_CXX_HEADERS} "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.client.pb.h")
 
-    list(APPEND ${SRCS} "${CMAKE_CURRENT_BINARY_DIR}/${FIL_WE}.server.pb.cc")
-    list(APPEND ${HDRS} "${CMAKE_CURRENT_BINARY_DIR}/${FIL_WE}.server.pb.h")
+    list(APPEND ${_ARGS_CXX_SOURCES} "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.server.pb.cc")
+    list(APPEND ${_ARGS_CXX_HEADERS} "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.server.pb.h")
 
     set(TEMPLATES
-        ${CMAKE_CURRENT_SOURCE_DIR}/api/service_compiler/templates/cpp/client.pb.cc
-        ${CMAKE_CURRENT_SOURCE_DIR}/api/service_compiler/templates/cpp/client.pb.h
-        ${CMAKE_CURRENT_SOURCE_DIR}/api/service_compiler/templates/cpp/server.pb.cc
-        ${CMAKE_CURRENT_SOURCE_DIR}/api/service_compiler/templates/cpp/server.pb.h
+        ${CMAKE_SOURCE_DIR}/src/lib_spark/spark/api/service_compiler/templates/cpp/client.pb.cc
+        ${CMAKE_SOURCE_DIR}/src/lib_spark/spark/api/service_compiler/templates/cpp/client.pb.h
+        ${CMAKE_SOURCE_DIR}/src/lib_spark/spark/api/service_compiler/templates/cpp/server.pb.cc
+        ${CMAKE_SOURCE_DIR}/src/lib_spark/spark/api/service_compiler/templates/cpp/server.pb.h
     )
 
+    set(_plugin_py ${CMAKE_SOURCE_DIR}/src/lib_spark/spark/api/service_compiler/plugin.py)
     if(WIN32)
+      set(_plugin_cmd ${CMAKE_SOURCE_DIR}/src/lib_spark/spark/api/service_compiler/plugin.cmd)
       add_custom_command(
-        OUTPUT "${CMAKE_CURRENT_BINARY_DIR}/${FIL_WE}.pb.cc"
-               "${CMAKE_CURRENT_BINARY_DIR}/${FIL_WE}.pb.h"
-               "${CMAKE_CURRENT_BINARY_DIR}/${FIL_WE}.client.pb.cc"
-               "${CMAKE_CURRENT_BINARY_DIR}/${FIL_WE}.client.pb.h"
-               "${CMAKE_CURRENT_BINARY_DIR}/${FIL_WE}.server.pb.cc"
-               "${CMAKE_CURRENT_BINARY_DIR}/${FIL_WE}.server.pb.h"
+        OUTPUT "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.pb.cc"
+               "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.pb.h"
+               "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.client.pb.cc"
+               "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.client.pb.h"
+               "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.server.pb.cc"
+               "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.server.pb.h"
         COMMAND SET PYTHONPATH=${SPARK_PYTHON}
         COMMAND ${CMAKE_SOURCE_DIR}/.venv/Scripts/activate.bat
         COMMAND
-          ${Protobuf_PROTOC_EXECUTABLE} ARGS --cpp_out=${CMAKE_BINARY_DIR}/src
-          --plugin=protoc-gen-service=${CMAKE_SOURCE_DIR}/src/lib/api/service_compiler/plugin.cmd
-          --service_out=lang=cpp:${CMAKE_BINARY_DIR}/src -I
-          ${PROTOBUF_INCLUDE_DIRS} -I ${CMAKE_SOURCE_DIR}/src ${ABS_FIL}
-        DEPENDS ${ABS_FIL} ${TEMPLATES}
-                ${CMAKE_CURRENT_SOURCE_DIR}/api/service_compiler/plugin.py
+          ${Protobuf_PROTOC_EXECUTABLE} ARGS --cpp_out=${_ARGS_CXX_OUT_DIR}
+          --plugin=protoc-gen-service=${_plugin_cmd}
+          --service_out=lang=cpp:${_ARGS_CXX_OUT_DIR} -I
+          ${PROTOBUF_INCLUDE_DIRS} -I ${_protobuf_src_dir} ${ABS_FIL}
+        DEPENDS ${ABS_FIL} ${TEMPLATES} ${_plugin_cmd} ${_plugin_py}
         COMMENT "Running C++ service compiler on ${FIL}"
         VERBATIM)
     else()
       add_custom_command(
-        OUTPUT "${CMAKE_CURRENT_BINARY_DIR}/${FIL_WE}.pb.cc"
-               "${CMAKE_CURRENT_BINARY_DIR}/${FIL_WE}.pb.h"
-               "${CMAKE_CURRENT_BINARY_DIR}/${FIL_WE}.client.pb.cc"
-               "${CMAKE_CURRENT_BINARY_DIR}/${FIL_WE}.client.pb.h"
-               "${CMAKE_CURRENT_BINARY_DIR}/${FIL_WE}.server.pb.cc"
-               "${CMAKE_CURRENT_BINARY_DIR}/${FIL_WE}.server.pb.h"
+        OUTPUT "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.pb.cc"
+               "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.pb.h"
+               "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.client.pb.cc"
+               "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.client.pb.h"
+               "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.server.pb.cc"
+               "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.server.pb.h"
         COMMAND ${CMAKE_SOURCE_DIR}/tools/venv_exec.sh
         ARGS
           ${Protobuf_PROTOC_EXECUTABLE}
-          --cpp_out=${CMAKE_BINARY_DIR}/src
-          --plugin=protoc-gen-service=${CMAKE_SOURCE_DIR}/src/lib/api/service_compiler/plugin.py
-          --service_out=lang=cpp,make_file_name=${ABS_FIL}:${CMAKE_BINARY_DIR}/src
-          -I ${PROTOBUF_INCLUDE_DIRS} -I ${CMAKE_SOURCE_DIR}/src ${ABS_FIL}
-        DEPENDS ${ABS_FIL} ${TEMPLATES}
-                ${CMAKE_CURRENT_SOURCE_DIR}/api/service_compiler/plugin.py
+          --cpp_out=${_ARGS_CXX_OUT_DIR}
+          --plugin=protoc-gen-service=${_plugin_py}
+          --service_out=lang=cpp,make_file_name=${ABS_FIL}:${_ARGS_CXX_OUT_DIR}
+          ${_proto_include_cli_args} ${ABS_FIL}
+        DEPENDS ${ABS_FIL} ${TEMPLATES} ${_plugin_py}
         COMMENT "Running C++ service compiler on ${FIL}"
         VERBATIM)
     endif()
@@ -123,22 +142,22 @@ function(Service_Compiler_Generate_Cpp SRCS HDRS)
 
   # remove some warnings from protobuf generated files
   if(WIN32)
-    set_source_files_properties(${${SRCS}} PROPERTIES COMPILE_FLAGS
+    set_source_files_properties(${${_ARGS_CXX_SOURCES}} PROPERTIES COMPILE_FLAGS
                                                       "/wd4309 /wd4838")
   else()
     set_source_files_properties(
-      ${${SRCS}}
+      ${${_ARGS_CXX_SOURCES}}
       PROPERTIES
         COMPILE_FLAGS
         "-Wno-unused-parameter -Wno-effc++ -Wno-pedantic -Wno-sign-compare -Wno-uninitialized"
     )
   endif()
 
-  set(${SRCS}
-      ${${SRCS}}
+  set(${_ARGS_CXX_SOURCES}
+      ${${_ARGS_CXX_SOURCES}}
       PARENT_SCOPE)
-  set(${HDRS}
-      ${${HDRS}}
+  set(${_ARGS_CXX_HEADERS}
+      ${${_ARGS_CXX_HEADERS}}
       PARENT_SCOPE)
 
 endfunction()

cmake/ProtobufGenerateCpp.cmake

@@ -0,0 +1,83 @@
+include_guard(GLOBAL)
+
+function(protobuf_generate_cpp)
+  set(_options)
+  set(_one_value_args
+    CXX_SOURCES
+    CXX_HEADERS
+    CXX_OUT_DIR
+    PROTOC
+  )
+  set(_multi_value_args
+    PROTO_INCLUDE_DIRS
+    PROTO_SOURCES
+  )
+  cmake_parse_arguments(_ARGS "${_options}" "${_one_value_args}" "${_multi_value_args}" ${ARGN})
+
+  if (NOT DEFINED _ARGS_PROTOC)
+    if (WIN32)
+      set(_ARGS_PROTOC "${EXTERNALS_BIN}/protoc.exe")
+    else()
+      set(_ARGS_PROTOC "${EXTERNALS_BIN}/protoc")
+    endif()
+  endif()
+  set(${_ARGS_CXX_SOURCES})
+  set(${_ARGS_CXX_HEADERS})
+
+  set(_proto_include_cli_args)
+  foreach(_dir ${_ARGS_PROTO_INCLUDE_DIRS})
+    list(APPEND _proto_include_cli_args -I ${_dir})
+  endforeach()
+
+  foreach(_fil ${_ARGS_PROTO_SOURCES})
+    get_filename_component(_abs_fil ${_fil} ABSOLUTE)
+    get_filename_component(_fil_we ${_fil} NAME_WE)
+    get_filename_component(_fil_dir ${_fil} DIRECTORY)
+    if(_fil_dir)
+      set(_fil_we "${_fil_dir}/${_fil_we}")
+    endif()
+    list(APPEND ${_ARGS_CXX_SOURCES} "${_ARGS_CXX_OUT_DIR}/${_fil_we}.pb.cc")
+    list(APPEND ${_ARGS_CXX_HEADERS} "${_ARGS_CXX_OUT_DIR}/${_fil_we}.pb.h")
+    message(STATUS "add_custom_command for ${_fil} -> ${_ARGS_CXX_OUT_DIR}/${_fil_we}.pb.h")
+    # TODO: Pass proto sources by relative path (so this fn can be called from
+    # arbitrary subdirs).
+    add_custom_command(
+      OUTPUT
+        "${_ARGS_CXX_OUT_DIR}/${_fil_we}.pb.cc"
+        "${_ARGS_CXX_OUT_DIR}/${_fil_we}.pb.h"
+      COMMAND
+        ${_ARGS_PROTOC}
+      ARGS
+        --cpp_out=${_ARGS_CXX_OUT_DIR} ${_proto_include_cli_args} ${_abs_fil}
+      DEPENDS ${_abs_fil}
+      COMMENT "Running C++ protocol buffer compiler on ${_fil}"
+      VERBATIM
+    )
+  endforeach()
+
+  set_source_files_properties(
+    ${${_ARGS_CXX_SOURCES}}
+    ${${_ARGS_CXX_HEADERS}}
+    PROPERTIES
+      GENERATED TRUE
+  )
+
+  # remove some warnings from protobuf generated files
+  if(WIN32)
+    set_source_files_properties(
+      ${${_ARGS_CXX_SOURCES}}
+      PROPERTIES
+        COMPILE_FLAGS "/wd4309 /wd4838"
+    )
+  else()
+    set_source_files_properties(
+      ${${_ARGS_CXX_SOURCES}}
+      PROPERTIES
+        COMPILE_FLAGS "-Wno-unused-parameter -Wno-effc++ -Wno-pedantic -Wno-sign-compare -Wno-uninitialized"
+    )
+  endif()
+
+  set(${_ARGS_CXX_SOURCES} ${${_ARGS_CXX_SOURCES}} PARENT_SCOPE)
+  set(${_ARGS_CXX_HEADERS} ${${_ARGS_CXX_HEADERS}} PARENT_SCOPE)
+endfunction()
+

cmake/ProtobufGenerateCppConfig.cmake

@@ -0,0 +1,61 @@
+include_guard(GLOBAL)
+
+function(protobuf_generate_cpp_config)
+  set(_options)
+  set(_one_value_args
+    PROTO_SOURCES_HEADER
+    PROTO_DEFAULTS_HEADER
+  )
+  set(_multi_value_args
+    PROTO_SOURCES
+    PROTO_DEFAULTS
+  )
+  cmake_parse_arguments(_ARGS "${_options}" "${_one_value_args}" "${_multi_value_args}" ${ARGN})
+
+  # Configure the 'config_proto_files' header. It provides an array of strings populated with the
+  # files in PROTO_SOURCES. At runtime, we use this array to generate a special 'config' proto
+  # message with fields for each message defined in said files. A global instance of this 'config'
+  # proto is then constructed, and populated with [compile-time] defaults, and [runtime] overrides
+  # (i.e., in spark.conf). Calls to `bts::config::get<T>()` retrieve the value from the field
+  # associated with type `T` in this global instance.
+  message(STATUS "PROTO_SOURCES: ${_ARGS_PROTO_SOURCES}")
+  message(STATUS "PROTO_SOURCES_HEADER: ${_ARGS_PROTO_SOURCES_HEADER}")
+  message(STATUS "PROTO_DEFAULTS_HEADER: ${_ARGS_PROTO_DEFAULTS_HEADER}")
+  if (_ARGS_PROTO_SOURCES_HEADER)
+    set(PROTO_SOURCES)
+    set(PROTO_SOURCES_CXX_ARRAY_ITEMS "")
+    foreach(_proto_path ${_ARGS_PROTO_SOURCES})
+      list(APPEND PROTO_SOURCES ${_proto_path})
+      string(APPEND PROTO_SOURCES_CXX_ARRAY_ITEMS "\"${_proto_path}\",\n")
+    endforeach()
+    list(LENGTH PROTO_SOURCES PROTO_SOURCES_LENGTH)
+    message(STATUS "CONFIGURING PROTO_SOURCES_HEADER): ${_ARGS_PROTO_SOURCES_HEADER}")
+    configure_file(${CMAKE_SOURCE_DIR}/cmake/config_proto_files.h.in ${_ARGS_PROTO_SOURCES_HEADER})
+  endif()
+
+  # Generate static defaults. Ideally, this would be a single string, but we use an array because
+  # the compiler complains if the string is longer than 65k.
+  set(_cfg_defaults_str "inline const char * config_static_defaults[] = { ")
+
+  foreach(_cfg_file_path ${_ARGS_PROTO_DEFAULTS})
+    file(READ ${_cfg_file_path} _cfg_file_content)
+    string(REPLACE "\"" "\\\"" _cfg_file_content_escaped ${_cfg_file_content})
+    string(REPLACE "\n" ";" _cfg_lines ${_cfg_file_content_escaped})
+
+    foreach(_line ${_cfg_lines})
+      string(APPEND _cfg_defaults_str "\n  \"${_line}\",")
+    endforeach(_line)
+  endforeach(_cfg_file_path)
+
+  string(APPEND _cfg_defaults_str "};")
+  set(_old_cfg_defaults_str "")
+
+  if(EXISTS ${_ARGS_PROTO_DEFAULTS_HEADER})
+    file(READ ${_ARGS_PROTO_DEFAULTS_HEADER} _old_cfg_defaults_str)
+  endif()
+
+  if(NOT _cfg_defaults_str STREQUAL _old_cfg_defaults_str)
+    file(WRITE "${_ARGS_PROTO_DEFAULTS_HEADER}" "${_cfg_defaults_str}")
+  endif()
+endfunction()
+

cmake/ProtobufGenerateFs.cmake

@@ -0,0 +1,49 @@
+include_guard(DIRECTORY)
+
+function(protobuf_generate_fs)
+  set(_options)
+  set(_one_value_args TARGET_NAME OUT_DIR PROTOC)
+  set(_multi_value_args PROTO_INCLUDE_DIRS PROTO_SOURCES)
+  cmake_parse_arguments(_ARGS "${_options}" "${_one_value_args}" "${_multi_value_args}" ${ARGN})
+
+  if (NOT DEFINED _ARGS_TARGET_NAME)
+    set(_ARGS_TARGET_NAME "${PROJECT_NAME}_generate_fs_protos")
+  endif()
+  if (NOT DEFINED _ARGS_PROTOC)
+    set(_ARGS_PROTOC "${EXTERNALS_BIN}/protoc")
+  endif()
+
+  set(_proto_include_cli_args)
+  foreach(_dir ${_ARGS_PROTO_INCLUDE_DIRS})
+    list(APPEND _proto_include_cli_args -I ${_dir})
+  endforeach()
+
+  set(_outputs)
+  foreach(_proto_source ${_ARGS_PROTO_SOURCES})
+    # get_filename_component(_abs_proto_source ${_proto_source} ABSOLUTE)
+    get_filename_component(_proto_source_we ${_proto_source} NAME_WE)
+    # get_filename_component(_proto_source_dir ${_proto_source} DIRECTORY)
+    list(APPEND _outputs ${_ARGS_OUT_DIR}/${_proto_source_we}.fable.fs)
+  endforeach()
+
+  set(_venv_exec ${CMAKE_SOURCE_DIR}/tools/venv_exec.sh)
+  set(_plugin_py ${CMAKE_SOURCE_DIR}/src/lib_spark/spark/api/service_compiler/plugin.py)
+  set(_template_fs ${CMAKE_SOURCE_DIR}/src/lib_spark/spark/api/service_compiler/templates/fs/fable.fs)
+  add_custom_command(
+    OUTPUT ${_outputs}
+    WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}
+    COMMAND ${_venv_exec}
+      ${_ARGS_PROTOC}
+      --plugin=protoc-gen-service=${_plugin_py}
+      --service_out=lang=fs,strip_out_path=1:${_ARGS_OUT_DIR}
+      ${_proto_include_cli_args}
+      ${_ARGS_PROTO_SOURCES}
+    DEPENDS
+      ${_venv_exec}
+      ${_plugin_py}
+      ${_template_fs}
+      ${CMAKE_CURRENT_SOURCE_DIR}/package-lock.json
+      ${_ARGS_PROTO_SOURCES}
+  )
+  add_custom_target(${_ARGS_TARGET_NAME} ALL DEPENDS "${_outputs}")
+endfunction()

cmake/ProtobufGenerateJs.cmake

@@ -0,0 +1,33 @@
+include_guard(DIRECTORY)
+
+function(protobuf_generate_js)
+  set(_options)
+  set(_one_value_args TARGET_NAME OUTPUT)
+  set(_multi_value_args PROTO_INCLUDE_DIRS PROTO_SOURCES)
+  cmake_parse_arguments(_ARGS "${_options}" "${_one_value_args}" "${_multi_value_args}" ${ARGN})
+
+  if (NOT DEFINED _ARGS_TARGET_NAME)
+    set(_ARGS_TARGET_NAME "${PROJECT_NAME}_generate_js_protos")
+  endif()
+
+  set(_proto_include_cli_args)
+  foreach(_dir ${_ARGS_PROTO_INCLUDE_DIRS})
+    list(APPEND _proto_include_cli_args -p ${_dir})
+  endforeach()
+
+  add_custom_command(
+    OUTPUT ${_ARGS_OUTPUT}
+    WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}
+    COMMAND npx
+      --no-install pbjs
+      -t json-module
+      -w commonjs
+      -o ${_ARGS_OUTPUT}
+      ${_proto_include_cli_args}
+      ${_ARGS_PROTO_SOURCES}
+    DEPENDS
+      ${CMAKE_CURRENT_SOURCE_DIR}/package-lock.json
+      ${_ARGS_PROTO_SOURCES}
+  )
+  add_custom_target(${_ARGS_TARGET_NAME} ALL DEPENDS ${_ARGS_OUTPUT})
+endfunction()

cmake/SetFirstDefined.cmake

@@ -0,0 +1,10 @@
+include_guard(GLOBAL)
+
+macro(set_first_defined RESULT)
+  foreach(_arg ${ARGN})
+    if (DEFINED ${_arg})
+      set(${RESULT} ${${_arg}})
+      break()
+    endif()
+  endforeach()
+endmacro()

cmake/SymLink.cmake

@@ -0,0 +1,49 @@
+include_guard(DIRECTORY)
+include(${CMAKE_SOURCE_DIR}/cmake/TernaryOperator.cmake)
+
+function(sym_link_dir)
+  set(_options)
+  set(_one_value_args TARGET_NAME SOURCE DEST COMMENT)
+  set(_multi_value_args)
+  cmake_parse_arguments(_ARGS "${_options}" "${_one_value_args}" "${_multi_value_args}" ${ARGN})
+  add_custom_command(
+    OUTPUT ${_ARGS_DEST}
+    COMMAND ln ARGS -sf ${_ARGS_SOURCE} ${_ARGS_DEST}
+  )
+  set_ternary(_comment _ARGS_COMMENT "${_ARGS_COMMENT}" "linking dir ${_ARGS_SOURCE} -> ${_ARGS_DEST}")
+  add_custom_target(
+    ${_ARGS_TARGET_NAME} ALL
+    DEPENDS ${_ARGS_DEST}
+    COMMENT "${_comment}"
+  )
+endfunction()
+
+function(sym_link_file)
+  set(_options)
+  set(_one_value_args SOURCE DEST INSTALL_DEST)
+  set(_multi_value_args)
+  cmake_parse_arguments(_ARGS "${_options}" "${_one_value_args}" "${_multi_value_args}" ${ARGN})
+  file(
+    CREATE_LINK
+      ${_ARGS_SOURCE}
+      ${_ARGS_DEST}
+    SYMBOLIC
+  )
+  if (DEFINED _ARGS_INSTALL_DEST)
+    install(
+      FILES
+      ${_ARGS_SOURCE}
+      DESTINATION ${_ARGS_INSTALL_DEST}
+    )
+  endif()
+endfunction()
+
+function(sym_link_local_lib LIB_NAME)
+  set(_lib_file_name "${CMAKE_SHARED_LIBRARY_PREFIX}${LIB_NAME}.so")
+  file(
+    CREATE_LINK
+      "${CMAKE_BINARY_DIR}/src/lib_${LIB_NAME}/${_lib_file_name}"
+      "${CMAKE_CURRENT_BINARY_DIR}/${_lib_file_name}"
+    SYMBOLIC
+  )
+endfunction()

cmake/TernaryOperator.cmake

@@ -0,0 +1,9 @@
+include_guard(GLOBAL)
+
+macro(set_ternary RESULT CONDITION TRUE_VALUE FALSE_VALUE)
+  if(${CONDITION})
+    set(${RESULT} ${TRUE_VALUE})
+  else()
+    set(${RESULT} ${FALSE_VALUE})
+  endif()
+endmacro()

cmake/config_proto_files.h.in

@@ -0,0 +1,11 @@
+#pragma once
+
+#include <array>
+
+namespace bts::@PROJECT_NAME@::protobufs {
+
+inline std::array<const char *, @PROTO_SOURCES_LENGTH@> config_proto_files = {
+@PROTO_SOURCES_CXX_ARRAY_ITEMS@
+};
+
+} // namespace bts::@PROJECT_NAME@::protobufs

cmake/version.h.in

@@ -0,0 +1,11 @@
+#ifndef BTS_SPARK_VERSION_H
+#define BTS_SPARK_VERSION_H
+
+#define BTS_SPARK_VERSION_MAJOR @PROJECT_VERSION_MAJOR@
+#define BTS_SPARK_VERSION_MINOR @PROJECT_VERSION_MINOR@
+#define BTS_SPARK_VERSION_PATCH @PROJECT_VERSION_PATCH@
+#define BTS_SPARK_VERSION_BUILD_METADATA @PROJECT_VERSION_BUILD_METADATA@
+#define BTS_SPARK_VERSION "@PROJECT_VERSION@"
+#define BTS_SPARK_VERSION_FULL "@PROJECT_VERSION_FULL@"
+
+#endif

cmake/vs_filters.cmake

@@ -1,24 +0,0 @@
-cmake_minimum_required(VERSION 3.5)
-
-# Explicitly set policy to convert relative paths to absolute (although this is a default policy,
-# setting it explicitly prevents a compile-warning)
-cmake_policy(SET CMP0076 NEW)
-
-# add protos to the library to facilitate source_group
-target_sources(libspark PUBLIC ${Config_Protos} ${NonConfig_Protos} ${Service_Protos})
-
-function(assign_source_group)
-  foreach(_source IN ITEMS ${ARGN})
-    if(IS_ABSOLUTE "${_source}")
-      file(RELATIVE_PATH _source_rel "${CMAKE_CURRENT_SOURCE_DIR}" "${_source}")
-    else()
-      set(_source_rel "${_source}")
-    endif()
-    get_filename_component(_source_path "${_source_rel}" PATH)
-    string(REPLACE "/" "\\" _source_path_msvc "${_source_path}")
-    source_group("${_source_path_msvc}" FILES "${_source}")
-    set(SOURCE_FILES "${_source}")
-  endforeach()
-endfunction(assign_source_group)
-
-assign_source_group(${SparkLib_Sources} ${Config_Protos} ${NonConfig_Protos} ${Service_Protos})

compose.yaml

@@ -0,0 +1,104 @@
+services:
+  sparkd:
+    volumes:
+      - ${SPARKD__DATA_DIR:-sparkd_data}:/home/bts/sparkdata/data
+    build:
+      dockerfile: src/app_sparkd/Dockerfile
+      args:
+        - IMAGE=${SPARKD__IMAGE}
+        - SOURCE_DIR=${SPARKD__SOURCE_DIR}
+        - CONFIG_DIR=${SPARKD__CONFIG_DIR}
+    shm_size: "${SPARKD__SHM_SIZE:-512mb}"
+    command: >
+      sh -c "
+        echo bts | sudo -S chown -R bts:bts /home/bts/sparkdata/data
+        if [ -e "/home/bts/sparkdata/data/logging.sparkd/log" ]; then
+          mv "/home/bts/sparkdata/data/logging.sparkd/log" "/home/bts/sparkdata/data/logging.sparkd/log_$(date '+%Y%m%d-%H%M')"
+        fi
+        if [ -e "/home/bts/sparkdata/conf/.env.local" ]; then
+          echo 'Sourcing /home/bts/sparkdata/conf/.env.local'
+          set -a
+          . /home/bts/sparkdata/conf/.env.local
+          set +a
+        fi
+        ./sparkd run"
+    env_file:
+      - .env
+      - ${SPARKD__CONFIG_DIR}/.env
+    ports:
+      - "${SPARKD__CONFIG_PORT}:${SPARKD__CONFIG_PORT}"
+      - "${SPARKD__QUICKBUS_PORT}:${SPARKD__QUICKBUS_PORT}"
+      - "${SPARKD__SIM_FIX_PORT}:${SPARKD__SIM_FIX_PORT}"
+    deploy:
+      resources:
+        limits:
+          memory: "${SPARKD__MEMORY:-4gb}"
+
+  sparkd-symbols:
+    volumes:
+      - ${SPARKD__DATA_DIR:-sparkd_data}:/home/bts/sparkdata/data
+      - ${SPARKD__RUNTIME_CONF_DIR:-sparkd_conf}:/home/bts/sparkdata/conf
+    build:
+      dockerfile: src/server/sparkd/Dockerfile
+      args:
+        - IMAGE=${SPARKD__IMAGE}
+        - SOURCE_DIR=${SPARKD__SOURCE_DIR}
+        - CONFIG_DIR=${SPARKD__CONFIG_DIR}
+    shm_size: "${SPARKD__SHM_SIZE:-512mb}"
+    command: >
+      sh -c "
+        echo bts | sudo -S chown -R bts:bts /home/bts/sparkdata
+        if [ -e "/home/bts/sparkdata/data/logging.sparkd/log" ]; then
+          mv "/home/bts/sparkdata/data/logging.sparkd/log" "/home/bts/sparkdata/data/logging.sparkd/log_$(date '+%Y%m%d-%H%M')"
+        fi
+        ls -alh /home/bts/sparkdata/conf/
+        cat /home/bts/sparkdata/conf/.env.local
+        if [ -e "/home/bts/sparkdata/conf/.env.local" ]; then
+          echo 'Sourcing /home/bts/sparkdata/conf/.env.local'
+          set -a
+          . /home/bts/sparkdata/conf/.env.local
+          set +a
+        fi
+        ./sparkd run"
+    env_file:
+      - .env
+      - ${SPARKD__CONFIG_DIR}/.env
+    ports:
+      - "${SPARKD__CONFIG_PORT}:${SPARKD__CONFIG_PORT}"
+      - "${SPARKD__QUICKBUS_PORT}:${SPARKD__QUICKBUS_PORT}"
+      - "${SPARKD__SIM_FIX_PORT}:${SPARKD__SIM_FIX_PORT}"
+    deploy:
+      restart_policy:
+        condition: any
+        delay: 3s
+      resources:
+        limits:
+          memory: "${SPARKD__MEMORY:-4gb}"
+
+  sparkd-tail:
+    image: bluetradingsystems/linux_dotnet7
+    volumes:
+      - type: volume
+        read_only: true
+        source: sparkd_data
+        target: /home/bts/sparkdata/data/logging.sparkd
+        volume:
+          subpath: logging.sparkd
+    command: ["tail", "-f", "/home/bts/sparkdata/data/logging.sparkd/log"]
+
+  sparkd-ping-test:
+    depends_on:
+      - sparkd
+    profiles:
+      - ping-test
+    build:
+      context: .
+      dockerfile_inline: |
+        FROM bluetradingsystems/linux_dotnet7
+        USER "root"
+        RUN apt-get update && apt-get -y install iputils-ping
+    command: [ "ping", "sparkd" ]
+
+volumes:
+  sparkd_data:
+    name: "sparkd_${SPARKD__CONFIG_PORT:-8000}"

doc/api/content/client/getting-started/services.md

@@ -124,8 +124,8 @@ service OrderService {
 ```
 ### C++ Example
 ```cpp
-#include "lib/api/public/ServiceContext.h"
-#include "lib/api/services/OrderService.client.pb.h"
+#include "spark/api/public/ServiceContext.h"
+#include "spark/api/services/OrderService.client.pb.h"
 using namespace bts::api;
 ServiceContext ctx;
 auto orders = ctx.locate<orderman::OrderService>();

doc/api/content/client/services/marketdata.md

@@ -29,9 +29,9 @@ namespace sample {
 ```
 
 ```cpp
-#include "lib/api/public/ServiceContext.h"
-#include "spark/lib/domain_model/Security.h"
-#include "lib/api/services/MarketDataService.client.pb.h"
+#include "spark/api/public/ServiceContext.h"
+#include "spark/domain_model/Security.h"
+#include "spark/api/services/MarketDataService.client.pb.h"
 #include <thread>
 
 using namespace bts::api;

docker/images/bluetradingsystems/linux_dotnet7/Dockerfile

@@ -0,0 +1,38 @@
+FROM ubuntu:22.04
+
+ARG DEBIAN_FRONTEND=noninteractive
+
+RUN apt-get update && apt-get install -y pkg-config
+RUN apt-get update && apt-get install -y locales
+RUN apt-get update && apt-get install -y ca-certificates
+RUN apt-get update && apt-get install -y libssh2-1-dev
+RUN apt-get update && apt-get install -y software-properties-common
+RUN apt-get update && apt-get install -y sudo
+
+# GCC
+ARG gcc_major=13
+RUN add-apt-repository ppa:ubuntu-toolchain-r/test \
+  && apt-get update \
+  && apt-get install -y g++-$gcc_major
+
+RUN apt-get update \
+  && apt-get install -y dotnet7
+
+RUN locale-gen en_US.UTF-8
+
+ARG docker_user="bts"
+ARG docker_uid="1000"
+ARG docker_pass="bts"
+
+RUN \
+  groupadd -g "$docker_uid" "$docker_user" \
+  && useradd \
+  -m \
+  -u "$docker_uid" \
+  -g "$docker_uid" \
+  -G sudo \
+  --password "$(openssl passwd -1 "$docker_pass")" \
+  "$docker_user"
+
+USER bts
+WORKDIR /home/bts

requirements.txt

@@ -1,11 +1,20 @@
 astroid==2.7.3
 autopep8==1.5.7
+blinker==1.9.0
+boto3==1.37.9
+botocore==1.37.9
 certifi==2023.5.7
 charset-normalizer==3.1.0
+click==8.1.8
 colorama==0.4.4
+Flask==3.1.0
 idna==3.4
 isort==5.9.3
+itsdangerous==2.2.0
+Jinja2==3.1.6
+jmespath==1.0.1
 lazy-object-proxy==1.6.0
+MarkupSafe==3.0.2
 mccabe==0.6.1
 platformdirs==2.3.0
 protobuf==4.21.12
@@ -14,9 +23,14 @@ pycodestyle==2.7.0
 pycparser==2.20
 pylint==2.10.2
 PyMeta3==0.5.1
+python-dateutil==2.9.0.post0
+python-dotenv==1.0.1
 requests==2.31.0
+s3transfer==0.11.4
+six==1.17.0
 termcolor==1.1.0
 toml==0.10.2
 urllib3==2.0.3
+Werkzeug==3.1.3
 wrapt==1.12.1
 yapf==0.31.0

src/DataStreamSuite/ArithmeticExpressions/ArithmeticExpressions.csproj

@@ -1,7 +1,7 @@
 <Project Sdk="Microsoft.NET.Sdk">
 
   <PropertyGroup>
-    <TargetFramework>net7.0</TargetFramework>
+    <TargetFramework>net8.0</TargetFramework>
     <ImplicitUsings>enable</ImplicitUsings>
     <Nullable>enable</Nullable>
   </PropertyGroup>

src/DataStreamSuite/BaseUtilities/BaseUtilities.csproj

@@ -1,7 +1,7 @@
 <Project Sdk="Microsoft.NET.Sdk">
 
   <PropertyGroup>
-    <TargetFramework>net7.0</TargetFramework>
+    <TargetFramework>net8.0</TargetFramework>
     <ImplicitUsings>enable</ImplicitUsings>
     <Nullable>enable</Nullable>
   </PropertyGroup>

src/DataStreamSuite/BlueRegistry/BlueRegistry.csproj

@@ -1,7 +1,7 @@
 <Project Sdk="Microsoft.NET.Sdk">
 
   <PropertyGroup>
-    <TargetFramework>net7.0</TargetFramework>
+    <TargetFramework>net8.0</TargetFramework>
     <ImplicitUsings>enable</ImplicitUsings>
     <Nullable>enable</Nullable>
   </PropertyGroup>

src/DataStreamSuite/BooleanExpressions/BooleanExpressions.csproj

@@ -1,14 +1,14 @@
 <Project Sdk="Microsoft.NET.Sdk">
 
   <PropertyGroup>
-    <TargetFramework>net7.0</TargetFramework>
+    <TargetFramework>net8.0</TargetFramework>
     <ImplicitUsings>enable</ImplicitUsings>
     <Nullable>enable</Nullable>
   </PropertyGroup>
 
   <ItemGroup>
     <PackageReference Include="Google.Protobuf" Version="3.24.3" />
-    <PackageReference Include="YaccLexTools" Version="1.2.0" />
+    <PackageReference Include="YaccLexTools" Version="1.2.3" />
   </ItemGroup>
 
   <ItemGroup>

src/DataStreamSuite/BooleanExpressions/GplexBuffers.cs

@@ -24,8 +24,6 @@ namespace QUT.GplexBuffers
         public BufferException(string message) : base(message) { }
         public BufferException(string message, Exception innerException)
             : base(message, innerException) { }
-        protected BufferException(SerializationInfo info, StreamingContext context)
-            : base(info, context) { }
     }
 
     internal abstract class ScanBuff
@@ -60,12 +58,10 @@ namespace QUT.GplexBuffers
             return new BuildBuffer(source);
         }
 
-#if (!BYTEMODE)
         public static ScanBuff GetBuffer(Stream source, int fallbackCodePage)
         {
             return new BuildBuffer(source, fallbackCodePage);
         }
-#endif // !BYTEMODE
 #endif // !NOFILES
     }
 
@@ -353,17 +349,15 @@ namespace QUT.GplexBuffers
         {
             FileStream fStrm = (stream as FileStream);
             if (fStrm != null) FileName = fStrm.Name;
-            NextBlk = BlockReaderFactory.Raw(stream);
+            NextBlk = BlockReaderFactory.Get(stream);
         }
 
-#if (!BYTEMODE)
         public BuildBuffer(Stream stream, int fallbackCodePage)
         {
             FileStream fStrm = (stream as FileStream);
             if (fStrm != null) FileName = fStrm.Name;
             NextBlk = BlockReaderFactory.Get(stream, fallbackCodePage);
         }
-#endif
 
         /// <summary>
         /// Marks a conservative lower bound for the buffer,
@@ -434,43 +428,19 @@ namespace QUT.GplexBuffers
     //
     internal static class BlockReaderFactory
     {
-        public static BlockReader Raw(Stream stream)
-        {
-            return delegate(char[] block, int index, int number)
-            {
-                byte[] b = new byte[number];
-                int count = stream.Read(b, 0, number);
-                int i = 0;
-                int j = index;
-                for (; i < count; i++, j++)
-                    block[j] = (char)b[i];
-                return count;
-            };
-        }
 
-#if (!BYTEMODE)
-        public static BlockReader Get(Stream stream, int fallbackCodePage)
+        public static BlockReader Get(Stream stream, int fallbackCodePage = -2)
         {
             Encoding encoding;
             int preamble = Preamble(stream);
 
             if (preamble != 0)  // There is a valid BOM here!
                 encoding = Encoding.GetEncoding(preamble);
-            else if (fallbackCodePage == -1) // Fallback is "raw" bytes
-                return Raw(stream);
-            else if (fallbackCodePage != -2) // Anything but "guess"
+            else if (fallbackCodePage >= 0) // Anything but "guess"
                 encoding = Encoding.GetEncoding(fallbackCodePage);
             else // This is the "guess" option
-            {
-                int guess = new Guesser(stream).GuessCodePage();
-                stream.Seek(0, SeekOrigin.Begin);
-                if (guess == -1) // ==> this is a 7-bit file
-                    encoding = Encoding.ASCII;
-                else if (guess == 65001)
-                    encoding = Encoding.UTF8;
-                else             // ==> use the machine default
-                    encoding = Encoding.Default;
-            }
+                encoding = GuessEncoding(stream);
+
             StreamReader reader = new StreamReader(stream, encoding);
             return reader.Read;
         }
@@ -488,6 +458,14 @@ namespace QUT.GplexBuffers
             int b2 = stream.ReadByte();
             if (b0 == 0xef && b1 == 0xbb && b2 == 0xbf)
                 return 65001; // UTF8
+
+            int b3 = stream.ReadByte();
+            // Check for UTF-32 BOM (Byte Order Mark)
+            if (b0 == 0x00 && b1 == 0x00 && b2 == 0xFE && b3 == 0xFF)
+                return 12001; // UTF-32BE
+            if (b0 == 0xFF && b1 == 0xFE && b2 == 0x00 && b3 == 0x00)
+                return 12000; // UTF-32LE
+
             //
             // There is no unicode preamble, so we
             // return denoter for the machine default.
@@ -495,7 +473,150 @@ namespace QUT.GplexBuffers
             stream.Seek(0, SeekOrigin.Begin);
             return 0;
         }
-#endif // !BYTEMODE
+
+        private static Encoding GuessEncoding(Stream stream)
+        {
+            stream.Seek(0, SeekOrigin.Begin);
+            if (IsUtf16(stream, out Encoding encoding))
+            {
+                stream.Seek(0, SeekOrigin.Begin);
+                return encoding;
+            }
+
+            stream.Seek(0, SeekOrigin.Begin);
+            if (IsUtf8(stream))
+            {
+                stream.Seek(0, SeekOrigin.Begin);
+                return Encoding.UTF8;
+            }
+
+            stream.Seek(0, SeekOrigin.Begin);
+            if (IsAscii(stream))
+            {
+                stream.Seek(0, SeekOrigin.Begin);
+                return Encoding.ASCII;
+            }
+
+#if !NET462_OR_GREATER
+            stream.Seek(0, SeekOrigin.Begin);
+            if (IsLatin1(stream))
+            {
+                stream.Seek(0, SeekOrigin.Begin);
+                return Encoding.Latin1;
+            }
+#endif
+            stream.Seek(0, SeekOrigin.Begin);
+            return Encoding.Default;
+        }
+
+        private static bool IsUtf8(Stream stream)
+        {
+            int b;
+            while ((b = stream.ReadByte()) != -1)
+            {
+                if ((b & 0x80) == 0) // 0xxxxxxx
+                {
+                    continue;
+                }
+                else if ((b & 0xE0) == 0xC0) // 110xxxxx 10xxxxxx
+                {
+                    if (!IsValidUtf8ContinuationByte(stream)) return false;
+                }
+                else if ((b & 0xF0) == 0xE0) // 1110xxxx 10xxxxxx 10xxxxxx
+                {
+                    if (!IsValidUtf8ContinuationByte(stream) || !IsValidUtf8ContinuationByte(stream)) return false;
+                }
+                else if ((b & 0xF8) == 0xF0) // 11110xxx 10xxxxxx 10xxxxxx 10xxxxxx
+                {
+                    if (!IsValidUtf8ContinuationByte(stream) || !IsValidUtf8ContinuationByte(stream) || !IsValidUtf8ContinuationByte(stream)) return false;
+                }
+                else
+                {
+                    return false;
+                }
+            }
+            return true;
+        }
+
+        private static bool IsValidUtf8ContinuationByte(Stream stream)
+        {
+            int b = stream.ReadByte();
+            return b != -1 && (b & 0xC0) == 0x80; // 10xxxxxx
+        }
+
+        private static bool IsUtf16(Stream stream, out Encoding encoding)
+        {
+            encoding = Encoding.Default;
+
+            if (stream.Length >= 2)
+            {
+                // Read the first two bytes
+                int b0 = stream.ReadByte();
+                int b1 = stream.ReadByte();
+
+                // Check for UTF-16 BOM (Byte Order Mark)
+                if (b0 == 0xFF && b1 == 0xFE)
+                {
+                    encoding = Encoding.Unicode;
+                    return true; // UTF-16LE
+                }
+                if (b0 == 0xFE && b1 == 0xFF)
+                {
+                    encoding = Encoding.BigEndianUnicode;
+                    return true; // UTF-16BE
+                }
+
+                // Reset the stream position
+                stream.Seek(0, SeekOrigin.Begin);
+
+                // Check for common UTF-16 patterns
+                byte[] buffer = new byte[2];
+                while (stream.Read(buffer, 0, 2) == 2)
+                {
+                    if (buffer[0] == 0x00 && buffer[1] != 0x00)
+                    {
+                        encoding = Encoding.BigEndianUnicode;
+                        return true; // Likely UTF-16BE
+                    }
+                    if (buffer[0] != 0x00 && buffer[1] == 0x00)
+                    {
+                        encoding = Encoding.Unicode;
+                        return true; // Likely UTF-16LE
+                    }
+                }
+            }
+
+            return false;
+        }
+
+        private static bool IsAscii(Stream stream)
+        {
+            int b;
+            while ((b = stream.ReadByte()) != -1)
+            {
+                // Latin1 includes byte values from 0x00 to 0xFF
+                if (b > 0x7F)
+                {
+                    return false;
+                }
+            }
+            return true;
+        }
+
+        private static bool IsLatin1(Stream stream)
+        {
+            int b;
+            while ((b = stream.ReadByte()) != -1)
+            {
+                // Latin1 includes byte values from 0x00 to 0xFF
+                if (b > 0xFF)
+                {
+                    return false;
+                }
+            }
+            return true;
+        }
+
     }
 #endif // !NOFILES
     #endregion Buffer classes
@@ -540,180 +661,6 @@ namespace QUT.GplexBuffers
             return 0;
         }
     }
-#region guesser
-#if (!BYTEMODE)
-    // ==============================================================
-    // ============          Encoding Guesser           =============
-    // ==============================================================
-
-    /// <summary>
-    /// This class provides a simple finite state automaton that
-    /// scans the file looking for (1) valid UTF-8 byte patterns,
-    /// (2) bytes >= 0x80 which are not part of a UTF-8 sequence.
-    /// The method then guesses whether it is UTF-8 or maybe some 
-    /// local machine default encoding.  This works well for the
-    /// various Latin encodings.
-    /// </summary>
-    internal class Guesser
-    {
-        ScanBuff buffer;
-
-        public int GuessCodePage() { return Scan(); }
-
-        const int maxAccept = 10;
-        const int initial = 0;
-        const int eofNum = 0;
-        const int goStart = -1;
-        const int INITIAL = 0;
-        const int EndToken = 0;
-
-        #region user code
-        /* 
-         *  Reads the bytes of a file to determine if it is 
-         *  UTF-8 or a single-byte code page file.
-         */
-        public long utfX;
-        public long uppr;
-        #endregion user code
-
-        int state;
-        int currentStart = startState[0];
-        int code;
-
-        #region ScannerTables
-        static int[] startState = new int[] { 11, 0 };
-
-        #region CharacterMap
-        static sbyte[] map = new sbyte[256] {
-/*     '\0' */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 
-/*   '\x10' */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 
-/*   '\x20' */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 
-/*      '0' */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 
-/*      '@' */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 
-/*      'P' */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 
-/*      '`' */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 
-/*      'p' */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 
-/*   '\x80' */ 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 
-/*   '\x90' */ 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 
-/*   '\xA0' */ 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 
-/*   '\xB0' */ 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 
-/*   '\xC0' */ 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 
-/*   '\xD0' */ 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 
-/*   '\xE0' */ 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 
-/*   '\xF0' */ 4, 4, 4, 4, 4, 4, 4, 4, 5, 5, 5, 5, 5, 5, 5, 5 };
-        #endregion
-
-        static sbyte[][] nextState = new sbyte[][] {
-            new sbyte[] {0, 0, 0, 0, 0, 0},
-            new sbyte[] {-1, -1, 10, -1, -1, -1},
-            new sbyte[] {-1, -1, -1, -1, -1, -1},
-            new sbyte[] {-1, -1, 8, -1, -1, -1},
-            new sbyte[] {-1, -1, 5, -1, -1, -1},
-            new sbyte[] {-1, -1, 6, -1, -1, -1},
-            new sbyte[] {-1, -1, 7, -1, -1, -1},
-            null,
-            new sbyte[] {-1, -1, 9, -1, -1, -1},
-            null,
-            null,
-            new sbyte[] {-1, 1, 2, 3, 4, 2}
-        };
-
-
-        [SuppressMessage("Microsoft.Performance", "CA1810:InitializeReferenceTypeStaticFieldsInline")]
-        // Reason for suppression: cannot have self-reference in array initializer.
-        static Guesser()
-        {
-            nextState[7] = nextState[2];
-            nextState[9] = nextState[2];
-            nextState[10] = nextState[2];
-        }
-
-        int NextState()
-        {
-            if (code == ScanBuff.EndOfFile)
-                return eofNum;
-            else
-                return nextState[state][map[code]];
-        }
-        #endregion
-
-        public Guesser(System.IO.Stream file) { SetSource(file); }
-
-        public void SetSource(System.IO.Stream source)
-        {
-            this.buffer = new BuildBuffer(source);
-            code = buffer.Read();
-        }
-
-        int Scan()
-        {
-            for (; ; )
-            {
-                int next;
-                state = currentStart;
-                while ((next = NextState()) == goStart)
-                    code = buffer.Read();
-
-                state = next;
-                code = buffer.Read();
-
-                while ((next = NextState()) > eofNum)
-                {
-                    state = next;
-                    code = buffer.Read();
-                }
-                if (state <= maxAccept)
-                {
-                    #region ActionSwitch
-#pragma warning disable 162
-                    switch (state)
-                    {
-                        case eofNum:
-                            switch (currentStart)
-                            {
-                                case 11:
-                                    if (utfX == 0 && uppr == 0) return -1; /* raw ascii */
-                                    else if (uppr * 10 > utfX) return 0;   /* default code page */
-                                    else return 65001;                     /* UTF-8 encoding */
-                                    break;
-                            }
-                            return EndToken;
-                        case 1: // Recognized '{Upper128}',	Shortest string "\xC0"
-                        case 2: // Recognized '{Upper128}',	Shortest string "\x80"
-                        case 3: // Recognized '{Upper128}',	Shortest string "\xE0"
-                        case 4: // Recognized '{Upper128}',	Shortest string "\xF0"
-                            uppr++;
-                            break;
-                        case 5: // Recognized '{Utf8pfx4}{Utf8cont}',	Shortest string "\xF0\x80"
-                            uppr += 2;
-                            break;
-                        case 6: // Recognized '{Utf8pfx4}{Utf8cont}{2}',	Shortest string "\xF0\x80\x80"
-                            uppr += 3;
-                            break;
-                        case 7: // Recognized '{Utf8pfx4}{Utf8cont}{3}',	Shortest string "\xF0\x80\x80\x80"
-                            utfX += 3;
-                            break;
-                        case 8: // Recognized '{Utf8pfx3}{Utf8cont}',	Shortest string "\xE0\x80"
-                            uppr += 2;
-                            break;
-                        case 9: // Recognized '{Utf8pfx3}{Utf8cont}{2}',	Shortest string "\xE0\x80\x80"
-                            utfX += 2;
-                            break;
-                        case 10: // Recognized '{Utf8pfx2}{Utf8cont}',	Shortest string "\xC0\x80"
-                            utfX++;
-                            break;
-                        default:
-                            break;
-                    }
-#pragma warning restore 162
-                    #endregion
-                }
-            }
-        }
-    } // end class Guesser
-    
-#endif // !BYTEMODE
-#endregion
 #endif // !NOFILES
 
 // End of code copied from embedded resource

src/DataStreamSuite/Bts.ColumnExpressions/Bts.ColumnExpressions.csproj

@@ -1,7 +1,7 @@
 <Project Sdk="Microsoft.NET.Sdk">
 
   <PropertyGroup>
-    <TargetFramework>net7.0</TargetFramework>
+    <TargetFramework>net8.0</TargetFramework>
     <ImplicitUsings>enable</ImplicitUsings>
     <Nullable>enable</Nullable>
   </PropertyGroup>

src/DataStreamSuite/Bts.ColumnSortExpressions/Bts.ColumnSortExpressions.csproj

@@ -1,7 +1,7 @@
 <Project Sdk="Microsoft.NET.Sdk">
 
   <PropertyGroup>
-    <TargetFramework>net7.0</TargetFramework>
+    <TargetFramework>net8.0</TargetFramework>
     <ImplicitUsings>enable</ImplicitUsings>
     <Nullable>enable</Nullable>
   </PropertyGroup>

src/DataStreamSuite/ColumnMapExpressions/ColumnMapExpressions.csproj

@@ -1,7 +1,7 @@
 <Project Sdk="Microsoft.NET.Sdk">
 
   <PropertyGroup>
-    <TargetFramework>net7.0</TargetFramework>
+    <TargetFramework>net8.0</TargetFramework>
     <ImplicitUsings>enable</ImplicitUsings>
     <Nullable>enable</Nullable>
   </PropertyGroup>

src/DataStreamSuite/DataStream/DataStream.csproj

@@ -1,7 +1,7 @@
 <Project Sdk="Microsoft.NET.Sdk">
 
   <PropertyGroup>
-    <TargetFramework>net7.0</TargetFramework>
+    <TargetFramework>net8.0</TargetFramework>
     <Nullable>enable</Nullable>
   </PropertyGroup>
 

src/DataStreamSuite/DataStream/Primitives/Join/NonIncremental/NonIncJoinBookkeeper.cs

@@ -74,7 +74,7 @@ namespace BCG {
       }
 
       public void _on_stream_cb(object dp_obj) {
-         Log.Gen.info($"NonIncJoinBookkeeper._on_stream_cb dp_obj: {dp_obj}");
+         //Log.Gen.info($"NonIncJoinBookkeeper._on_stream_cb dp_obj: {dp_obj}");
          var timer = Stopwatch.StartNew();
          DataPacket dp = (DataPacket)dp_obj;
          _fill_missing_data(dp);

src/DataStreamSuite/DataStream/Primitives/Join/NonIncremental/NonIncJoinStream.cs

@@ -50,7 +50,7 @@ namespace BCG {
          int data_col_no = 1;
          for (int jp = 0; jp < _join_specs.Length; jp++) {
             NonIncJoinBookkeeper s = new NonIncJoinBookkeeper(_join_specs[jp], jp, data_col_no, _row_hasher);
-            Log.Gen.info($"NonIncJoinStream.myOnFirstSubscriber jp: {jp} spec: {_join_specs[jp]}");
+            //Log.Gen.info($"NonIncJoinStream.myOnFirstSubscriber jp: {jp} spec: {_join_specs[jp]}");
             data_col_no += _join_specs[jp].data_col_names.Length;
             s.on_new_data += _stream_callback;
             _book_keeping_data.Add(s);
@@ -71,7 +71,7 @@ namespace BCG {
       }
 
       private void _stream_callback() {
-         Log.Gen.info("NonIncJoinStream._stream_callback received callback");
+         //Log.Gen.info("NonIncJoinStream._stream_callback received callback");
          _is_dirty = true;
          doWork();
       }
@@ -99,7 +99,7 @@ namespace BCG {
 
       private int _n_total_fires = 0, _n_expensive_fires = 0;
       private void _form_and_fire() {
-         Log.Gen.info($"NonIncJoinStream._form_and_fire _is_ready(): {_is_ready()}");
+         //Log.Gen.info($"NonIncJoinStream._form_and_fire _is_ready(): {_is_ready()}");
          if (_is_ready()) {
             _construct_schema();
             var timer = Stopwatch.StartNew();
@@ -126,11 +126,11 @@ namespace BCG {
             _n_total_fires++;
             //if (timer.ElapsedMilliseconds >= 250) {
             _n_expensive_fires++;
-            string log_msg = $"Join and map time for {_book_keeping_data.Count} "
-               + $"tables {timer.Elapsed.TotalMilliseconds} msecs, ratio {_n_expensive_fires}/{_n_total_fires}";
-            Log.Gen.info(log_msg);
+            //string log_msg = $"Join and map time for {_book_keeping_data.Count} "
+            //   + $"tables {timer.Elapsed.TotalMilliseconds} msecs, ratio {_n_expensive_fires}/{_n_total_fires}";
+            //Log.Gen.info(log_msg);
             //}
-            var dp_out = BlueDataUtility.build_dp_inc(dp_output_rows, myLastValue as DataPacket);
+            var dp_out = BlueDataUtility.build_dp_inc(dp_output_rows, myLastValue as DataPacket, true);
             myFireEvent(dp_out);
             //myFireEvent(new DataPacket(dp_output_rows, dp_output_rows, myLastValue == null ? new BlueTable(_output_schema) : (myLastValue as DataPacket).newRows));
          }
@@ -152,7 +152,7 @@ namespace BCG {
       }
 
       public void doWork() {
-         Log.Gen.info($"NonIncJoinStream.doWork _last_fire_time: {_last_fire_time} _is_dirty: {_is_dirty}");
+         //Log.Gen.info($"NonIncJoinStream.doWork _last_fire_time: {_last_fire_time} _is_dirty: {_is_dirty}");
          if (DateTime.Now - _last_fire_time >= _throttle_period && _is_dirty) {
             _is_dirty = false;
             _last_fire_time = DateTime.Now;

src/DataStreamSuite/DataStream/Primitives/Time/DateTimeExpanstionStream.cs

@@ -4,7 +4,7 @@ namespace bts.SilverPine {
    using BCG;
 
    using System;
-
+   [DataStreamUserRespresentation("DateTimeExpanderStream")]
    public class DateTimeExpanstionStream : ViewStream {
       public DateTimeExpanstionStream(ViewStream parent_stream, string inp_col) {
          Func<IBlueTableSchema, IBlueTableSchema> get_schema = (s) => {
@@ -24,11 +24,11 @@ namespace bts.SilverPine {
             IBlueTableSchema s = get_schema(r.schema);
             MutableBlueRow rv = new MutableBlueRow(s, r);
             rv.setValue($"{inp_col}_date", dt.Date);
-            rv.setValue($"{inp_col}_day", dt.DayOfWeek.ToString());
+            rv.setValue($"{inp_col}_day_of_week", dt.DayOfWeek.ToString());
             rv.setValue($"{inp_col}_day_of_month", dt.Day);
             rv.setValue($"{inp_col}_month", dt.Date.Month);
             rv.setValue($"{inp_col}_hour", dt.Hour);
-            rv.setValue($"{inp_col}_min", dt.Minute);
+            rv.setValue($"{inp_col}_minute", dt.Minute);
             rv.setValue($"{inp_col}_seconds_of_day", (dt - dt.Date).TotalSeconds);
             return rv;
          });

src/DataStreamSuite/DataStream/Primitives/Time/SecondStream.cs

@@ -37,4 +37,42 @@ namespace BCG
       }
          
    }
+
+   [DataStreamUserRespresentation("throttle")]
+   public class ThrottleStream : ViewStream{
+      private readonly ViewStream _main_stream;
+      private readonly TimeSpan _ts;
+      private readonly ViewStreamDatabase _vsd;
+      private DateTime _last_fire_time = DateTime.MinValue;
+
+      public ThrottleStream (ViewStream main_stream, int throttle_seconds)
+      :
+      this(main_stream, TimeSpan.FromSeconds(throttle_seconds), ViewStreamDatabase.instance)
+      {
+      }
+
+      public ThrottleStream (ViewStream main_stream, TimeSpan ts, ViewStreamDatabase vsd){
+         _main_stream = main_stream;
+         _ts = ts;
+         _vsd = vsd;
+      }
+
+       protected override void myOnFirstSubscriber() {
+         _main_stream.onChange += _handle_base_stream_update;
+      }
+
+      protected override void myOnLastSubscriber() {
+         _main_stream.onChange -= _handle_base_stream_update;
+      }
+
+      private void _handle_base_stream_update(object dp_obj){
+         if (DateTime.Now - _last_fire_time >= _ts){
+            DataPacket dp_in = dp_obj as DataPacket;
+            _last_fire_time = DateTime.Now;
+            DataPacket dp = BlueDataUtility.build_dp_inc(dp_in.newRows, myLastValue as DataPacket);
+            myFireEvent(dp);
+         }
+      }
+ }
+
 }

src/DataStreamSuite/DataStream/Primitives/Time/ThrottleStream

@@ -0,0 +1,40 @@
+namespace bts.SilverPine{
+   using BGC;
+   using System;
+   [DataStreamUserRespresentation("throttle")]
+   public class ThrottleStream : ViewStream{
+      private readonly ViewStream _main_stream;
+      private readonly TimeSpan _ts;
+      private readonly ViewStreamDatabase _vsd;
+      private DateTime _last_fire_time = DateTime.MinValue;
+
+      public ThrottleStream (ViewStream main_stream, int throttle_seconds)
+      :
+      this(main_stream, TimeSpan.FromSeconds(throttle_seconds), ViewStreamDatabase.instance)
+      {
+      }
+
+      public ThrottleStream (ViewStream main_stream, TimeSpan ts, ViewStreamDatabase vsd){
+         _main_stream = main_stream;
+         _ts = ts;
+         _vsd;
+      }
+
+       protected override void myOnFirstSubscriber() {
+         _main_stream.onChange += _handle_base_stream_update;
+      }
+
+      protected override void myOnLastSubscriber() {
+         _main_stream.onChange -= _handle_base_stream_update;
+      }
+
+      private void _handle_base_stream_update(object dp_obj){
+         if (DateTime.Now - _last_fire_time >= _ts){
+            DataPacket dp_in = dp_obj;
+            _last_fire_time = DateTime.Now;
+            DataPacket dp = BlueDataUtility.build_dp_inc(dp_in.newRow, myLastValue as DataPacket);
+            myFireEvent(dp);
+         }
+      }
+ }
+}

src/DataStreamSuite/DataStreamCore/DataStreamCore.csproj

@@ -2,7 +2,7 @@
 
   <PropertyGroup>
     <AllowUnsafeBlocks>true</AllowUnsafeBlocks>
-    <TargetFramework>net7.0</TargetFramework>
+    <TargetFramework>net8.0</TargetFramework>
     <Nullable>disable</Nullable>
   </PropertyGroup>
 
@@ -19,8 +19,8 @@
   </ItemGroup>
 
   <ItemGroup>
-    <Proto Include="../../lib/config/protos/data_stream_config.proto" CSName="DataStreamConfig">
-      <OutDir>$(ProtoOutputPath)/lib/config/protos</OutDir>
+    <Proto Include="../../lib_spark/spark/config/protos/data_stream_config.proto" CSName="DataStreamConfig">
+      <OutDir>$(ProtoOutputPath)/lib_spark/spark/config/protos</OutDir>
     </Proto>
   </ItemGroup>
 

src/DataStreamSuite/DataStreamExpressions/DataStreamExpressions.csproj

@@ -1,7 +1,7 @@
 <Project Sdk="Microsoft.NET.Sdk">
 
   <PropertyGroup>
-    <TargetFramework>net7.0</TargetFramework>
+    <TargetFramework>net8.0</TargetFramework>
     <ImplicitUsings>enable</ImplicitUsings>
     <Nullable>enable</Nullable>
   </PropertyGroup>

src/DataStreamSuite/DataStreamPublisher/DataStreamPublisher.csproj

@@ -2,7 +2,7 @@
 
   <PropertyGroup>
     <OutputType>Exe</OutputType>
-    <TargetFramework>net7.0</TargetFramework>
+    <TargetFramework>net8.0</TargetFramework>
     <Nullable>enable</Nullable>
     <AllowUnsafeBlocks>true</AllowUnsafeBlocks>
   </PropertyGroup>

src/DataStreamSuite/DataStreamQueryAnalyzer/DataStreamQueryAnalyzer.csproj

@@ -2,7 +2,7 @@
 
   <PropertyGroup>
     <OutputType>Exe</OutputType>
-    <TargetFramework>net7.0-windows</TargetFramework>
+    <TargetFramework>net8.0-windows</TargetFramework>
     <Nullable>enable</Nullable>
     <UseWindowsForms>true</UseWindowsForms>
   </PropertyGroup>

src/DataStreamSuite/DataStreamSuite.sln

@@ -36,6 +36,8 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Bts.ColumnExpressions", "Bt
 EndProject
 Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Bts.ColumnSortExpressions", "Bts.ColumnSortExpressions\Bts.ColumnSortExpressions.csproj", "{B038CB72-4E81-4AF1-976F-ED8655C2C3D6}"
 EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "DataStreamQueryAnalyzer", "DataStreamQueryAnalyzer\DataStreamQueryAnalyzer.csproj", "{BFA5BA5C-927A-41A9-A696-1E2F4B5FFD15}"
+EndProject
 Global
 	GlobalSection(SolutionConfigurationPlatforms) = preSolution
 		Debug|Any CPU = Debug|Any CPU
@@ -113,5 +115,9 @@ Global
 		{B038CB72-4E81-4AF1-976F-ED8655C2C3D6}.Debug|Any CPU.Build.0 = Debug|Any CPU
 		{B038CB72-4E81-4AF1-976F-ED8655C2C3D6}.Release|Any CPU.ActiveCfg = Release|Any CPU
 		{B038CB72-4E81-4AF1-976F-ED8655C2C3D6}.Release|Any CPU.Build.0 = Release|Any CPU
+		{BFA5BA5C-927A-41A9-A696-1E2F4B5FFD15}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{BFA5BA5C-927A-41A9-A696-1E2F4B5FFD15}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{BFA5BA5C-927A-41A9-A696-1E2F4B5FFD15}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{BFA5BA5C-927A-41A9-A696-1E2F4B5FFD15}.Release|Any CPU.Build.0 = Release|Any CPU
 	EndGlobalSection
 EndGlobal

src/DataStreamSuite/DataStreamTests/DataStreamTests.csproj

@@ -1,7 +1,7 @@
 <Project Sdk="Microsoft.NET.Sdk">
 
   <PropertyGroup>
-    <TargetFramework>net7.0</TargetFramework>
+    <TargetFramework>net8.0</TargetFramework>
     <ImplicitUsings>enable</ImplicitUsings>
     <Nullable>enable</Nullable>
 

src/DataStreamSuite/ExpressionCore/ExpressionCore.csproj

@@ -1,7 +1,7 @@
 <Project Sdk="Microsoft.NET.Sdk">
 
   <PropertyGroup>
-    <TargetFramework>net7.0</TargetFramework>
+    <TargetFramework>net8.0</TargetFramework>
     <ImplicitUsings>enable</ImplicitUsings>
     <Nullable>enable</Nullable>
   </PropertyGroup>

src/DataStreamSuite/GuiUtilities/GuiUtilities.csproj

@@ -1,7 +1,7 @@
 <Project Sdk="Microsoft.NET.Sdk">
 
   <PropertyGroup>
-    <TargetFramework>net7.0-windows</TargetFramework>
+    <TargetFramework>net8.0-windows</TargetFramework>
     <ImplicitUsings>enable</ImplicitUsings>
     <Nullable>enable</Nullable>
     <UseWindowsForms>true</UseWindowsForms>

src/DataStreamSuite/PubSub/Client/PublishingUtilities/IPublisher.cs

@@ -6,12 +6,18 @@ namespace BCG
 	public interface IPublisher
 	{
       void publish( IPubItem pi );
+#pragma warning disable SYSLIB0011 // Type or member is obsolete
       void publish( IPubItem pi, IFormatter f );
+#pragma warning restore SYSLIB0011 // Type or member is obsolete
       void publish( IPubItem[] pi );
+#pragma warning disable SYSLIB0011 // Type or member is obsolete
       void publish( IPubItem[] pi, IFormatter f );
+#pragma warning restore SYSLIB0011 // Type or member is obsolete
    }
 
+#pragma warning disable SYSLIB0011 // Type or member is obsolete
    public interface IPublisher<T> where T : IFormatter
+#pragma warning restore SYSLIB0011 // Type or member is obsolete
    {
       void publish (IPubItem pi);
       void publish (IPubItem[] pi);

src/DataStreamSuite/PubSub/Misc/GenericFormatter.cs

@@ -5,23 +5,21 @@ using System.Runtime.Serialization.Formatters.Binary;
 
 namespace BCG
 {
+#pragma warning disable SYSLIB0011 // Type or member is obsolete
    public class GenericFormatter<F> : IGenericFormatter where F : IFormatter, new ()
    {
       IFormatter myFormatter = new F ();
 
       public T Deserialize<T> (Stream serializationStream)
       {
-         #pragma warning disable 
          return (T) myFormatter.Deserialize (serializationStream);
-         #pragma warning restore SYSLIB0011
       }
       public void Serialize<T> (Stream serializationStream, T graph)
       {
-         #pragma warning disable 
          myFormatter.Serialize (serializationStream, graph);
-         #pragma warning restore SYSLIB0011
       }
    }
 
    public class GenericBinaryFormatter : GenericFormatter<BinaryFormatter> { }
+#pragma warning restore SYSLIB0011 // Type or member is obsolete
 }

src/DataStreamSuite/PubSub/PubSub.csproj

@@ -6,7 +6,7 @@
 
   <PropertyGroup>
       <AllowUnsafeBlocks>true</AllowUnsafeBlocks>
-    <TargetFramework>net7.0</TargetFramework>
+    <TargetFramework>net8.0</TargetFramework>
     <Nullable>enable</Nullable>
   </PropertyGroup>
 

src/DataStreamSuite/PubSubServer/PubSubServer.csproj

@@ -2,7 +2,7 @@
 
   <PropertyGroup>
     <OutputType>Exe</OutputType>
-    <TargetFramework>net7.0</TargetFramework>
+    <TargetFramework>net8.0</TargetFramework>
     <Nullable>enable</Nullable>
   </PropertyGroup>
 

src/DataStreamSuite/SharedStreams/Business/MarketData/Analytics/SpatialArbScoreStream.cs

@@ -0,0 +1,17 @@
+namespace bts.SilverPine {
+   using BCG;
+   using System;
+
+   [DataStreamUserRespresentation("SpatialArbScoreStream")]
+   public class SpatialArbScoreStream : ViewStream {
+      public SpatialArbScoreStream(int window_sz_secs, int sample_period_sz)
+         : this(TimeSpan.FromSeconds(window_sz_secs), TimeSpan.FromSeconds(sample_period_sz), ViewStreamDatabase.instance) { }
+
+      public SpatialArbScoreStream(TimeSpan window_sz, TimeSpan sample_period, ViewStreamDatabase vsd) {
+         ViewStream rs = new SpatialArbStatsStream(window_sz, sample_period, vsd);
+         rs = new ThrottleStream(rs, TimeSpan.FromSeconds(1), vsd);
+         rs = new SortStream(rs, "-n_trade_ticks");
+         mySetRealSource(rs);
+      }
+   }
+}

src/DataStreamSuite/SharedStreams/Business/MarketData/Analytics/SpatialArbStatsStream.cs

@@ -0,0 +1,113 @@
+namespace bts.SilverPine {
+   using BCG;
+
+   using SharpCompress.Common.Rar;
+
+   using System;
+   using System.Collections.Generic;
+   using System.Linq;
+   using System.Text;
+   using System.Threading.Tasks;
+
+   // For all the pairs generated by SpatialFuturePairsStream, this stream 
+   // generates the statistics, by appending together a SpreadStatsStream
+   [DataStreamUserRespresentation("SpatialArbStatsStream")]
+   public class SpatialArbStatsStream : ViewStream{
+      private ViewStream _spatial_pairs_stream;
+      private ViewStreamDatabase _vsd;
+      private ViewStream _tot_str = null;
+      private HashSet<string> _key_map = new HashSet<string>();
+      private ViewStream _bbo_stream;
+      private readonly TimeSpan _window_sz, _sampling_period_sz;
+
+      public SpatialArbStatsStream(int window_sz_seconds, int sampling_period_sz_seconds)
+         : this(TimeSpan.FromSeconds(window_sz_seconds), TimeSpan.FromSeconds(sampling_period_sz_seconds), ViewStreamDatabase.instance) { }
+
+      public SpatialArbStatsStream(TimeSpan window_sz, TimeSpan sampling_period_sz, ViewStreamDatabase vsd) {
+         _vsd = vsd;
+         _window_sz = window_sz;
+         _sampling_period_sz = sampling_period_sz;
+         _spatial_pairs_stream = new SpatialFuturePairsStream(vsd);
+         _bbo_stream = new BBOStream(vsd);
+      }
+
+      protected override void myOnFirstSubscriber() {
+         _spatial_pairs_stream.onChange += _spatial_pairs_stream_onchange;
+      }
+
+      protected override void myOnLastSubscriber() {
+         _spatial_pairs_stream.onChange -= _spatial_pairs_stream_onchange;
+      }
+
+      private ViewStream _build_mapped_stream(ViewStream parent_str) {
+         Func<IBlueTableSchema, IBlueTableSchema> build_schema = (inp_schema) => {
+            IBlueTableSchema rv = new BlueTableSchema("SpreaddStats", new ColumnInfo("id", 0, TypeCode.Int32));
+            rv = rv.add_column(TypeCode.String, "spread");
+            rv = rv.add_column(TypeCode.String, "window");
+            // bids
+            rv = rv.add_column(TypeCode.Double, "bid_mean");
+            rv = rv.add_column(TypeCode.Double, "bid_stddev");
+            rv = rv.add_column(TypeCode.Double, "bid_skewness");
+            rv = rv.add_column(TypeCode.Double, "bid_kurtosis");
+            rv = rv.add_column(TypeCode.Double, "bid_max");
+            rv = rv.add_column(TypeCode.Double, "bid_min");
+            // asks
+            rv = rv.add_column(TypeCode.Double, "ask_mean");
+            rv = rv.add_column(TypeCode.Double, "ask_stddev");
+            rv = rv.add_column(TypeCode.Double, "ask_skewness");
+            rv = rv.add_column(TypeCode.Double, "ask_kurtosis");
+            rv = rv.add_column(TypeCode.Double, "ask_max");
+            rv = rv.add_column(TypeCode.Double, "ask_min");
+            // tail distribution
+            rv = rv.add_column(TypeCode.Double, "bid Z ≥ 2σ");
+            rv = rv.add_column(TypeCode.Double, "ask Z < -2σ");
+            // trading ticks
+            rv = rv.add_column(TypeCode.Int32, "n_buy_ticks");
+            rv = rv.add_column(TypeCode.Double, "cumm_buy_price");
+            rv = rv.add_column(TypeCode.Int32, "n_sell_ticks");
+            rv = rv.add_column(TypeCode.Double, "cumm_sell_price");
+            rv = rv.add_column(TypeCode.Int32, "n_trade_ticks");
+            return rv;
+         };
+         Func<IBlueRow, IBlueRow> build_row = (r) => {
+            IBlueTableSchema s = build_schema(r.schema);
+            return new BlueRow(s, r["id"], r["spread"], r["window"],
+                           r["bid_mean"], r["bid_stddev"], r["bid_skewness"], r["bid_kurtosis"], r["bid_max"], r["bid_min"],
+                           r["ask_mean"], r["ask_stddev"], r["ask_skewness"], r["ask_kurtosis"], r["ask_max"], r["ask_min"],
+                           r["bid Z ≥ 2σ"], r["ask Z < -2σ"],
+                           r["n_buy_ticks"], r["cumm_buy_price"], r["n_sell_ticks"], r["cumm_sell_price"], r["n_trade_ticks"]);
+         };
+         MapStream rs = new MapStream(parent_str, inp_schema => build_schema(inp_schema),
+            inp_r => build_row(inp_r));
+         return rs;
+      }
+
+      private void _spatial_pairs_stream_onchange(object dp_obj) {
+         if (_tot_str != null)
+            return;
+         MutableAppendStream append_str = new MutableAppendStream();
+         _tot_str = _build_mapped_stream(append_str);
+         //_tot_str = new ThrottleStream(_tot_str, 1000);
+         //_tot_str = new SortStream(_tot_str, "-n_trade_ticks");
+         _tot_str.onChange += _handle_tot_stream;
+
+         DataPacket dp = dp_obj as DataPacket;         
+         foreach (IBlueRow r in dp.newRows) {
+            string s1 = r.get<string>("s1");
+            string s2 = r.get<string>("s2");
+            string key = SpatialFuturePairsStream.get_key(s1, s2);
+            if (!_key_map.Contains(key)) {
+               _key_map.Add(key);
+               ViewStream spread_stats_stream = new SpreadStatsStream(s1, s2, 1, 1, Convert.ToInt32(_window_sz.TotalSeconds), 
+                                                                     Convert.ToInt32(_sampling_period_sz.TotalSeconds), _bbo_stream, _vsd);
+               append_str.addStream(spread_stats_stream);
+            }
+         }
+      }
+
+      private void _handle_tot_stream(object dp_obj) {
+         myFireEvent(dp_obj);
+      }
+   }
+
+}

src/DataStreamSuite/SharedStreams/Business/MarketData/Analytics/SpatialFuturePairsStream.cs

@@ -0,0 +1,89 @@
+namespace bts.SilverPine {
+   using BCG;
+
+   using bts.api;
+
+   using System;
+   using System.Collections.Generic;
+   using System.Linq;
+   using System.Text;
+   using System.Threading.Tasks;
+
+   [DataStreamUserRespresentation("SpatialFuturePairsStream")]
+   public class SpatialFuturePairsStream : ViewStream {
+      private ViewStream _main_stream;
+      private IBlueTableSchema _schema;
+      private HashSet<string> _key_map = new HashSet<string>();
+      public SpatialFuturePairsStream()
+         : this(ViewStreamDatabase.instance) { }
+
+      public SpatialFuturePairsStream(ViewStreamDatabase vsd) {
+         _main_stream = new InstrumentsStream(vsd);
+         _main_stream = new FilterStream(_main_stream, "sec_type='Future'");
+         _schema = new BlueTableSchema("SpatialFutures", new ColumnInfo("id", 0, TypeCode.Int32));
+         _schema = _schema.add_column(TypeCode.String, "s1");
+         _schema = _schema.add_column(TypeCode.String, "venue_1");
+         _schema = _schema.add_column(TypeCode.String, "s2");
+         _schema = _schema.add_column(TypeCode.String, "venue_2");
+
+      }
+
+      protected override void myOnFirstSubscriber() {
+         _main_stream.onChange += _main_stream_onchange;
+      }
+
+      protected override void myOnLastSubscriber() {
+         _main_stream.onChange -= _main_stream_onchange;
+      }
+
+      public static string get_key(string s1, string s2) {
+         string key = string.Compare(s1, s2, StringComparison.Ordinal) <= 0 ? s1 + s2 : s2 + s1;
+         return key;
+      }
+
+      public static string get_key(IBlueRow r1, IBlueRow r2) {
+         string s1 = r1.get<string>("sec");
+         string s2 = r2.get<string>("sec");
+         return get_key(s1, s2);
+      }
+
+      private bool _combine(IBlueRow r1, IBlueRow r2) {
+         string key = get_key(r1, r2);
+         if (_key_map.Contains(key))
+            return false;
+         if (r1.get<string>("symbol") != r2.get<string>("symbol"))
+            return false;
+         if (r1.get_enum<Venue>("venue") == r2.get_enum<Venue>("venue"))
+            return false;
+         if (r1.get<int>("sec_key") == r2.get<int>("sec_key")) {
+            return false;
+         }
+         DateTime e1 = r1.get_datetime("expiration");
+         DateTime e2 = r2.get_datetime("expiration");
+         bool exps_eq = e1 == e2 || (e1.Year >= 2100 && e2.Year >= 2100);
+         if (!exps_eq)
+            return false;         
+         return true;
+      }
+
+      private void _main_stream_onchange(object dp_obj) {
+         _key_map.Clear();
+         DataPacket dp = dp_obj as DataPacket;
+         BlueTable table = new BlueTable(_schema);
+         var rows = dp.newRows.to_list();
+         foreach (var r1 in rows) {
+            foreach (var r2 in rows) {
+               if (_combine(r1, r2)) {
+                  IBlueRow nr = new BlueRow(_schema, table.Count, r1.get<string>("sec"), r1.get<string>("venue"),
+                     r2.get<string>("sec"), r2.get<string>("venue"));
+                  table.addRow(nr);
+                  string key = get_key(r1, r2);
+                  _key_map.Add(key);
+               }
+            }
+         }
+         var dp_out = BlueDataUtility.build_dp(table, myLastValue as DataPacket);
+         myFireEvent(dp_out);
+      }
+   }
+}

src/DataStreamSuite/SharedStreams/Business/MarketData/Analytics/SpreadPerformanceEvaluator.cs

@@ -0,0 +1,656 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using MathNet.Numerics.Statistics;
+using MathNet.Numerics.LinearRegression;
+using System.Collections.Generic;
+using System.Linq;
+using MathNet.Numerics.Statistics;
+using MathNet.Numerics.LinearAlgebra;
+using Amazon.Runtime;
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using MathNet.Numerics.Statistics;
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using MathNet.Numerics.Statistics;
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using MathNet.Numerics.Statistics;
+// stats calculator
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using MathNet.Numerics.Statistics;
+
+/// <summary>
+/// Provides evaluation metrics to assess the quality of a synthetic spread for mean-reversion trading.
+/// Assumes the input spread data is a trailing, time-causal window of recent ticks (e.g., last 120 seconds).
+/// </summary>
+public static class SpreadPerformanceEvaluator
+{
+
+   public static class SpreadStatistics {
+      public class SpreadStatsResult {
+         public Dictionary<string, double> bid_zscore_distribution { get; set; }
+         public Dictionary<string, double> ask_zscore_distribution { get; set; }
+         public Dictionary<string, double> mid_zscore_distribution { get; set; }
+
+         public DescriptiveStatistics bid_stats { get; set; }
+         public DescriptiveStatistics ask_stats { get; set; }
+         public DescriptiveStatistics mid_stats { get; set; }
+      }
+
+      private static readonly string[] _zscore_labels = new[]
+      {
+        "Z < -2σ",
+        "-2σ ≤ Z < -1σ",
+        "-1σ ≤ Z < 0σ",
+        "0σ ≤ Z < 1σ",
+        "1σ ≤ Z < 2σ",
+        "Z ≥ 2σ"
+    };
+
+      public static SpreadStatsResult analyze_spread_quotes(List<(DateTime timestamp, double bid, double bid_qty, double ask, double ask_qty)> quotes) {
+         if (quotes == null || quotes.Count == 0)
+            throw new ArgumentException("Spread quote list cannot be null or empty.");
+
+         var bid_prices = quotes.Select(q => q.bid).ToList();
+         var ask_prices = quotes.Select(q => q.ask).ToList();
+         var mid_prices = quotes.Select(q => (q.bid + q.ask) / 2.0).ToList();
+
+         var bid_stats = new DescriptiveStatistics(bid_prices);
+         var ask_stats = new DescriptiveStatistics(ask_prices);
+         var mid_stats = new DescriptiveStatistics(mid_prices);
+
+         var bid_zscores = bid_prices.Select(x => zscore(x, bid_stats.Mean, bid_stats.StandardDeviation)).ToList();
+         var ask_zscores = ask_prices.Select(x => zscore(x, ask_stats.Mean, ask_stats.StandardDeviation)).ToList();
+         var mid_zscores = mid_prices.Select(x => zscore(x, mid_stats.Mean, mid_stats.StandardDeviation)).ToList();
+
+         var bid_distribution = bin_zscores_to_percentages(bid_zscores);
+         var ask_distribution = bin_zscores_to_percentages(ask_zscores);
+         var mid_distribution = bin_zscores_to_percentages(mid_zscores);
+
+         return new SpreadStatsResult {
+            bid_zscore_distribution = bid_distribution,
+            ask_zscore_distribution = ask_distribution,
+            mid_zscore_distribution = mid_distribution,
+            bid_stats = bid_stats,
+            ask_stats = ask_stats,
+            mid_stats = mid_stats
+         };
+      }
+
+      private static double zscore(double x, double mean, double std_dev) {
+         return std_dev == 0 ? 0 : (x - mean) / std_dev;
+      }
+
+      private static Dictionary<string, double> bin_zscores_to_percentages(List<double> zscores) {
+         var bins = _zscore_labels.ToDictionary(label => label, _ => 0);
+
+         foreach (var z in zscores) {
+            if (z < -2) bins["Z < -2σ"]++;
+            else if (z < -1) bins["-2σ ≤ Z < -1σ"]++;
+            else if (z < 0) bins["-1σ ≤ Z < 0σ"]++;
+            else if (z < 1) bins["0σ ≤ Z < 1σ"]++;
+            else if (z < 2) bins["1σ ≤ Z < 2σ"]++;
+            else bins["Z ≥ 2σ"]++;
+         }
+
+         double total = zscores.Count;
+         return bins.ToDictionary(kvp => kvp.Key, kvp => total == 0 ? 0 : (kvp.Value / total) * 100.0);
+      }
+
+      public static double score_spread(SpreadStatsResult result) {
+         var ask_bins = result.ask_zscore_distribution;
+         var bid_bins = result.bid_zscore_distribution;
+         var mid_bins = result.mid_zscore_distribution;
+
+         double buy_opportunity = ask_bins["Z < -2σ"] + ask_bins["-2σ ≤ Z < -1σ"];
+         double sell_opportunity = bid_bins["Z ≥ 2σ"] + bid_bins["1σ ≤ Z < 2σ"];
+         double opportunity_score = buy_opportunity + sell_opportunity;
+
+         double below_mean = mid_bins["Z < -2σ"] + mid_bins["-2σ ≤ Z < -1σ"] + mid_bins["-1σ ≤ Z < 0σ"];
+         double above_mean = mid_bins["Z ≥ 2σ"] + mid_bins["1σ ≤ Z < 2σ"] + mid_bins["0σ ≤ Z < 1σ"];
+         double symmetry_score = 1.0 - Math.Abs(below_mean - above_mean) / (below_mean + above_mean + 1e-6);
+
+         double amplitude_score = result.mid_stats.StandardDeviation;
+         double skew_penalty = 1.0 / (1.0 + Math.Abs(result.mid_stats.Skewness));
+         double kurtosis_bonus = Math.Min(result.mid_stats.Kurtosis, 6.0) / 6.0;
+
+         double score =
+             opportunity_score *
+             symmetry_score *
+             amplitude_score *
+             skew_penalty *
+             (1.0 + kurtosis_bonus * 0.1);
+
+         return score;
+      }
+   }
+
+
+
+
+
+/// <summary>
+/// Represents a single trade action (entry or exit) taken during the simulation.
+/// </summary>
+public struct TradeAction
+    {
+        public bool IsEntry { get; set; }    // True if this action opened a position
+        public bool IsLong { get; set; }     // True = Long, False = Short
+        public double Price { get; set; }    // Executed price
+        public int Index { get; set; }       // Tick index of the action
+
+        public override string ToString()
+        {
+            string side = IsLong ? "Long" : "Short";
+            string type = IsEntry ? "Open" : "Close";
+            return $"{type} {side} @ {Price:F4} (Tick {Index})";
+        }
+    }
+
+    /// <summary>
+/// Simulates a mean-reversion strategy using rolling, causal mid-price stats.
+/// ✅ Entry:
+///   - Buy at ask when ask < rollingMean - entryThreshold × rollingStdDev
+///   - Sell at bid when bid > rollingMean + entryThreshold × rollingStdDev
+/// ✅ Exit:
+///   - Close when mid returns within rollingMean ± exitThreshold × rollingStdDev
+///   - Long exits at bid, short exits at ask
+/// This version uses only past and current data at each step (no forward-looking bias).
+/// </summary>
+public static (double TotalPnL, List<TradeAction> TradeActions, int BuyTurns, int SellTurns) EvaluateSimulatedPnL(
+    List<Tuple<double, double, double, double>> spreadData,
+    int statWindow = 30,
+    double entryThresholdMultiplier = 0.5,
+    double exitThresholdMultiplier = 0.1)
+{
+    var trades = new List<TradeAction>();
+    var mids = spreadData.Select(x => (x.Item1 + x.Item3) / 2.0).ToList();
+
+    if (spreadData.Count < statWindow)
+        return (0.0, trades, 0, 0);
+
+    bool positionOpen = false;
+    bool isLong = false;
+    double entryPrice = 0.0;
+    double pnl = 0.0;
+    int buyTurns = 0;
+    int sellTurns = 0;
+
+    for (int i = statWindow; i < spreadData.Count; i++)
+    {
+        double bid = spreadData[i].Item1;
+        double ask = spreadData[i].Item3;
+        double mid = mids[i];
+
+        // Rolling window
+        var window = mids.Skip(i - statWindow).Take(statWindow).ToList();
+        double mean = window.Average();
+        double stddev = Math.Sqrt(window.Select(x => Math.Pow(x - mean, 2)).Average());
+
+        double lowerThreshold = mean - entryThresholdMultiplier * stddev;
+        double upperThreshold = mean + entryThresholdMultiplier * stddev;
+        double exitBand = exitThresholdMultiplier * stddev;
+
+        if (!positionOpen)
+        {
+            if (ask < lowerThreshold)
+            {
+                positionOpen = true;
+                isLong = true;
+                entryPrice = ask;
+                trades.Add(new TradeAction { IsEntry = true, IsLong = true, Price = entryPrice, Index = i });
+            }
+            else if (bid > upperThreshold)
+            {
+                positionOpen = true;
+                isLong = false;
+                entryPrice = bid;
+                trades.Add(new TradeAction { IsEntry = true, IsLong = false, Price = entryPrice, Index = i });
+            }
+        }
+        else
+        {
+            if (Math.Abs(mid - mean) < exitBand)
+            {
+                double exitPrice = isLong ? bid : ask;
+                pnl += isLong ? (exitPrice - entryPrice) : (entryPrice - exitPrice);
+
+                trades.Add(new TradeAction { IsEntry = false, IsLong = isLong, Price = exitPrice, Index = i });
+
+                if (isLong) buyTurns++;
+                else sellTurns++;
+
+                positionOpen = false;
+            }
+        }
+    }
+
+    return (pnl, trades, buyTurns, sellTurns);
+}
+
+
+
+    /// <summary>
+    /// Estimates the Hurst exponent of the spread mid-prices.
+    /// Assumes the input is a trailing window of recent data.
+    /// Hurst < 0.5 implies mean-reversion; ideal is 0.35–0.45.
+    /// </summary>
+    public static double EvaluateHurstExponent(List<Tuple<double, double, double, double>> spreadData)
+    {
+        const int MinLength = 21;
+
+        if (spreadData == null || spreadData.Count < MinLength)
+            return double.NaN;
+
+        var spreadMids = spreadData.Select(x => (x.Item1 + x.Item3) / 2.0).ToArray();
+        int n = spreadMids.Length;
+
+        var lags = new List<double>();
+        var tau = new List<double>();
+
+        for (int lag = 2; lag <= 20; lag++)
+        {
+            if (n - lag <= 1)
+                continue;
+
+            double[] diffs = new double[n - lag];
+            for (int i = 0; i < diffs.Length; i++)
+                diffs[i] = spreadMids[i + lag] - spreadMids[i];
+
+            double stddev = diffs.StandardDeviation();
+
+            if (stddev > 0)
+            {
+                lags.Add(Math.Log(lag));
+                tau.Add(Math.Log(stddev));
+            }
+        }
+
+        if (lags.Count < 2)
+            return double.NaN;
+
+        var (intercept, slope) = SimpleRegression.Fit(lags.ToArray(), tau.ToArray());
+        double hurst = slope * 0.5 + 0.5;
+        return hurst;
+    }
+
+    /// <summary>
+/// Computes the mean and standard deviation of the spread mid-prices
+/// over a trailing, causal window.
+/// </summary>
+/// <param name="spreadData">List of (bid, bidQty, ask, askQty)</param>
+/// <returns>Tuple of (Mean, StdDev). Returns (NaN, NaN) if input is too short.</returns>
+public static (double Mean, double StdDev) EvaluateMeanAndStdDev(List<Tuple<double, double, double, double>> spreadData)
+{
+    if (spreadData == null || spreadData.Count < 2)
+        return (double.NaN, double.NaN);
+
+    var mids = spreadData.Select(x => (x.Item1 + x.Item3) / 2.0).ToList();
+    double mean = mids.Average();
+    double stddev = Math.Sqrt(mids.Select(x => Math.Pow(x - mean, 2)).Average());
+
+    return (mean, stddev);
+}
+
+
+   /// <summary>
+   /// Counts how many times the spread ask dropped below (mean - threshold) [buy signal]
+   /// and how many times the spread bid rose above (mean + threshold) [sell signal].
+   /// Assumes spreadData is a trailing, causal window.
+   /// </summary>
+   /// <returns>(BuyExcursions, SellExcursions)</returns>
+   public static (uint BuyExcursions, double cumm_buy_price, uint SellExcursions, double cumm_sell_price) EvaluateExcursionCount(
+       List<Tuple<double, double, double, double>> spreadData,
+       double entryThresholdMultiplier = 0.5) {
+      if (spreadData == null || spreadData.Count < 2)
+         return (0, Double.NaN, 0, Double.NaN);
+
+      var midPrices = spreadData.Select(x => (x.Item1 + x.Item3) / 2.0).ToList();
+      double mean = midPrices.Average();
+      double stddev = midPrices.StandardDeviation();
+      double buy_threshold = mean - entryThresholdMultiplier * stddev;
+      double sell_threshold = mean + entryThresholdMultiplier * stddev;
+
+      int buyCount = 0;
+      int sellCount = 0;
+      //bool in_buy_zone = false;
+      //bool in_sell_zone = false;
+
+      double cumm_buy_price = 0.0, cumm_sell_price = 0.0;
+      foreach (var (bid, _, ask, _) in spreadData) {
+         if (ask <= buy_threshold) {
+            //if (!in_buy_zone) {
+            cumm_buy_price += ask;
+            buyCount++;
+            //in_buy_zone = true;
+         }
+         //} else {
+         //   in_buy_zone = false;
+         //}
+
+         if (bid >= sell_threshold) {
+            //if (!in_sell_zone) {
+            cumm_sell_price += bid;
+            sellCount++;
+            //in_sell_zone = true;
+         }
+         //} else {
+         //   in_sell_zone = false;
+         //}
+      }
+
+      return ((uint)buyCount, cumm_buy_price / buyCount, (uint)sellCount, cumm_sell_price / sellCount);
+   }
+
+/// <summary>
+/// Counts how many times the spread mid-price crosses its mean level.
+/// Each crossing indicates a potential oscillation turn (cycle midpoint).
+/// Assumes the input is a trailing, causal window of spread ticks.
+/// </summary>
+/// <param name="spreadData">List of (bid, bidQty, ask, askQty)</param>
+/// <returns>Number of zero crossings in the mid-price series</returns>
+public static UInt32 EvaluateZeroCrossingCount(List<Tuple<double, double, double, double>> spreadData)
+{
+    if (spreadData == null || spreadData.Count < 2)
+        return 0;
+
+    var mids = spreadData.Select(x => (x.Item1 + x.Item3) / 2.0).ToList();
+    double mean = mids.Average();
+    int zeroCrossings = 0;
+
+    for (int i = 1; i < mids.Count; i++)
+    {
+        double prevOffset = mids[i - 1] - mean;
+        double currOffset = mids[i] - mean;
+
+        if (prevOffset * currOffset < 0) // Sign changed
+            zeroCrossings++;
+    }
+
+    return (UInt32)zeroCrossings;
+}
+
+   /// <summary>
+   /// Computes the half-life of mean reversion.
+   /// Lower half-life means the spread reverts to its mean more quickly.
+   /// </summary>
+   public static double ComputeHalfLife(List<Tuple<double, double, double, double>> spreadData) {
+      var prices = spreadData.Select(x => (x.Item1 + x.Item3) / 2.0).ToList();
+      var lagged = prices.Take(prices.Count - 1).ToArray();
+      var delta = prices.Skip(1).Zip(lagged, (cur, prev) => cur - prev).ToArray();
+
+      if (lagged.Length < 2) return double.NaN;
+
+      var x = Vector<double>.Build.Dense(lagged);
+      var y = Vector<double>.Build.Dense(delta);
+      var beta = x.DotProduct(y) / x.DotProduct(x);
+
+      return -Math.Log(2) / beta;
+   }
+
+   /// <summary>
+   /// Computes the lag-1 autocorrelation of the spread.
+   /// Lower values indicate quicker mean-reversion.
+   /// </summary>
+   public static double ComputeLag1Autocorrelation(List<Tuple<double, double, double, double>> spreadData) {
+      var prices = spreadData.Select(x => (x.Item1 + x.Item3) / 2.0).ToList();
+      var mean = prices.Average();
+      var numerator = 0.0;
+      var denominator = 0.0;
+
+      for (int i = 1; i < prices.Count; i++) {
+         numerator += (prices[i] - mean) * (prices[i - 1] - mean);
+         denominator += Math.Pow(prices[i - 1] - mean, 2);
+      }
+
+      return denominator == 0 ? 0 : numerator / denominator;
+   }
+
+   /// <summary>
+   /// Estimates how quickly extreme Z-score values decay back toward the mean.
+   /// Shorter average decay time indicates stronger mean-reversion.
+   /// </summary>
+   public static double ComputeAverageZScoreDecay(List<Tuple<double, double, double, double>> spreadData, int zScoreWindow = 50) {
+      var prices = spreadData.Select(x => (x.Item1 + x.Item3) / 2.0).ToList();
+      if (prices.Count < zScoreWindow + 2) return double.NaN;
+
+      var decayTimes = new List<int>();
+      for (int i = zScoreWindow; i < prices.Count - 1; i++) {
+         var window = prices.Skip(i - zScoreWindow).Take(zScoreWindow).ToList();
+         double mean = window.Average();
+         double stdDev = Statistics.StandardDeviation(window);
+         if (stdDev == 0) continue;
+
+         double z = (prices[i] - mean) / stdDev;
+         if (Math.Abs(z) > 2) {
+            for (int j = 1; j < prices.Count - i; j++) {
+               double zNext = (prices[i + j] - mean) / stdDev;
+               if (Math.Abs(zNext) < 1) {
+                  decayTimes.Add(j);
+                  break;
+               }
+            }
+         }
+      }
+
+      return decayTimes.Count == 0 ? double.NaN : decayTimes.Average();
+   }
+
+   /// <summary>
+   /// Computes the variance ratio: short-term variance divided by long-term variance.
+   /// Lower ratio suggests stronger mean-reversion.
+   /// </summary>
+   public static double ComputeVarianceRatio(List<Tuple<double, double, double, double>> spreadData, int shortWindow, int longWindow) {
+      var prices = spreadData.Select(x => (x.Item1 + x.Item3) / 2.0).ToList();
+      if (prices.Count < longWindow) return double.NaN;
+
+      double shortVar = Statistics.Variance(prices.Skip(prices.Count - shortWindow).Take(shortWindow));
+      double longVar = Statistics.Variance(prices.Skip(prices.Count - longWindow).Take(longWindow));
+
+      return longVar == 0 ? double.NaN : shortVar / longVar;
+   }
+
+   /// <summary>
+   /// Computes the highest bid and the lowest ask from a spread tick series.
+   /// Useful for evaluating the best market opportunity range.
+   /// </summary>
+   /// <param name="spreadData">List of (bid, bidQty, ask, askQty)</param>
+   /// <returns>Tuple of (MaxBid, MinAsk)</returns>
+   public static (double MaxBid, double MinAsk) EvaluateMaxBidMinAsk(List<Tuple<double, double, double, double>> spreadData)
+{
+    if (spreadData == null || spreadData.Count == 0)
+        return (double.NaN, double.NaN);
+
+    double maxBid = double.NegativeInfinity;
+    double minAsk = double.PositiveInfinity;
+
+    foreach (var (bid, _, ask, _) in spreadData)
+    {
+        if (bid > maxBid) maxBid = bid;
+        if (ask < minAsk) minAsk = ask;
+    }
+
+    return (maxBid, minAsk);
+}
+
+/// <summary>
+/// Computes the distribution of spread mid-prices around the mean,
+/// grouped by caller-defined distance bands (in standard deviations).
+///
+/// This helps answer:
+/// - Is the spread tightly hugging the mean?
+/// - Are there wide excursions or heavy tails?
+///
+/// You define breakpoints like: [0.5, 1.0, 2.0] → it will return:
+/// - "≤ 0.5σ"
+/// - "0.5σ – 1.0σ"
+/// - "1.0σ – 2.0σ"
+/// - "> 2.0σ"
+///
+/// How to Interpret It:
+/// - ≤ 0.5σ        → Tight clustering near the mean (strong mean-reversion)
+/// - 0.5σ – 1.0σ   → Mild oscillation (common trade region)
+/// - 1.0σ – 2.0σ   → Moderate deviation (potential trade signals)
+/// - > 2.0σ        → Rare, extreme moves (tail events, potential entries/exits)
+///
+/// Use this to tune entry thresholds or detect regime shifts.
+/// </summary>
+/// <param name="spreadData">List of (bid, bidQty, ask, askQty)</param>
+/// <param name="regions">List of positive breakpoints in std deviations (e.g., [0.5, 1.0, 2.0])</param>
+/// <returns>Dictionary mapping label to % of mid-prices in that region</returns>
+public static Dictionary<string, double> EvaluateDistributionAroundMean(
+    List<Tuple<double, double, double, double>> spreadData,
+    List<double> regions)
+{
+    var result = new Dictionary<string, double>();
+
+    if (spreadData == null || spreadData.Count < 2 || regions == null || regions.Count == 0)
+        return result;
+
+    var mids = spreadData.Select(x => (x.Item1 + x.Item3) / 2.0).ToList();
+    double mean = mids.Average();
+    double stddev = Math.Sqrt(mids.Select(x => Math.Pow(x - mean, 2)).Average());
+
+    if (stddev == 0)
+    {
+        result["Zero Volatility"] = 100.0;
+        return result;
+    }
+
+    var buckets = new int[regions.Count + 1];
+    foreach (var mid in mids)
+    {
+        double distance = Math.Abs(mid - mean);
+        bool placed = false;
+
+        for (int i = 0; i < regions.Count; i++)
+        {
+            if (distance <= regions[i])
+            {
+                buckets[i]++;
+                placed = true;
+                break;
+            }
+        }
+
+        if (!placed)
+            buckets[regions.Count]++;
+    }
+
+    int total = mids.Count;
+    for (int i = 0; i < regions.Count; i++)
+    {
+        string label = i == 0
+            ? $"≤ {regions[i]:0.##}σ"
+            : $"{regions[i - 1]:0.##}σ – {regions[i]:0.##}σ";
+        result[label] = 100.0 * buckets[i] / total;
+    }
+
+    result[$"> {regions.Last():0.##}σ"] = 100.0 * buckets.Last() / total;
+
+    return result;
+}
+
+/// <summary>
+/// Computes two drift metrics indicating how much the spread's mean changes over time,
+/// based on the standard deviation of rolling mean values.
+///
+/// ✅ Absolute Drift: Standard deviation of the rolling mean
+/// ✅ Relative Drift: Drift normalized by the overall mean (unitless ratio)
+///
+/// This shows whether the center of the spread is stable — critical for mean-reversion.
+///
+/// How to Interpret the Output:
+/// - ✅ Relative Drift < 1% → Very stable mean — ideal
+/// - ⚠️ 1–5% → Mild drift — consider shorter holds or adaptive mean
+/// - ❌ >5% → Drifting mean — fixed mean may no longer work
+/// </summary>
+/// <param name="spreadData">List of (bid, bidQty, ask, askQty)</param>
+/// <param name="window">Window size for rolling mean (e.g., 20 ticks)</param>
+/// <returns>
+/// Tuple of (Absolute Drift, Relative Drift) — where:
+/// - Absolute Drift = StdDev of rolling means
+/// - Relative Drift = Drift / global mean
+/// </returns>
+public static (double AbsoluteDrift, double RelativeDrift) EvaluateMeanDriftScore(
+    List<Tuple<double, double, double, double>> spreadData,
+    int window = 20)
+{
+    if (spreadData == null || spreadData.Count < window)
+        return (double.NaN, double.NaN);
+
+    var mids = spreadData.Select(x => (x.Item1 + x.Item3) / 2.0).ToArray();
+    var rollingMeans = new List<double>();
+
+    for (int i = window; i <= mids.Length; i++)
+    {
+        double rollingMean = mids.Skip(i - window).Take(window).Average();
+        rollingMeans.Add(rollingMean);
+    }
+
+
+    double overallMean = mids.Average(); // use full mean for normalization
+    //double drift = Math.Sqrt(rollingMeans.Select(m => Math.Pow(m - rollingMeans.Average(), 2)).Average());
+    double drift = Math.Sqrt(rollingMeans.Select(m => Math.Pow(m - rollingMeans.Average(), 2)).Average());
+
+
+double relativeDrift = Math.Abs(overallMean) > 1e-8 ? drift / Math.Abs(overallMean) : double.NaN;
+    return (drift, relativeDrift);
+}
+
+/// <summary>
+/// Returns the latest Kalman Filter estimate of the true mean of the spread mid-prices.
+/// This tracks the spread's central value while allowing for gradual drift and suppressing noise.
+///
+/// Internally, this applies a recursive 1D Kalman filter over the input mid-prices.
+/// 
+/// Tuning Parameters:
+/// - Q: Process noise variance (higher = faster adaptation to drift)
+/// - R: Measurement noise variance (higher = more smoothing)
+/// 
+/// Use this instead of SMA if your mean drifts or if price series is noisy.
+///
+/// </summary>
+/// <param name="spreadData">List of (bid, bidQty, ask, askQty)</param>
+/// <param name="Q">Process noise variance (default = 0.0001)</param>
+/// <param name="R">Measurement noise variance (default = 0.01)</param>
+/// <returns>Latest estimated Kalman-filtered mean, or NaN if input is invalid</returns>
+public static double GetLatestKalmanMean(
+    List<Tuple<double, double, double, double>> spreadData,
+    double Q = 0.0001,
+    double R = 0.01)
+{
+    if (spreadData == null || spreadData.Count == 0)
+        return double.NaN;
+
+    var mids = spreadData.Select(x => (x.Item1 + x.Item3) / 2.0).ToList();
+
+    double x = mids[0];  // initial estimate of mean
+    double P = 1.0;       // initial uncertainty
+
+    for (int i = 1; i < mids.Count; i++)
+    {
+        // Predict step
+        double x_prior = x;
+        double P_prior = P + Q;
+
+        // Update step
+        double K = P_prior / (P_prior + R);
+        x = x_prior + K * (mids[i] - x_prior);
+        P = (1 - K) * P_prior;
+    }
+
+    return x; // the latest estimated mean
+}
+
+
+
+
+}

src/DataStreamSuite/SharedStreams/Business/MarketData/Analytics/SpreadPerformanceStream.cs

@@ -0,0 +1,137 @@
+namespace bts.SilverPine {
+   using BCG;
+
+   using System.Diagnostics;
+
+   public class SpreadPerformanceStream : ViewStream {
+      ViewStream _spread_price_stream = null;
+      private readonly IBlueTableSchema _schema;
+      private readonly int _window_secs;
+      private readonly double _buy_threshold, _sell_threshold, _flat_threshold;
+      private ViewStream _bbo_stream;
+      public SpreadPerformanceStream(string s1, string s2, double w1, double w2, int window_secs,
+                                       double buy_threshold, double sell_threshold, double flat_threshold)
+      : this(s1, s2, w1, w2, window_secs, buy_threshold, sell_threshold, flat_threshold, ViewStreamDatabase.instance) { }
+
+      public SpreadPerformanceStream(string s1, string s2, double w1, double w2, int window_secs,
+                                      double buy_threshold, double sell_threshold, double flat_threshold,
+                                      ViewStreamDatabase vsd) {
+         _bbo_stream = new BBOStream(vsd);
+         _schema = BlueDataUtility.form_schema("SpreadPerformance", Tuple.Create("id", TypeCode.Int32),                                                
+                                                Tuple.Create("t0", TypeCode.DateTime),
+                                                Tuple.Create("t1", TypeCode.DateTime),
+                                                Tuple.Create("n_ticks", TypeCode.UInt32),
+                                                Tuple.Create("curr_windows_seconds", TypeCode.Double),
+                                                // means / std dev                                          
+                                                Tuple.Create("stddev", TypeCode.Double),
+                                                Tuple.Create("mean", TypeCode.Double),
+                                                Tuple.Create("kalman_filter_mean", TypeCode.Double),
+                                                Tuple.Create("absolute_mean_drift", TypeCode.Double),
+                                                Tuple.Create("relative_mean_drift", TypeCode.Double),
+                                                // statistical measures
+                                                Tuple.Create("hurst_exponent", TypeCode.Double),
+                                                Tuple.Create("half_life", TypeCode.Double),
+                                                Tuple.Create("lag1_autocorrelation", TypeCode.Double),
+                                                Tuple.Create("average_zscore_decay", TypeCode.Double),
+                                                Tuple.Create("variance_ratio", TypeCode.Double),
+                                                Tuple.Create("n_zero_crossings", TypeCode.UInt32),                                                
+                                                // distributions
+                                                Tuple.Create("Z ≤ 0.5σ", TypeCode.Double),
+                                                Tuple.Create("0.5σ – 1.0σ", TypeCode.Double),
+                                                Tuple.Create("1.0σ – 2.0σ", TypeCode.Double),
+                                                Tuple.Create("> 2.0σ", TypeCode.Double),
+                                                ////////////////////////////////////////                                                   
+                                                Tuple.Create("max_bid", TypeCode.Double),
+                                                Tuple.Create("min_ask", TypeCode.Double),
+                                                Tuple.Create("n_buy_signals", TypeCode.UInt32),
+                                                Tuple.Create("cumm_buy_price", TypeCode.Double),
+                                                Tuple.Create("n_sell_signals", TypeCode.UInt32),
+                                                Tuple.Create("cumm_sell_price", TypeCode.Double)
+                                                //Tuple.Create("pnl", TypeCode.Double),
+                                                //Tuple.Create("buy_turns", TypeCode.UInt32),
+                                                //Tuple.Create("sell_turns", TypeCode.UInt32),
+                                                //Tuple.Create("trades", TypeCode.String)
+                                                );
+         _window_secs = window_secs;
+         _buy_threshold = buy_threshold;
+         _sell_threshold = sell_threshold;
+         _flat_threshold = flat_threshold;
+         _spread_price_stream = new SyntheticSpreadStream(s1, s2, w1, w2, _bbo_stream, vsd);
+         _spread_price_stream = new TimeSeriesStream(_spread_price_stream);
+         _spread_price_stream = new ThrottleStream(_spread_price_stream, TimeSpan.FromSeconds(5.0), vsd);
+      }
+
+      protected override void myOnFirstSubscriber() {
+         _spread_price_stream.onChange += _handle_spread_prices;
+      }
+
+      protected override void myOnLastSubscriber() {
+         _spread_price_stream.onChange -= _handle_spread_prices;
+      }
+
+      private void _handle_spread_prices(object dp_obj) {
+         DataPacket dp = dp_obj as DataPacket;
+         List<Tuple<DateTime, double, double, double, double>> spread_ts_prices = new List<Tuple<DateTime, double, double, double, double>>();
+         DateTime min_time = DateTime.MaxValue; DateTime max_time = DateTime.MinValue;
+         var sw = Stopwatch.StartNew();
+         sw.Start();
+         foreach (IBlueRow r in dp.newRows) {
+            DateTime ts = r.get_datetime("timestamp");
+            TimeSpan elapsed_time = DateTime.Now - ts;
+            if (elapsed_time.TotalSeconds >= _window_secs)
+               continue;
+            spread_ts_prices.Add(Tuple.Create(
+                                          r.get_datetime("timestamp"),
+                                          r.get<double>("spread_bid_price"),
+                                          r.get<double>("spread_bid_sz"),
+                                          r.get<double>("spread_ask_price"),
+                                          r.get<double>("spread_ask_sz")));
+            if (ts <= min_time) {
+               min_time = ts;
+            }
+            if (ts >= max_time) {
+               max_time = ts;
+            }
+         }
+         List<Tuple<double, double, double, double>> spread_prices = spread_ts_prices
+                                                                                             .OrderBy(t => t.Item1)
+                                                                                             .Select(t => Tuple.Create(t.Item2, t.Item3, t.Item4, t.Item5))
+                                                                                             .ToList();
+
+         uint n_ticks = (uint)spread_prices.Count;
+         double curr_windows_seconds = (max_time - min_time).TotalSeconds;
+         double hurst_exponent = SpreadPerformanceEvaluator.EvaluateHurstExponent(spread_prices);
+         double half_life = SpreadPerformanceEvaluator.ComputeHalfLife (spread_prices);
+         double lag1_autocorrelation = SpreadPerformanceEvaluator.ComputeLag1Autocorrelation(spread_prices);
+         double average_zscore_decay = SpreadPerformanceEvaluator.ComputeAverageZScoreDecay(spread_prices);
+         double variance_ratio = SpreadPerformanceEvaluator.ComputeVarianceRatio(spread_prices, 60, 360);
+
+         (double mean, double stddev) = SpreadPerformanceEvaluator.EvaluateMeanAndStdDev(spread_prices);
+         double kalman_filter_mean = SpreadPerformanceEvaluator.GetLatestKalmanMean(spread_prices);
+         uint n_zero_crossings = SpreadPerformanceEvaluator.EvaluateZeroCrossingCount(spread_prices);
+         (double max_bid, double min_ask) = SpreadPerformanceEvaluator.EvaluateMaxBidMinAsk(spread_prices);
+         (uint n_buy_signals, double cumm_buy_price, uint n_sell_signals, double cumm_sell_price) = SpreadPerformanceEvaluator.EvaluateExcursionCount(spread_prices, _buy_threshold);
+         //(double pnl, List<SpreadPerformanceEvaluator.TradeAction> trades, int buy_turns, int sell_turns) = SpreadPerformanceEvaluator.EvaluateSimulatedPnL(spread_prices, 30, _buy_threshold, _flat_threshold);
+         //string trade_str = String.Empty;
+         //foreach (var tr in trades) {
+         //   trade_str += tr.ToString() + " , ";
+         //}
+         Dictionary<string, double> dist_map = SpreadPerformanceEvaluator.EvaluateDistributionAroundMean(spread_prices,
+                                                                       new List<double>(new double[] { 0.5, 1.0, 2.0 }));
+         var dist_vals = dist_map.Count < 4 ? new double[] { Double.NaN, Double.NaN, Double.NaN, Double.NaN } : dist_map.Values.ToArray();
+         (double absolute_drift, double relative_drift) = SpreadPerformanceEvaluator.EvaluateMeanDriftScore(spread_prices);
+         IBlueRow nr = new BlueRow(_schema, 0, min_time, max_time, n_ticks, curr_windows_seconds, stddev, mean, kalman_filter_mean, absolute_drift, relative_drift,
+            hurst_exponent, half_life, lag1_autocorrelation, average_zscore_decay, variance_ratio, n_zero_crossings,
+                                    dist_vals[0], dist_vals[1], dist_vals[2], dist_vals[3],
+                                    max_bid, min_ask, n_buy_signals, cumm_buy_price, n_sell_signals, cumm_sell_price);
+         BlueTable t = new BlueTable(_schema);
+         t.addRow(nr);
+         var dp_out = BlueDataUtility.build_dp_inc(t, myLastValue as DataPacket, true);
+         var perf_msecs = sw.ElapsedMilliseconds;
+         sw.Stop();
+         Log.Gen.info($"SpreadPerformanceStream._handle_spread_prices comp_time: {perf_msecs} msecs");
+         myFireEvent(dp_out);
+
+      }
+   }
+}

src/DataStreamSuite/SharedStreams/Business/MarketData/Analytics/SpreadStatArbsWeightLinearRegressionStream.cs

@@ -0,0 +1,68 @@
+namespace bts.SilverPine{
+   using BCG;
+   public class SpreadStatArbsWeightLinearRegressionStream : ViewStream {
+      private IBlueTableSchema _schema;
+      private ViewStream _leg_price_stream;
+      private readonly double _window_secs;
+      private readonly string _s1, _s2;
+
+      public SpreadStatArbsWeightLinearRegressionStream(string s1, string s2, double window_secs, double sample_period_secs)
+         : this(s1, s2, window_secs, sample_period_secs, ViewStreamDatabase.instance) { }
+
+
+      public SpreadStatArbsWeightLinearRegressionStream(string s1, string s2, double window_secs, double sample_period_secs, ViewStreamDatabase vsd) {
+         _window_secs = window_secs;
+         _s1 = s1;
+         _s2 = s2;
+         _schema = BlueDataUtility.form_schema("LinearRegressionWeights",
+                                                Tuple.Create("id", TypeCode.Int32),
+                                                Tuple.Create("t0", TypeCode.DateTime),
+                                                Tuple.Create("t1", TypeCode.DateTime),
+                                                Tuple.Create("curr_windows_seconds", TypeCode.Double),
+                                                Tuple.Create("n_ticks", TypeCode.UInt32),
+                                                Tuple.Create("s1", TypeCode.String),
+                                                Tuple.Create("w1", TypeCode.Double),
+                                                Tuple.Create("s2", TypeCode.String),
+                                                Tuple.Create("w2", TypeCode.String));
+
+         _leg_price_stream = new TimeSeriesStream(new SyntheticSpreadLegsStream(s1, s2));
+         _leg_price_stream = new ThrottleStream(_leg_price_stream, TimeSpan.FromSeconds(1), vsd);
+      }
+
+      protected override void myOnFirstSubscriber() {
+         _leg_price_stream.onChange += _handle_leg_prices;
+      }
+
+      protected override void myOnLastSubscriber() {
+         _leg_price_stream.onChange -= _handle_leg_prices;
+      }
+
+      private void _handle_leg_prices(object dp_obj){
+         DataPacket dp = dp_obj as DataPacket;
+         List<Tuple<double, double, DateTime>> leg_prices = new List<Tuple<double, double, DateTime>>();
+         foreach (IBlueRow r in dp.newRows){
+            double s1_mid = (r.get<double>("s1_bid_price") + r.get<double>("s1_ask_price")) /2.0;
+            double s2_mid = (r.get<double>("s2_bid_price") + r.get<double>("s2_ask_price")) /2.0;
+            DateTime ts = r.get_datetime("timestamp");
+            leg_prices.Add(new Tuple<double, double, DateTime>(s1_mid, s2_mid, ts));
+         }
+         if (leg_prices.Count <= 2)
+            return;
+         leg_prices = leg_prices.Where(t => (DateTime.Now - t.Item3).TotalSeconds <= _window_secs).ToList();
+         DateTime t0 = leg_prices.Select(t => t.Item3).Min();
+         DateTime t1 = leg_prices.Select(t => t.Item3).Max();
+         double curr_windows_seconds = (t1 - t0).TotalSeconds;
+         int n_ticks = leg_prices.Count;
+         List<double> s1_mids = leg_prices.Select(t => t.Item1).ToList();
+         List<double> s2_mids = leg_prices.Select(t => t.Item2).ToList();
+         (double w1, double w2) = SpreadWeightCalculator.CalculateWeightsLinearRegression(s1_mids, s2_mids);
+         BlueTable state = new BlueTable(_schema);
+         IBlueRow nr = new BlueRow(_schema, 0, t0, t1, curr_windows_seconds, n_ticks,_s1, w1, _s2, w2);
+         state.addRow(nr);
+         var dp_out = BlueDataUtility.build_dp_inc(state, myLastValue as DataPacket, true);
+         myFireEvent(dp_out);
+
+      }
+   }
+}
+

src/DataStreamSuite/SharedStreams/Business/MarketData/Analytics/SpreadStatsStream.cs

@@ -0,0 +1,143 @@
+namespace bts.SilverPine {
+   using BCG;
+
+   using MathNet.Numerics.LinearAlgebra.Factorization;
+
+   using System;
+   using System.Collections.Generic;
+   using System.Diagnostics;
+   using System.Linq;
+   using System.Text;
+   using System.Threading.Tasks;
+   [DataStreamUserRespresentation("SpreadStatsStream")]
+   // This is the fundamental stream that computes the stats (including buy and sell zones) 
+   // of a spread. the legs and the weights are passed as args
+   public class SpreadStatsStream : ViewStream {
+      private ViewStream _spread_price_stream;
+      private readonly IBlueTableSchema _schema;
+      private readonly TimeSpan _window_sz;
+      private readonly string _spread;
+      public SpreadStatsStream(string s1, string s2, double w1, double w2, int rolling_window_sz_seconds, int window_sampling_sz_secs)
+         : this(s1, s2, w1, w2, rolling_window_sz_seconds, window_sampling_sz_secs, new BBOStream(ViewStreamDatabase.instance), ViewStreamDatabase.instance) { }
+      public SpreadStatsStream(string s1, string s2, double w1, double w2, int rolling_window_sz_seconds, int window_sampling_sz_secs, ViewStream bbo_stream, ViewStreamDatabase vsd) {
+         _window_sz = TimeSpan.FromSeconds(rolling_window_sz_seconds);
+         _spread = $"{w1} X {s1} - {w2} X {s2}";
+         _schema = new BlueTableSchema("SpreaddStats", new ColumnInfo ("id", 0, TypeCode.Int32));
+         _schema = _schema.add_column(TypeCode.String, "spread");
+         _schema = _schema.add_column(TypeCode.String, "window");
+         // basic stats
+         _schema = _schema.add_column(TypeCode.Double, "bid_mean");
+         _schema = _schema.add_column(TypeCode.Double, "ask_mean");
+         _schema = _schema.add_column(TypeCode.Double, "bid_stddev");
+         _schema = _schema.add_column(TypeCode.Double, "ask_stddev");
+         _schema = _schema.add_column(TypeCode.Double, "bid_skewness");
+         _schema = _schema.add_column(TypeCode.Double, "ask_skewness");
+         _schema = _schema.add_column(TypeCode.Double, "bid_kurtosis");
+         _schema = _schema.add_column(TypeCode.Double, "ask_kurtosis");
+         _schema = _schema.add_column(TypeCode.Double, "bid_max");
+         _schema = _schema.add_column(TypeCode.Double, "ask_max");
+         _schema = _schema.add_column(TypeCode.Double, "bid_min");
+         _schema = _schema.add_column(TypeCode.Double, "ask_min");
+         // bid bins
+         _schema = _schema.add_column(TypeCode.Double, "bid Z < -2σ");
+         _schema = _schema.add_column(TypeCode.Double, "bid -2σ ≤ Z < -1σ");
+         _schema = _schema.add_column(TypeCode.Double, "bid -1σ ≤ Z < 0σ");
+         // positive z's
+         _schema = _schema.add_column(TypeCode.Double, "bid 0σ ≤ Z < 1σ");
+         _schema = _schema.add_column(TypeCode.Double, "bid 1σ ≤ Z < 2σ");
+         _schema = _schema.add_column(TypeCode.Double, "bid Z ≥ 2σ");
+         // ask bins
+         _schema = _schema.add_column(TypeCode.Double, "ask Z < -2σ");
+         _schema = _schema.add_column(TypeCode.Double, "ask -2σ ≤ Z < -1σ");
+         _schema = _schema.add_column(TypeCode.Double, "ask -1σ ≤ Z < 0σ");
+         // positive z's
+         _schema = _schema.add_column(TypeCode.Double, "ask 0σ ≤ Z < 1σ");
+         _schema = _schema.add_column(TypeCode.Double, "ask 1σ ≤ Z < 2σ");
+         _schema = _schema.add_column(TypeCode.Double, "ask Z ≥ 2σ");
+         // opps
+         _schema = _schema.add_column(TypeCode.Int32, "n_buy_ticks");
+         _schema = _schema.add_column(TypeCode.Double, "cumm_buy_price");
+         _schema = _schema.add_column(TypeCode.Int32, "n_sell_ticks");
+         _schema = _schema.add_column(TypeCode.Double, "cumm_sell_price");
+         _schema = _schema.add_column(TypeCode.Int32, "n_trade_ticks");
+         _spread_price_stream = new SyntheticSpreadStream(s1, s2, w1, w2, bbo_stream, vsd);
+         _spread_price_stream = new TimeSeriesStream(_spread_price_stream);
+         double window_sampling_sz_smsecs = window_sampling_sz_secs * 1000.0 + new Random(DateTime.Now.Second).Next(-100, 100);
+         TimeSpan sampling_period = TimeSpan.FromMilliseconds(window_sampling_sz_smsecs);
+         _spread_price_stream = new ThrottleStream(_spread_price_stream, sampling_period, vsd);
+      }
+
+      protected override void myOnFirstSubscriber() {
+         _spread_price_stream.onChange += _handle_spread_prices;
+      }
+
+      protected override void myOnLastSubscriber() {
+         _spread_price_stream.onChange -= _handle_spread_prices;
+      }
+
+      private void _handle_spread_prices(object dp_obj) {
+         List<IBlueRow> rows = (dp_obj as DataPacket).newRows.to_list();
+         Func<IBlueRow, bool> row_filter = (r) => {
+            DateTime ts = r.get_datetime("timestamp");
+            DateTime start_time = DateTime.Now.Subtract(_window_sz);
+            return ts >= start_time;
+         };
+         var rolling_win_rows = rows.Where(r => row_filter(r));
+         var spread_prices = rolling_win_rows.Select(r => (
+                                          r.get_datetime("timestamp"),
+                                          r.get<double>("spread_bid_price"),
+                                          r.get<double>("spread_bid_sz"),
+                                          r.get<double>("spread_ask_price"),
+                                          r.get<double>("spread_ask_sz"))).ToList();
+         spread_prices.Sort((a,b) => a.Item1.CompareTo(b.Item1));
+         DateTime t0 = spread_prices.First().Item1, t1 = spread_prices.Last().Item1;
+         var tot_secs = (t1 - t0).TotalSeconds;
+         var stopwatch = Stopwatch.StartNew();
+         Log.Gen.info($"SpreadStatsStream.analyze_spread_quotes executed in {stopwatch.ElapsedMilliseconds} ms");
+         var result = SpreadPerformanceEvaluator.SpreadStatistics.analyze_spread_quotes(spread_prices);
+         (uint n_buy_signals, double cumm_buy_price, uint n_sell_signals, double cumm_sell_price) = 
+              SpreadPerformanceEvaluator.EvaluateExcursionCount(spread_prices.Select(t => Tuple.Create(t.Item2, t.Item3, t.Item4, t.Item5)).ToList(), 2.0);
+         IBlueRow nr = new BlueRow(_schema,
+            0,
+            _spread,
+            $"n_ticks: {spread_prices.Count} t0: {t0.ToLongTimeString()} -> t1: {t1.ToLongTimeString()} ({tot_secs}) secs",
+            result.bid_stats.Mean,
+            result.ask_stats.Mean,
+            result.bid_stats.StandardDeviation,
+            result.ask_stats.StandardDeviation,
+            result.bid_stats.Skewness,
+            result.ask_stats.Skewness,
+            result.bid_stats.Kurtosis,
+            result.ask_stats.Kurtosis,
+            result.bid_stats.Maximum,
+            result.ask_stats.Maximum,
+            result.bid_stats.Minimum,
+            result.ask_stats.Minimum,
+            // bid distribution
+            result.bid_zscore_distribution["Z < -2σ"],
+            result.bid_zscore_distribution["-2σ ≤ Z < -1σ"],
+            result.bid_zscore_distribution["-1σ ≤ Z < 0σ"],
+            result.bid_zscore_distribution["0σ ≤ Z < 1σ"],
+            result.bid_zscore_distribution["1σ ≤ Z < 2σ"],
+            result.bid_zscore_distribution["Z ≥ 2σ"],
+            // ask distributions
+            result.ask_zscore_distribution["Z < -2σ"],
+            result.ask_zscore_distribution["-2σ ≤ Z < -1σ"],
+            result.ask_zscore_distribution["-1σ ≤ Z < 0σ"],
+            result.ask_zscore_distribution["0σ ≤ Z < 1σ"],
+            result.ask_zscore_distribution["1σ ≤ Z < 2σ"],
+            result.ask_zscore_distribution["Z ≥ 2σ"],
+            n_buy_signals,
+            cumm_buy_price,
+            n_sell_signals,
+            cumm_sell_price,
+            n_buy_signals + n_sell_signals
+            );
+         stopwatch.Stop();
+         BlueTable t = new BlueTable(_schema);
+         t.addRow(nr);
+         var dp_out = BlueDataUtility.build_dp(t, myLastValue as DataPacket);
+         myFireEvent(dp_out);
+      }
+   }
+}

src/DataStreamSuite/SharedStreams/Business/MarketData/Analytics/SpreadWeightCalculator.cs

@@ -0,0 +1,49 @@
+using System;
+using System.Collections.Generic;
+using MathNet.Numerics.LinearRegression;
+
+namespace bts.SilverPine {
+
+   public static class SpreadWeightCalculator {
+      /// <summary>
+      /// Estimates hedge weights (w1, w2) such that the spread s = w1 * s1 - w2 * s2 
+      /// is potentially mean-reverting. Uses linear regression: s1 = beta * s2 + ε.
+      /// 
+      /// Assumes mid-prices are precomputed and aligned over the same time window.
+      /// </summary>
+      /// <param name="s1MidPrices">List of mid-prices for asset 1 (dependent variable)</param>
+      /// <param name="s2MidPrices">List of mid-prices for asset 2 (independent variable)</param>
+      /// <param name="includeIntercept">
+      /// Whether to include an intercept in the regression model (default: true).
+      /// </param>
+      /// <returns>(w1, w2): Tuple of hedge weights</returns>
+      public static (double w1, double w2) CalculateWeightsLinearRegression(
+          List<double> s1MidPrices,
+          List<double> s2MidPrices,
+          bool includeIntercept = true) {
+         if (s1MidPrices == null || s2MidPrices == null)
+            throw new ArgumentNullException("Price series cannot be null.");
+         if (s1MidPrices.Count != s2MidPrices.Count)
+            throw new ArgumentException("Price series must be of equal length.");
+         if (s1MidPrices.Count < 2)
+            throw new ArgumentException("Need at least two data points for regression.");
+
+         double[] x = s2MidPrices.ToArray(); // Independent variable
+         double[] y = s1MidPrices.ToArray(); // Dependent variable
+
+         double beta;
+
+         if (includeIntercept) {
+            // Returns (intercept, slope)
+            (double intercept, double slope) = SimpleRegression.Fit(x, y);
+            beta = slope;
+         } else {
+            // Returns only slope when forcing regression through origin
+            beta = SimpleRegression.FitThroughOrigin(x, y);
+         }
+
+         // Return fixed w1 = 1.0 and w2 = beta
+         return (1.0, beta);
+      }
+   }
+}

src/DataStreamSuite/SharedStreams/Business/MarketData/BBOStream.cs

@@ -13,7 +13,6 @@ namespace bts.SilverPine {
    using Spark.Api.MarketData;
 
    using SecurityKey = System.UInt32;
-   using System.Data.Entity.Infrastructure;
 
    public class FakeBBOStream : ViewStream, IWorker {
       private string _symbol;
@@ -105,14 +104,14 @@ namespace bts.SilverPine {
    public class BBOStream : ViewStream, IWorker {
       private readonly TimeSpan _poll_period = TimeSpan.FromMilliseconds(500);
       private DateTime _last_poll_time = DateTime.MinValue;
-      private ViewStreamDatabase _vsd;
-      private int _id = 0;
-      IReadOnlyDictionary<SecurityKey, BookTuple> _cached_state = null;
-      MarketDataClient _md_client;
+      private ViewStreamDatabase? _vsd;
+      IReadOnlyDictionary<SecurityKey, BookTuple>? _cached_state = null;
+      MarketDataClient? _md_client;
       internal static IBlueTableSchema _bbo_schema = BlueDataUtility.form_schema("bbo",
                                                 Tuple.Create("id", TypeCode.Int32),
                                                 Tuple.Create("sec_key", TypeCode.UInt32),
                                                 Tuple.Create("sec", TypeCode.String),
+                                                Tuple.Create("exchange", TypeCode.String),
                                                 Tuple.Create("bid_price", TypeCode.Double),
                                                 Tuple.Create("bid_sz", TypeCode.Single),
                                                 Tuple.Create("ask_price", TypeCode.Double),
@@ -124,17 +123,18 @@ namespace bts.SilverPine {
       }
 
       public BBOStream(ViewStreamDatabase vsd) {
-         // bts.Log.Gen.info("BBOStream.cons");
-         // _vsd = vsd;
-         // _md_client = new MarketDataClient(new MDSource(TimeSpan.FromSeconds(0.5)));
-         FakeBBOStream rs = new FakeBBOStream("Future", "CME", "ES", 5000, 0.25, vsd);
-         mySetRealSource(rs);
+         bts.Log.Gen.info("BBOStream.cons");
+         _vsd = vsd;
+         _md_client = new MarketDataClient(new MDSource(TimeSpan.FromSeconds(0.5)));
+         //FakeBBOStream rs = new FakeBBOStream("Future", "CME", "ES", 5000, 0.25, vsd);
+         //mySetRealSource(rs);
       }
 
       protected override void myOnFirstSubscriber() {
          _vsd.addWorker(this);
          _md_client.books().Subscribe(_on_books);
-
+         _vsd?.addWorker(this);
+         _md_client?.books().Subscribe(_on_books);
       }
 
       private void _log_updates(TableUpdate<BookTuple, uint> update) {
@@ -144,24 +144,24 @@ namespace bts.SilverPine {
          }
          foreach (Change<BookTuple> c in update.changes) {
             var bt = c.New;
-            bts.Log.Gen.info($"BBOStream changes: idx: {i} bid: {bt.book.bids[0].price} / {bt.book.bids[0].size} ask: {bt.book.asks[0].price} / {bt.book.asks[0].size}");
+            //bts.Log.Gen.info($"BBOStream changes: idx: {i} bid: {bt.book.bids[0].price} / {bt.book.bids[0].size} ask: {bt.book.asks[0].price} / {bt.book.asks[0].size}");
          }
       }
 
       private void _on_books(TableUpdate<BookTuple, uint> update) {
          _log_updates(update);
          // just cacching in here, will populate the rows in _poll_scan and fire the datapacket
-         bts.Log.Gen.info($"BBOStream._on_books state: {update.state.Count()} changes: {update.changes.Count()} added: {update.added.Count()}");
+         //bts.Log.Gen.info($"BBOStream._on_books state: {update.state.Count()} changes: {update.changes.Count()} added: {update.added.Count()}");
          _cached_state = update.state;
       }
 
       protected override void myOnLastSubscriber() {
-         _vsd.removeWorker(this);
+         _vsd?.removeWorker(this);
       }
 
       private void _scan_bbo() {
          // this shouldn be made incremental using the TableUpdate
-         bts.Log.Gen.info("BBOStream._scan_bbo");
+         //bts.Log.Gen.info("BBOStream._scan_bbo");
          if (_cached_state == null)
             return;
          BlueTable new_state = new BlueTable(_bbo_schema);
@@ -170,22 +170,23 @@ namespace bts.SilverPine {
             var sec = kv.Value.security;
             int id = Convert.ToInt32(skey);
             BookTuple bt = kv.Value;
-            IBlueRow r = new BlueRow(_bbo_schema, id, skey, sec.ToString(), bt.book.bids[0].price, bt.book.bids[0].size,
+            var venue = sec.Exchange;
+            IBlueRow r = new BlueRow(_bbo_schema, id, skey, sec.ToString(), venue.ToString(), bt.book.bids[0].price, bt.book.bids[0].size,
                                      bt.book.asks[0].price, bt.book.asks[0].size);
             new_state.addRow(r);
          }
-         DataPacket dp = null;
-         if (myLastValue == null) {
-            dp = new DataPacket(new_state, new_state, new BlueTable(_bbo_schema));
-         } else {
-            DataPacket old_dp = myLastValue as DataPacket;
-            IBlueRowCollection added_rows, del_rows;
-            BlueDataUtility.compute_differences_fast(new_state, old_dp.newRows, out del_rows, out added_rows);
-            dp = new DataPacket(new_state, added_rows, del_rows);
-         }
+         var dp = accumulate(myLastValue as DataPacket, new_state);
          if (dp.addedRows.Count == 0 && dp.deletedRows.Count == 0)
             return;
          myFireEvent(dp);
+
+         static DataPacket accumulate(DataPacket? old_dp, BlueTable new_state) {
+            if (old_dp is null) {
+               return new(new_state, new_state, new BlueTable(_bbo_schema));
+            }
+            BlueDataUtility.compute_differences_fast(new_state, old_dp.newRows, out var del_rows, out var added_rows);
+            return new(new_state, added_rows, del_rows);
+         }
       }
 
       public void doWork() {
@@ -195,4 +196,4 @@ namespace bts.SilverPine {
          }
       }
    }
-}
+}

src/DataStreamSuite/SharedStreams/Business/MarketData/Bars/HistoricalBarsStream.cs

@@ -148,27 +148,22 @@ namespace bts.SilverPine {
 
    [DataStreamUserRespresentation("FutureHistoricalBars")]
    public class FutureHistoricalBarsStream : ViewStream {
-      public FutureHistoricalBarsStream(string fut_contract, string bar_size_txt, string range_beg, string range_end) {
-         DateTime dt_range_beg = DateTime.Parse(range_beg);
-         DateTime dt_range_end = DateTime.Parse(range_end);
-         TimeSpan diff = dt_range_end - dt_range_beg;
-         BarSize bar_size = Enum.Parse<BarSize>(bar_size_txt, true);
-         ViewStream rs = new FutureHistoricalBarsStream(fut_contract, bar_size.ToString(), dt_range_beg, dt_range_end);
-         mySetRealSource(rs);
-      }
-
       public FutureHistoricalBarsStream(string fut_contract, string range_beg, string range_end) {
          DateTime dt_range_beg = DateTime.Parse(range_beg);
          DateTime dt_range_end = DateTime.Parse(range_end);
          TimeSpan diff = dt_range_end - dt_range_beg;
-         BarSize bar_size = diff < TimeSpan.FromDays(1.0) ? BarSize.ONE_DAY : BarSize.ONE_MIN;
+         BarSize bar_size = diff < TimeSpan.FromDays(1.0) ? BarSize.ONE_MIN : BarSize.ONE_DAY;
          ViewStream rs = new FutureHistoricalBarsStream(fut_contract, bar_size.ToString(), dt_range_beg, dt_range_end);
          mySetRealSource(rs);
       }
-
       public FutureHistoricalBarsStream(string fut_contract, string bar_size)
       : this(fut_contract, bar_size, DateTime.MinValue, DateTime.MaxValue) { }
 
+      public FutureHistoricalBarsStream(string fut_contract, string bar_size,
+                                        string dt_range_beg, string dt_range_end)
+                                        : this(fut_contract, bar_size, DateTime.Parse(dt_range_beg),
+                                        DateTime.Parse(dt_range_end)) { }
+
       public FutureHistoricalBarsStream(string fut_contract, string bar_size,
                                         DateTime dt_range_beg, DateTime dt_range_end)
                                         :

src/DataStreamSuite/SharedStreams/Business/MarketData/BookStream.cs

@@ -21,7 +21,7 @@ namespace BCG {
       private DateTime _last_poll_time = DateTime.MinValue;
       private ViewStreamDatabase _vsd;
       private int _id = 0;
-      IReadOnlyDictionary<SecurityKey, BookTuple> _cached_state = null;
+      IReadOnlyDictionary<SecurityKey, BookTuple>? _cached_state = null;
       MarketDataClient _md_client;
       readonly Stopwatch _timer = new();
       private readonly IBlueTableSchema _bbo_schema = BlueDataUtility.form_schema("books",
@@ -73,14 +73,8 @@ namespace BCG {
 
       private Dictionary<SecurityKey, Dictionary<int, int>> _id_map = new Dictionary<SecurityKey, Dictionary<int, int>>();
       private int _get_id(SecurityKey skey, int depth) {
-         Dictionary<int, int> d_map = null;
-         if (!_id_map.TryGetValue(skey, out d_map)) {
-            d_map = new Dictionary<int, int>();
-            d_map.Add(depth, _id++);
-            _id_map.Add(skey, d_map);
-         }
-         int rv = 0;
-         if (!d_map.TryGetValue(depth, out rv)) {
+         var d_map = _id_map.GetOrAdd(skey, _ => new() { {depth, _id++} });
+         if (!d_map.TryGetValue(depth, out int rv)) {
             rv = _id++;
             d_map.Add(depth, rv);
          }

src/DataStreamSuite/SharedStreams/Business/MarketData/InstrumentsStream.cs

@@ -1,8 +1,9 @@
-namespace BCG {
+namespace bts.SilverPine {
    using BCG.Utilities.General;
 
    using bts.api;
    using bts.md;
+   using BCG;
 
    using Spark.Core.Reactive;
 
@@ -31,6 +32,7 @@ namespace BCG {
                                                 Tuple.Create("sec_type", TypeCode.String),
                                                 Tuple.Create("venue", TypeCode.String),
                                                 Tuple.Create("symbol", TypeCode.String),
+                                                Tuple.Create("currency", TypeCode.String),
                                                 Tuple.Create("expiration", TypeCode.DateTime),
                                                 Tuple.Create("strike", TypeCode.Double),
                                                 Tuple.Create("callput", TypeCode.String)
@@ -81,7 +83,7 @@ namespace BCG {
          foreach (var kv in _cached_state) {
             SecurityKey skey = kv.Key;
             int id = Convert.ToInt32(skey);
-            SecurityImmutable sec = kv.Value;
+            SecurityImmutable sec = kv.Value;            
             bts.api.SecurityType sec_type = sec.security_type();
             object exp = DBNull.Value;
             if (sec_type == bts.api.SecurityType.Future || sec_type == bts.api.SecurityType.Option)
@@ -92,7 +94,7 @@ namespace BCG {
             object cp = DBNull.Value;
             if (sec_type == bts.api.SecurityType.Option)
                cp = sec.Option.CallPut.ToString();
-            IBlueRow r = new BlueRow(_instrument_schema, id, skey, sec.security_desc(), sec.security_type().ToString(), sec.display_exchange().ToString(), sec.Symbol, exp, strike, cp);
+            IBlueRow r = new BlueRow(_instrument_schema, id, skey, sec.security_desc(), sec.security_type().ToString(), sec.display_exchange().ToString(), sec.Symbol, sec.Currency, exp, strike, cp);
             new_state.addRow(r);
          }
          DataPacket dp = null;
@@ -117,4 +119,4 @@ namespace BCG {
          }
       }
    }
-}
+}

src/DataStreamSuite/SharedStreams/Business/MarketData/OHLCVStream.cs

@@ -174,10 +174,10 @@ namespace bts.SilverPine {
                // TODO: assuming 1 day only for historic               
                string table_name = get_table_from_barsize(ohlcv_params.bar_size, symbol);// $"{symbol}_OHLCV_1DAY";
                                                                                          //string table_name = $"{symbol}_{ohlcv_params.bar_size}";
-               string db_str = $"Data Source={_location}{_db_name};Version=3";
+               string db_name = _db_name;// $"Data Source={_location}{_db_name};Version=3";
                string sql_str = $"select * from {table_name}";
                // "Data Source=C:\\Users\\m_tah\\development\\data\\stream_data.db;Version=3;"
-               ViewStream rs = new SQLiteStream(db_str, sql_str);
+               ViewStream rs = new SQLiteStream(db_name, sql_str);
                rs = new SchemaGuesserStream(rs);
                if (ohlcv_params.bar_size == OHLCVBarSize.ONE_DAY)
                   rs = new DateStandardizationStream(rs, "ts_event", DateConversion.ChopTime);
@@ -207,4 +207,4 @@ namespace bts.SilverPine {
 
 
    }
-}
+}

src/DataStreamSuite/SharedStreams/Business/MarketData/SyntheticSpreadStream.cs

@@ -0,0 +1,125 @@
+using System;
+using BCG;
+
+namespace bts.SilverPine
+{
+/*
+   ----------------------------------------------------------------------
+                               *Expression*
+   ----------------------------------------------------------------------
+       s1 = filter(bbo(), "sec = 'ETH-USDT-SWAP'")
+       s1 = addColumns(s1, "ljc:0")
+       s2 = filter(bbo(), "sec = 'ETH-USDT-SWAP'")
+       s2 = addColumns(s2, "rjc:0")
+       
+       s = fastJoin(s1, "ljc", "sec:s1 bid_price:s1_bid_price bid_sz:s1_bid_sz ask_price:s1_ask_price ask_sz:s1_ask_sz", 
+                    s2, "rjc", "sec:s2 bid_price:s2_bid_price bid_sz:s2_bid_sz ask_price:s2_ask_price ask_sz:s2_ask_sz")
+       
+
+   ----------------------------------------------------------------------
+                                 *Schema*
+   ----------------------------------------------------------------------
+       id                              System.Int32
+       s1                              System.String
+       s1_bid_price                    System.Double
+       s1_bid_sz                       System.Single
+       s1_ask_price                    System.Double
+       s1_ask_sz                       System.Single
+       s2                              System.String
+       s2_bid_price                    System.Double
+       s2_bid_sz                       System.Single
+       s2_ask_price                    System.Double
+       s2_ask_sz                       System.Single
+*/
+
+
+   // Combines the market data of spread legs into one stream, lined up nicely in a row.
+   // this is useful for operations like finding weights, that need leg market data organized together succintly.
+   public class SyntheticSpreadLegsStream : ViewStream 
+   {
+      public SyntheticSpreadLegsStream(string s1, string s2) : 
+         this(s1, s2, new BBOStream(ViewStreamDatabase.instance), ViewStreamDatabase.instance) {}
+
+      public SyntheticSpreadLegsStream(string s1, string s2, ViewStream bbo_stream, ViewStreamDatabase vsb)
+      {
+         ViewStream vs1;
+         ViewStream vs2;
+         ViewStream rs;
+
+         vs1 = new FilterStream (new BBOStream (),
+                                $"sec = '{s1}'");
+         vs1 = new AddColumnsStream (vs1,
+                                    "ljc:0");
+         vs2 = new FilterStream (bbo_stream,
+                                $"sec = '{s2}'");
+         vs2 = new AddColumnsStream (vs2,
+                                    "rjc:0");
+         rs = new NonIncJoinStream (vs1,
+                                   "ljc",
+                                   "sec:s1 bid_price:s1_bid_price bid_sz:s1_bid_sz ask_price:s1_ask_price ask_sz:s1_ask_sz",
+                                   vs2,
+                                   "rjc",
+                                   "sec:s2 bid_price:s2_bid_price bid_sz:s2_bid_sz ask_price:s2_ask_price ask_sz:s2_ask_sz");
+         mySetRealSource(rs);
+      }
+   }
+   public class SyntheticSpreadStream : ViewStream{
+      public SyntheticSpreadStream (string s1, string s2, double w1, double w2)
+      : this(s1, s2, w1, w2, new BBOStream(ViewStreamDatabase.instance), ViewStreamDatabase.instance){}
+      public SyntheticSpreadStream (string s1, string s2, double w1, double w2, ViewStream bbo_stream, ViewStreamDatabase vsd){
+         ViewStream rs = new SyntheticSpreadLegsStream(s1, s2, bbo_stream, vsd);
+
+         Func<IBlueTableSchema, IBlueTableSchema> get_schema = (inp_schema) => {
+            IBlueTableSchema rv = inp_schema;
+            rv = rv.add_column(TypeCode.Double, "spread_bid_price");
+            rv = rv.add_column(TypeCode.Double, "spread_bid_sz");
+            rv = rv.add_column(TypeCode.Double, "spread_ask_price");
+            rv = rv.add_column(TypeCode.Double, "spread_ask_sz");
+            return rv;
+         };
+
+         rs = new MapStream(rs, (inp_schema) => {
+           return get_schema(inp_schema);
+         }, 
+         (r) => {
+              double s1_bid_price = r.get<double>("s1_bid_price");
+              double s1_bid_sz = r.get<double>("s1_bid_sz");
+              double s1_ask_price = r.get<double>("s1_ask_price");
+              double s1_ask_sz = r.get<double>("s1_ask_sz");
+               
+              double s2_bid_price = r.get<double>("s2_bid_price");
+              double s2_bid_sz = r.get<double>("s2_bid_sz");
+              double s2_ask_price = r.get<double>("s2_ask_price");
+              double s2_ask_sz = r.get<double>("s2_ask_sz");
+               
+              (double spread_bid_price, double spread_ask_price, double spread_bid_sz, double spread_ask_sz) =   
+              ComputeSpreadBidAsk(s1_bid_price, s1_ask_price, s2_bid_price, s2_ask_price, s1_bid_sz, s1_ask_sz, s2_bid_sz, s2_ask_sz, w1, w2);
+              IBlueTableSchema out_schema = get_schema(r.schema);
+              MutableBlueRow rv = new MutableBlueRow(out_schema, r);
+              rv.setValue("spread_bid_price", spread_bid_price);
+              rv.setValue("spread_ask_price", spread_ask_price);
+              rv.setValue("spread_bid_sz", spread_bid_sz);
+              rv.setValue("spread_ask_sz", spread_ask_sz);
+              return rv;
+              }
+           );
+         mySetRealSource(rs);
+
+      }
+      public static (double SpreadBid, double SpreadAsk, double SpreadBidQty, double SpreadAskQty) ComputeSpreadBidAsk(
+        double s1Bid, double s1Ask,
+        double s2Bid, double s2Ask,
+        double s1BidQty, double s1AskQty,
+        double s2BidQty, double s2AskQty,
+        double w1, double w2)
+    {
+        double spreadBid = (w1 * s1Bid) - (w2 * s2Ask);
+        double spreadAsk = (w1 * s1Ask) - (w2 * s2Bid);
+
+        double spreadBidQty = Math.Min(s1BidQty / w1, s2AskQty / w2);
+        double spreadAskQty = Math.Min(s1AskQty / w1, s2BidQty / w2);
+
+        return (spreadBid, spreadAsk, spreadBidQty, spreadAskQty);
+    }
+   }
+}

src/DataStreamSuite/SharedStreams/SharedStreams.csproj

@@ -1,7 +1,7 @@
 <Project Sdk="Microsoft.NET.Sdk">
 
   <PropertyGroup>
-    <TargetFramework>net7.0</TargetFramework>
+    <TargetFramework>net8.0</TargetFramework>
     <ImplicitUsings>enable</ImplicitUsings>
     <Nullable>enable</Nullable>
   </PropertyGroup>

src/Directory.Build.props

@@ -43,7 +43,7 @@
       Condition="Exists($(ComputerSpecificConfig))" />
 
   <PropertyGroup Label="ProtoBuf">
-    <ProtocPluginBase>$(SparkRoot)\src\lib\api\service_compiler</ProtocPluginBase>
+    <ProtocPluginBase>$(SparkRoot)\src\lib_spark\spark\api\service_compiler</ProtocPluginBase>
   </PropertyGroup>
   <Choose>
     <When Condition="'$(OS)' == 'Windows_NT'">

src/Directory.Build.targets

@@ -24,7 +24,7 @@
     <PropertyGroup>
       <GenProtosCmd>$(protoc) %(Proto.FullPath)</GenProtosCmd>
       <GenProtosCmd>$(GenProtosCmd) --error_format=msvs</GenProtosCmd>
-      <GenProtosCmd>$(GenProtosCmd) --proto_path=$(SourceRoot)</GenProtosCmd>
+      <GenProtosCmd>$(GenProtosCmd) --proto_path=$(SourceRoot)/lib_spark</GenProtosCmd>
       <GenProtosCmd>$(GenProtosCmd) --plugin=&quot;protoc-gen-spark-csharp=$(ProtocPlugin)&quot;</GenProtosCmd>
       <GenProtosCmd>$(GenProtosCmd) --spark-csharp_out=lang=c#,templates=immutable.pb.cs,strip_out_path=1:%(Proto.OutDir)</GenProtosCmd>
       <GenProtosCmd>$(GenProtosCmd) --csharp_out=%(Proto.OutDir)</GenProtosCmd>
@@ -65,7 +65,7 @@
     <PropertyGroup>
       <GenProtosCmd>$(protoc) %(ProtoService.FullPath)</GenProtosCmd>
       <GenProtosCmd>$(GenProtosCmd) --error_format=msvs</GenProtosCmd>
-      <GenProtosCmd>$(GenProtosCmd) --proto_path=$(SourceRoot)</GenProtosCmd>
+      <GenProtosCmd>$(GenProtosCmd) --proto_path=$(SourceRoot)/lib_spark</GenProtosCmd>
       <GenProtosCmd>$(GenProtosCmd) --plugin=&quot;protoc-gen-service=$(ProtocPlugin)&quot;</GenProtosCmd>
       <GenProtosCmd>$(GenProtosCmd) --service_out=lang=c#,strip_out_path=1:%(ProtoService.OutDir)</GenProtosCmd>
       <GenProtosCmd>$(GenProtosCmd) --csharp_out=%(ProtoService.OutDir)</GenProtosCmd>

src/Spark.Admin.WebApp/CMakeLists.txt

@@ -1,56 +1,42 @@
 cmake_minimum_required(VERSION 3.5)
 project(Spark.Admin.WebApp)
 
+include(GNUInstallDirs)
 include(${CMAKE_SOURCE_DIR}/cmake/safe_dep_glob.cmake)
-include(${CMAKE_SOURCE_DIR}/cmake/Protobuf.cmake)
+include(${CMAKE_SOURCE_DIR}/cmake/SymLink.cmake)
+include(${CMAKE_SOURCE_DIR}/cmake/ProtobufGenerateJs.cmake)
+include(${CMAKE_SOURCE_DIR}/cmake/ProtobufGenerateFs.cmake)
 
-# dependencies
-set(PROTO_GLOBS
-  ../lib/api/*.proto
-  ../lib/api/services/*.proto
-  ../lib/domain_model/*.proto
-  ../lib/sparkproto/*.proto
-  ../lib/md/protos/*.proto
+file(
+  GLOB_RECURSE ${PROJECT_NAME}_PROTO_SOURCES
+  RELATIVE ${CMAKE_CURRENT_SOURCE_DIR}
+  ../lib_spark/spark/api/*.proto
+  ../lib_spark/spark/api/services/*.proto
+  ../lib_spark/spark/domain_model/*.proto
+  ../lib_spark/spark/sparkproto/*.proto
+  ../lib_spark/spark/md/protos/*.proto
   ../../externals/linux/include/google/protobuf/descriptor.proto
-  ../../externals/linux/include/google/protobuf/timestamp.proto)
+  ../../externals/linux/include/google/protobuf/timestamp.proto
+)
 
-file(
-  GLOB_RECURSE _proto_rel_SOURCES
-  RELATIVE ${CMAKE_CURRENT_SOURCE_DIR}
-  ${PROTO_GLOBS})
-
-file(
-  GLOB_RECURSE _proto_abs_SOURCES
-  ${PROTO_GLOBS})
-
-file(
-  GLOB_RECURSE _ts_SOURCES
-  RELATIVE ${CMAKE_CURRENT_SOURCE_DIR}
-  src/*.ts)
-
-file(
-  GLOB_RECURSE _fs_SOURCES
-  RELATIVE ${CMAKE_CURRENT_SOURCE_DIR}
-  src/*.fs)
+add_custom_command(
+  OUTPUT ${CMAKE_CURRENT_SOURCE_DIR}/package-lock.json
+  COMMAND npm install --include=dev
+  DEPENDS ${CMAKE_CURRENT_SOURCE_DIR}/package.json
+  WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}
+)
 
 set(_tools ${CMAKE_SOURCE_DIR}/tools)
 set(_quiet_exec ${_tools}/quiet_exec.sh)
 set(_venv_exec ${_tools}/venv_exec.sh)
 set(_tools_py ${CMAKE_SOURCE_DIR}/tools/py)
 
-# The tsconfig file is generated as a function of the cmake build type, and
-# binary dir. Depending on the build type, it extends either
-# 'tsconfig.debug.json' or 'tsconfig.release.json'.
 set(_tsconfig_base "${CMAKE_CURRENT_SOURCE_DIR}/tsconfig.$<IF:$<CONFIG:Debug>,debug,release>.json")
-set(WEBPACK_MODE $<IF:$<CONFIG:Debug>,"development","production">)
 set(_tsconfig "${CMAKE_CURRENT_BINARY_DIR}/tsconfig.json")
 
 set(_venv_python ${CMAKE_SOURCE_DIR}/.venv/bin/python)
 set(_gen_tsconfig_py ${_tools_py}/gen_tsconfig.py)
 set(_build_admin_web_app_py ${_tools_py}/build_admin_web_app.py)
-
-set(_gen_tsconfig_deps ${_venv_exec} ${_venv_python} ${_gen_tsconfig_py} ${_tsconfig_base})
-
 set(_build_admin_web_app_deps
   ${_quiet_exec}
   ${_venv_exec}
@@ -58,51 +44,143 @@ set(_build_admin_web_app_deps
   ${_build_admin_web_app_py}
   ${CMAKE_CURRENT_SOURCE_DIR}/src/Spark.Admin.WebApp.fsproj
   ${CMAKE_CURRENT_SOURCE_DIR}/webpack.config.js
-  ${CMAKE_SOURCE_DIR}/src/lib/api/service_compiler/templates/fs/fable.fs
+  ${CMAKE_SOURCE_DIR}/src/lib_spark/spark/api/service_compiler/templates/fs/fable.fs
   ${CMAKE_CURRENT_SOURCE_DIR}/package-lock.json
-  ${_proto_rel_SOURCES}
+  ${${PROJECT_NAME}_PROTO_SOURCES}
   ${_api_ts_SOURCES}
   ${_ui_fs_SOURCES}
   ${_tsconfig})
 
 update_deps_file("${_build_admin_web_app_deps}")
 
-function(npm_install path)
-  add_custom_command(
-    OUTPUT ${path}/package-lock.json
-    COMMAND "npm" ARGS "install" "--include=dev"
-    DEPENDS ${path}/package.json
-    WORKING_DIRECTORY ${path}
-    COMMENT "Run 'npm install' in ${path}")
-endfunction()
-
-npm_install(${CMAKE_CURRENT_SOURCE_DIR})
-
 add_custom_command(
   OUTPUT ${_tsconfig}
   COMMAND ${_venv_exec}
   ARGS
-  ${_venv_python}
-  ${_gen_tsconfig_py}
-  --config-base ${_tsconfig_base}
-  --config-out ${_tsconfig}
-  --build-out ${CMAKE_CURRENT_BINARY_DIR}
-  DEPENDS ${_gen_tsconfig_deps})
+    ${_venv_python}
+    ${_gen_tsconfig_py}
+    --config-base ${_tsconfig_base}
+    --config-out ${_tsconfig}
+    --build-out ${CMAKE_CURRENT_BINARY_DIR}
+  DEPENDS
+    ${_venv_exec}
+    ${_venv_python}
+    ${_gen_tsconfig_py}
+    ${_tsconfig_base}
+    ${CMAKE_CURRENT_SOURCE_DIR}/package-lock.json
+)
+
+protobuf_generate_js(
+  TARGET_NAME generate_js_protos
+  PROTO_INCLUDE_DIRS
+    ../lib_spark
+    ../../externals/linux/include
+  PROTO_SOURCES ${${PROJECT_NAME}_PROTO_SOURCES}
+  OUTPUT ${CMAKE_CURRENT_BINARY_DIR}/proto.js
+)
+
+protobuf_generate_fs(
+  TARGET_NAME generate_fs_protos
+  PROTO_INCLUDE_DIRS
+    ../lib_spark
+    ../../externals/linux/include
+  PROTO_SOURCES ${${PROJECT_NAME}_PROTO_SOURCES}
+  OUT_DIR ${CMAKE_CURRENT_BINARY_DIR}/fable
+)
+
+# Each input file maps to multiple output files with different extensions
+# (js, js.map, d.ts, fs). Transpile ts -> d.ts (and js), then d.ts -> fs.
+# Who knew front-end development could be so complicated? `\_(=_=)_/`
+file(
+  GLOB_RECURSE ${PROJECT_NAME}_TS_SOURCES
+  RELATIVE ${CMAKE_CURRENT_SOURCE_DIR}
+  src/*.ts
+)
+set(${PROJECT_NAME}_JS_PROTOS)
+set(${PROJECT_NAME}_TS_PROTOS)
+set(${PROJECT_NAME}_FS_PROTOS)
+foreach(_ts_source ${${PROJECT_NAME}_TS_SOURCES})
+  get_filename_component(_ts_source_we ${_ts_source} NAME_WE)
+  set(_prefix ${CMAKE_CURRENT_BINARY_DIR}/${_ts_source_we})
+  list(APPEND ${PROJECT_NAME}_JS_PROTOS ${_prefix}.js ${_prefix}.js.map)
+  list(APPEND ${PROJECT_NAME}_TS_PROTOS ${_prefix}.d.ts ${_prefix}.d.ts.map)
+  list(APPEND ${PROJECT_NAME}_FS_PROTOS ${_prefix}.fs)
+  add_custom_command(
+    OUTPUT ${_prefix}.fs
+    WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}
+    COMMAND npx --no-install ts2fable ${_prefix}.d.ts ${_prefix}.fs
+    DEPENDS ${_prefix}.d.ts
+  )
+endforeach()
 
 add_custom_command(
-  OUTPUT ${CMAKE_CURRENT_BINARY_DIR}/bundle.js
-  COMMAND ${_quiet_exec}
-  ARGS
-  ${_venv_exec}
-  ${_venv_python}
-  ${_build_admin_web_app_py}
-  --protos ${_proto_abs_SOURCES}
-  --tsconfig ${_tsconfig}
-  --mode ${WEBPACK_MODE}
-  --output-dir ${CMAKE_CURRENT_BINARY_DIR}
-  COMMENT "Building Spark.Admin.WebApp"
-  DEPENDS ${_build_admin_web_app_deps})
+  OUTPUT
+    ${${PROJECT_NAME}_JS_PROTOS}
+    ${${PROJECT_NAME}_TS_PROTOS}
+  WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}
+  COMMAND npx
+    --no-install tsc
+    -p ${_tsconfig}
+  DEPENDS
+    ${_tsconfig}
+    ${CMAKE_CURRENT_BINARY_DIR}/proto.js
+)
+add_custom_target(
+  transpile_type_script
+  ALL
+  DEPENDS
+    ${${PROJECT_NAME}_JS_PROTOS}
+    ${${PROJECT_NAME}_TS_PROTOS}
+)
+add_custom_target(generate_fsts_stubs ALL DEPENDS ${${PROJECT_NAME}_FS_PROTOS})
 
-add_custom_target(admin_web_app_bundle ALL DEPENDS ${CMAKE_CURRENT_BINARY_DIR}/bundle.js)
+sym_link_dir(
+  TARGET_NAME link_webapp_public
+  SOURCE ${CMAKE_CURRENT_SOURCE_DIR}/public
+  DEST ${CMAKE_CURRENT_BINARY_DIR}/public
+)
+sym_link_dir(
+  TARGET_NAME link_webapp_root
+  SOURCE ${CMAKE_CURRENT_BINARY_DIR}
+  DEST ${CMAKE_SOURCE_DIR}/build/Spark.Admin.WebApp
+)
 
-install(DIRECTORY "${CMAKE_CURRENT_BINARY_DIR}" DESTINATION "${CMAKE_INSTALL_BINDIR}")
+file(
+  GLOB_RECURSE ${PROJECT_NAME}_FS_SOURCES
+  RELATIVE ${CMAKE_CURRENT_SOURCE_DIR}
+  src/*.fs
+)
+add_custom_command(
+  OUTPUT
+    ${CMAKE_CURRENT_BINARY_DIR}/bundle.js
+    $<$<CONFIG:DEBUG>:${CMAKE_CURRENT_BINARY_DIR}/bundle.js.map>
+  WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/src
+  COMMAND dotnet restore
+  COMMAND dotnet tool restore
+  COMMAND export SPARK_ADMIN_WEB_APP_OUTPUT_DIR=${CMAKE_CURRENT_BINARY_DIR}
+  COMMAND dotnet fable -o ${CMAKE_CURRENT_BINARY_DIR} --verbose
+  COMMAND npm run build -- --mode $<IF:$<CONFIG:Debug>,"development","production">
+  DEPENDS
+    ${${PROJECT_NAME}_FS_SOURCES}
+    ${${PROJECT_NAME}_FS_PROTOS}
+    ${CMAKE_CURRENT_SOURCE_DIR}/src/Spark.Admin.WebApp.fsproj
+)
+add_custom_target(
+  bundle_spark_admin_webapp ALL
+  DEPENDS
+    ${CMAKE_CURRENT_BINARY_DIR}/bundle.js
+    $<$<CONFIG:DEBUG>:${CMAKE_CURRENT_BINARY_DIR}/bundle.js.map>
+  COMMENT "Bundling Spark.Admin.WebApp"
+)
+
+# Exclude 'public' dir because it is a symlink
+install(
+  DIRECTORY "${CMAKE_CURRENT_BINARY_DIR}"
+  DESTINATION "${CMAKE_INSTALL_BINDIR}"
+  PATTERN "public" EXCLUDE
+)
+# Install actual (non-symlinked) 'public' dir
+install(
+  DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}/public"
+  DESTINATION "${CMAKE_INSTALL_BINDIR}/${PROJECT_NAME}"
+)

src/Spark.Admin.WebApp/package.json

@@ -42,7 +42,7 @@
     "ts-loader": "^9.4.2",
     "tsutils": "^3.21.0",
     "typescript": "^4.9.5",
-    "webpack-cli": "^5.0.1",
+    "webpack-cli": "^5.1.4",
     "webpack-dev-server": "^4.11.1",
     "ws": "^8.12.1"
   }

src/Spark.Admin.WebApp/src/Accounts/Types.fs

@@ -17,7 +17,7 @@ type Tab =
   | PositionManagementTab
   | MarginTab
 
-let venues = [| Venue.BINANCE; Venue.BinanceUs; Venue.BinanceUsdsFutures; Venue.CBOE; Venue.COINBASE; Venue.DERIBIT; Venue.GLBX; Venue.ICE; Venue.TAIFEX; Venue.TWSE; Venue.SIM; |]
+let venues = [| Venue.BINANCE; Venue.BinanceUs; Venue.BinanceUsdsFutures; Venue.CBOE; Venue.COINBASE; Venue.DERIBIT; Venue.OKX; Venue.GLBX; Venue.ICE; Venue.TAIFEX; Venue.TWSE; Venue.SIM; |]
 
 let securityTypes = [| SecurityType.SPOT; SecurityType.FUTURE; SecurityType.OPTION; SecurityType.SPREAD; |]
 
@@ -79,6 +79,11 @@ type TwseExchangeField =
   | Origin of CustomerOrFirm
   | ClearingAccount of string
 
+type OkxExchangeField =
+  | CTI of CTICode
+  | Origin of CustomerOrFirm
+  | ClearingAccount of string
+
 type SIMExchangeField =
   | CTI of CTICode
   | Origin of CustomerOrFirm

src/Spark.Admin.WebApp/src/Spark.Admin.WebApp.fsproj

@@ -1,7 +1,7 @@
 <Project Sdk="Microsoft.NET.Sdk">
 
   <PropertyGroup>
-    <TargetFramework>net7.0</TargetFramework>
+    <TargetFramework>net8.0</TargetFramework>
     <Version>1.0.0</Version>
     <AssemblyVersion>1.0.0</AssemblyVersion>
   </PropertyGroup>

src/app_sparkd/CMakeLists.txt

@@ -0,0 +1,186 @@
+cmake_minimum_required(VERSION 3.14)
+project(sparkd)
+
+include(FetchContent)
+include(${CMAKE_SOURCE_DIR}/cmake/safe_dep_glob.cmake)
+include(${CMAKE_SOURCE_DIR}/cmake/ProtobufGenerateCpp.cmake)
+include(${CMAKE_SOURCE_DIR}/cmake/InstallBasic.cmake)
+include(GNUInstallDirs)
+
+if (NOT DEFINED PROJECT_VERSION)
+  set(PROJECT_VERSION 1.0.0)
+endif()
+
+# dependencies
+file(
+  GLOB_RECURSE ${PROJECT_NAME}_PROTO_SOURCES
+  RELATIVE ${CMAKE_CURRENT_SOURCE_DIR}
+  *.proto
+)
+file(
+  GLOB_RECURSE ${PROJECT_NAME}_SOURCES
+  RELATIVE ${CMAKE_CURRENT_SOURCE_DIR}
+  *.cpp
+)
+update_deps_file("${${PROJECT_NAME}_PROTO_SOURCES};${${PROJECT_NAME}_SOURCES}")
+
+protobuf_generate_cpp(
+  CXX_SOURCES ${PROJECT_NAME}_PROTO_CXX_SOURCES
+  CXX_HEADERS ${PROJECT_NAME}_PROTO_CXX_HEADERS
+  CXX_OUT_DIR ${CMAKE_CURRENT_BINARY_DIR}/include
+  PROTO_INCLUDE_DIRS
+    ${EXTERNALS_INCLUDE}
+    ${CMAKE_SOURCE_DIR}/src/lib_spark
+    ${CMAKE_CURRENT_SOURCE_DIR}
+  PROTO_SOURCES
+    ${${PROJECT_NAME}_PROTO_SOURCES}
+)
+add_executable(sparkd ${${PROJECT_NAME}_SOURCES} ${${PROJECT_NAME}_PROTO_CXX_SOURCES})
+
+# include paths
+target_include_directories(
+  sparkd
+  PUBLIC
+    $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}>
+    $<BUILD_INTERFACE:${CMAKE_CURRENT_BINARY_DIR}>/include
+    $<BUILD_INTERFACE:${emumba-deribit-connector-library_INCLUDE_DIRS}>
+)
+
+find_package(PkgConfig REQUIRED)
+pkg_check_modules(emumba-deribit-connector-library REQUIRED emumba-deribit-connector)
+
+# This dir contains rapidjsonConfig.cmake
+list(APPEND CMAKE_PREFIX_PATH ${CMAKE_CURRENT_SOURCE_DIR}/cmake_modules)
+find_package(rapidjson REQUIRED)
+target_include_directories(
+  ${PROJECT_NAME} SYSTEM
+  PUBLIC
+    $<BUILD_INTERFACE:${rapidjson_INCLUDE_DIR}> 
+)
+add_dependencies(${PROJECT_NAME} rapidjson_project)
+
+# databento
+FetchContent_Declare(
+  databento
+  GIT_REPOSITORY https://github.com/databento/databento-cpp
+  GIT_TAG v0.31.0
+)
+FetchContent_MakeAvailable(databento)
+
+# deribit-demo log level
+if(NOT DEFINED DERIBIT_DEMO_LOG_LEVEL)
+  add_definitions(-DSPDLOG_ACTIVE_LEVEL=0)
+  message("-- Building of DERIBIT_DEMO_LOG_LEVEL=TRACE")
+elseif(DERIBIT_DEMO_LOG_LEVEL STREQUAL "TRACE")
+  add_definitions(-DSPDLOG_ACTIVE_LEVEL=0)
+  message("-- Building of DERIBIT_DEMO_LOG_LEVEL=TRACE")
+elseif(DERIBIT_DEMO_LOG_LEVEL STREQUAL "DEBUG")
+  add_definitions(-DSPDLOG_ACTIVE_LEVEL=1)
+  message("-- Building of DERIBIT_DEMO_LOG_LEVEL=DEBUG")
+elseif(DERIBIT_DEMO_LOG_LEVEL STREQUAL "INFO")
+  add_definitions(-DSPDLOG_ACTIVE_LEVEL=2)
+  message("-- Building of DERIBIT_DEMO_LOG_LEVEL=INFO")
+elseif(DERIBIT_DEMO_LOG_LEVEL STREQUAL "WARN")
+  add_definitions(-DSPDLOG_ACTIVE_LEVEL=3)
+  message("-- Building of DERIBIT_DEMO_LOG_LEVEL=WARN")
+elseif(DERIBIT_DEMO_LOG_LEVEL STREQUAL "ERROR")
+  add_definitions(-DSPDLOG_ACTIVE_LEVEL=4)
+  message("-- Building of DERIBIT_DEMO_LOG_LEVEL=ERROR")
+elseif(DERIBIT_DEMO_LOG_LEVEL STREQUAL "CRITICAL")
+  add_definitions(-DSPDLOG_ACTIVE_LEVEL=5)
+  message("-- Building of DERIBIT_DEMO_LOG_LEVEL=CRITICAL")
+elseif(DERIBIT_DEMO_LOG_LEVEL STREQUAL "OFF")
+  add_definitions(-DSPDLOG_ACTIVE_LEVEL=6)
+  message("-- Building of DERIBIT_DEMO_LOG_LEVEL=OFF")
+else()
+  add_definitions(-DSPDLOG_ACTIVE_LEVEL=0)
+  message("-- Building of DERIBIT_DEMO_LOG_LEVEL=TRACE")
+endif()
+
+target_link_directories(
+  ${PROJECT_NAME} PUBLIC
+  $<BUILD_INTERFACE:${CMAKE_CURRENT_BINARY_DIR}>
+  $<INSTALL_INTERFACE:${CMAKE_INSTALL_LIBDIR}>
+)
+
+# Set RPATH so sparkd knows where to find our shared libs (e.g., libspark).
+# The dirs are relative to sparkd's path, represented by "$ORIGIN".
+set_target_properties(
+  ${PROJECT_NAME} PROPERTIES
+  BUILD_RPATH "$ORIGIN/../lib_spark;$ORIGIN/../lib_python_md"
+  INSTALL_RPATH "$ORIGIN/../${CMAKE_INSTALL_LIBDIR}"
+)
+
+# Python interop (for python md providers)
+add_dependencies(${PROJECT_NAME}
+  python_md
+  python_md_module
+  python_md_providers
+  copy_python_md_providers
+)
+include(${CMAKE_SOURCE_DIR}/cmake/SymLink.cmake)
+
+sym_link_local_lib(python_md)
+sym_link_file(
+  SOURCE ${CMAKE_BINARY_DIR}/src/lib_python_md/python_md_module.cpython-310-x86_64-linux-gnu.so
+  DEST ${CMAKE_CURRENT_BINARY_DIR}/python_md_module.cpython-310-x86_64-linux-gnu.so
+  INSTALL_DEST ${CMAKE_INSTALL_BINDIR}/
+)
+get_target_property(providers python_md_providers INTERFACE_PYTHON_SCRIPTS_DIR)
+sym_link_dir(
+  TARGET_NAME link_python_md_providers
+  SOURCE ${providers}
+  DEST ${CMAKE_CURRENT_BINARY_DIR}/python_md_providers
+)
+
+target_link_libraries(
+  sparkd
+  spark
+  binance_execution
+  okx_execution
+  python_md
+  python_md_module
+  databento::databento
+  ${emumba-deribit-connector-library_LIBRARIES}
+  tbb
+  quickfix
+  benchmark
+  Common++
+  Packet++
+  Pcap++
+  lttng-ust
+  -lpcap
+  -static-libgcc
+  -static-libstdc++
+  -pthread)
+
+# Onixs
+target_link_libraries(
+  ${PROJECT_NAME}
+  onixs-cmemdh
+  OnixS.IceImpactMulticastPriceFeedHandlerCpp
+  # gcc doesn't resolve onixs-fix-engine and ACE in runtime path unless we specify
+  # --no-as-needed. These are onixs deps that aren't called directly by our code.
+  # Note: this isn't an issue for clang
+  -Wl,--no-as-needed
+  onixs-fix-engine
+  ACE
+)
+
+# Catch2
+find_package(Catch2 CONFIG REQUIRED)
+target_link_libraries(${PROJECT_NAME} Catch2::Catch2)
+
+install_basic()
+install(
+  FILES
+    "${EXTERNALS_LIB}/libonixs-cmemdh.so"
+    "${EXTERNALS_LIB}/libonixs-cmemdh.so.4"
+    "${EXTERNALS_LIB}/libonixs-cmemdh.so.4.8.0"
+    "${EXTERNALS_LIB}/libOnixS.IceImpactMulticastPriceFeedHandlerCpp.so.3.16.0.0"
+    "${EXTERNALS_LIB}/libonixs-fix-engine.so"
+    "${EXTERNALS_LIB}/libonixs-fix-engine.so.4"
+    "${EXTERNALS_LIB}/libonixs-fix-engine.so.4.10.1"
+    "${EXTERNALS_LIB}/libACE-5.7.5.so"
+  DESTINATION ${CMAKE_INSTALL_LIBDIR}
+)

src/app_sparkd/Config.cmake.in

@@ -0,0 +1,5 @@
+@PACKAGE_INIT@
+
+include("${CMAKE_CURRENT_LIST_DIR}/@PROJECT_NAME@Targets.cmake")
+
+check_required_components(@PROJECT_NAME@)

src/app_sparkd/Dockerfile

@@ -0,0 +1,19 @@
+FROM bluetradingsystems/linux_dotnet7
+
+ARG SOURCE_DIR="install/sparkd-release-gcc"
+ARG CONF_FILE="src/app_sparkd/spark.conf"
+
+USER "root"
+RUN apt-get update \
+  && apt-get install -y liblttng-ust-dev
+
+RUN apt-get update \
+  && apt-get install -y libpcap0.8-dev
+
+RUN apt-get update \
+  && apt-get install -y python3.10-dev
+
+USER "bts"
+COPY --chown=bts:bts "${SOURCE_DIR}" "."
+COPY --chown=bts:bts "${CONF_FILE}" ".local/share/bts/sparkd/conf/spark.conf"
+WORKDIR "/home/bts/bin"

src/app_sparkd/configurations/alphaworx_prod/.env

@@ -0,0 +1,4 @@
+SPARKD__CONFIG_DIR=src/app_sparkd/configurations/alphaworx_prod
+SPARKD__SHM_SIZE=8gb
+SPARKD__MEMORY=8gb
+SPARKD__RUNTIME_CONF_DIR=/home/ubuntu/8000/sparkdata/conf/

src/app_sparkd/configurations/alphaworx_prod/README.txt

@@ -0,0 +1,6 @@
+* Connects to Binance and Okx PRODUCTION sites for market data and execution
+* Api keys NOT included
+* Configured for Alphaworx
+
+To retrieve api keys, run "python init_env.py". It should dump them to a file
+called  ".env.local".

src/app_sparkd/configurations/alphaworx_prod/binance_1.json

@@ -0,0 +1,9 @@
+{
+  "api_key": "${BINANCE_API_KEY}",
+  "secret_key": "${BINANCE_SECRET_KEY}",
+  "account": "PROD",
+  "rest_endpoint": "https://api.binance.com",
+  "ws_endpoint": "wss://ws-api.binance.com:443/ws-api/v3",
+  "ws_stream_endpoint": "wss://stream.binance.com:9443",
+  "use_cross_margin_api": true
+}

src/app_sparkd/configurations/alphaworx_prod/binance_usds_futures_1.json

@@ -0,0 +1,9 @@
+{
+  "api_key": "${BINANCE_API_KEY}",
+  "secret_key": "${BINANCE_SECRET_KEY}",
+  "account": "PROD",
+  "rest_endpoint": "https://fapi.binance.com",
+  "ws_endpoint": "wss://ws-fapi.binance.com/ws-fapi/v1",
+  "ws_stream_endpoint": "wss://fstream.binance.com",
+  "use_cross_margin_api": false
+}

src/app_sparkd/configurations/alphaworx_prod/init_env.py

@@ -0,0 +1,88 @@
+import botocore
+import json
+import os
+
+from boto3.session import Session
+from botocore.exceptions import ClientError
+from fileinput import FileInput
+
+
+def update_env_file(path, env_vars: dict):
+    """Update or add vars in a .env file in place."""
+    if not os.path.exists(path):
+        with open(path, "w") as file:
+            for k, v in env_vars.items():
+                file.write(f"{k}={v}\n")
+        return
+
+    with FileInput(path, inplace=True) as file:
+        for line in file:
+            if not line.startswith("#"):
+                stripped: str = line.lstrip()
+                key_len = stripped.find("=")
+                if key_len >= 0:
+                    k = stripped[0:key_len]
+                    if k in env_vars:
+                        print(f"{k}={env_vars[k]}")
+                        del env_vars[k]
+                        continue
+            print(line, end="")
+
+    if len(env_vars) > 0:
+        with open(path, "a") as file:
+            for k, v in env_vars.items():
+                file.write(f"{k}={v}\n")
+
+
+def get_secret():
+    secret_name = "alphaworx-binance-api-keys"
+    region_name = "ap-northeast-1"
+    profile_name = "bts-ap-northeast-1"
+    session = Session(profile_name=profile_name)
+    client = session.client(
+        service_name="secretsmanager",
+        region_name=region_name,
+    )
+
+    def on_token_error(e: Exception):
+        print(f"""
+Failed to get secret value. {e}
+Verify you are logged in to aws, then try again.
+
+  aws sso login --profile {profile_name}
+
+If this error persists, verify you have access to these api keys.
+For more help or information, talk to John.
+""")
+
+    try:
+        get_secret_value_response = client.get_secret_value(
+            SecretId=secret_name)
+    except ClientError as e:
+        print(f"Failed to get secret value.\n{e}")
+        raise e
+    except botocore.exceptions.SSOTokenLoadError as tl_err:
+        on_token_error(tl_err)
+        return None
+    except botocore.exceptions.TokenRetrievalError as tr_err:
+        on_token_error(tr_err)
+        return None
+    return get_secret_value_response["SecretString"]
+
+
+def _run():
+    this_dir = os.path.dirname(__file__)
+    secret_str = get_secret()
+    if secret_str is None:
+        exit(1)
+    secret = json.loads(secret_str)
+    env_vars = {}
+    if "api-key" in secret:
+        env_vars["BINANCE_API_KEY"] = secret["api-key"]
+    if "secret-key" in secret:
+        env_vars["BINANCE_SECRET_KEY"] = secret["secret-key"]
+    update_env_file(f"{this_dir}/.env.local", env_vars)
+
+
+if __name__ == "__main__":
+    _run()

src/app_sparkd/configurations/alphaworx_prod/spark.conf

@@ -0,0 +1,124 @@
+QuickbusCfg {
+  value {
+    node {
+      reconnect_period_secs: 1
+      hb_period_msecs: 15000
+      max_missed_hbs: 5
+      parent_host: ""
+      parent_port: 0
+      my_port: 0
+    }
+    client {
+      reconnect_period_secs: 1
+      hb_period_msecs: 15000
+      max_missed_hbs: 5
+      parent_host: @SystemConfig.config_ip
+      parent_port: @Ports.qbus_port
+      my_port: 0
+    }
+  }
+}
+
+JetFIX {
+  value {
+    connection {
+      name: "binance_1"
+      venue: VENUE_BINANCE
+      max_order_msgs_per_window: 300
+      window_duration_msecs: 1000
+    }
+    connection {
+      name: "binance_usds_futures_1"
+      venue: VENUE_BINANCE_USDS_FUTURES
+      max_order_msgs_per_window: 300
+      window_duration_msecs: 1000
+    }
+    connection {
+      name: "okx_1"
+      venue: VENUE_OKX
+      max_order_msgs_per_window: 300
+      window_duration_msecs: 1000
+    }
+  }
+}
+
+MarketDataConfig {
+  value {
+    binance {
+      num_depth_levels: 5
+      url_rest_base: "https://api.binance.com"
+      url_ws_base: "wss://stream.binance.com:9443"
+      security_criteria {
+        clauses {
+          security_types: SECURITY_TYPE_SPOT
+          symbols: @SymbolLists.list[Binance].symbol
+          name_regexps: "USDC|USDT$"
+        }
+      }
+    }
+    binance_usds_futures {
+      num_depth_levels: 5
+      url_rest_base: "https://fapi.binance.com"
+      url_ws_base: "wss://fstream.binance.com"
+      security_criteria {
+        clauses {
+          security_types: SECURITY_TYPE_FUTURE
+          symbols: @SymbolLists.list[BinanceUsdsFutures].symbol
+        }
+      }
+    }
+    okx {
+      num_depth_levels: 5
+      url_rest_base: "https://www.okx.com"
+      url_ws_base: "wss://ws.okx.com:8443"
+      security_criteria {
+        clauses {
+          security_types: SECURITY_TYPE_SPOT
+          symbols: @SymbolLists.list[OkxSpots].symbol
+        }
+        clauses {
+          security_types: SECURITY_TYPE_FUTURE
+          symbols: @SymbolLists.list[OkxFutures].symbol
+        }
+      }
+      is_application_ping_pong_enabled: true
+      application_ping_interval_ms: 20000
+      application_pong_timeout_ms: 10000
+      is_protocol_ping_pong_enabled: true
+      protocol_ping_interval_ms: 60000
+      protocol_pong_timeout_ms: 30000
+    }
+  }
+}
+
+NotSimExecution {
+  value {
+    entries {
+      venue: VENUE_BINANCE
+    }
+    entries {
+      venue: VENUE_BINANCE_USDS_FUTURES
+    }
+    entries {
+      venue: VENUE_OKX
+    }
+  }
+}
+
+OkxExeCfg {
+  value {
+    account: "PROD"
+    ops_id: "op_id_bts"
+    secret_key: "240038D7A28D7F9461247CFCB8E43710"
+    api_key: "dc188532-c74c-4d40-affe-3c6bf4d0ca9a"
+    passphrase: ":,7i%tA%ivfj%SCbX?Ut"
+    ws_url: "wss://wsaws.okx.com:8443"
+    rest_url: "https://www.okx.com"
+    ws_ping_timeout_seconds: 10 
+    ws_ping_interval_seconds: 25
+    ws_reconnect_interval_seconds: 5
+    ws_reconnect_max_count: 5
+	  simulated_trading: false
+    tdmode: "cross"
+  }
+}

src/app_sparkd/configurations/alphaworx_prod/symbols.conf

@@ -0,0 +1,124 @@
+SymbolLists {
+  value {
+    list {
+      key: "Binance"
+      symbol: "ADA"
+      symbol: "APT"
+      symbol: "AVAX"
+      symbol: "BCH"
+      symbol: "BNB"
+      symbol: "BTC"
+      symbol: "DOGE"
+      symbol: "DOT"
+      symbol: "ETH"
+      symbol: "HBAR"
+      symbol: "LINK"
+      symbol: "LTC"
+      symbol: "OM"
+      symbol: "PAXG"
+      symbol: "PEPE"
+      symbol: "S"
+      symbol: "SHIB"
+      symbol: "SOL"
+      symbol: "SUI"
+      symbol: "TAO"
+      symbol: "TRUMP"
+      symbol: "USDC"
+      symbol: "XRP"
+    }
+    list {
+      key: "BinanceUsdsFutures"
+      symbol: "1000PEPE"
+      symbol: "1000SHIB"
+      symbol: "ADA"
+      symbol: "APT"
+      symbol: "AVAX"
+      symbol: "BCH"
+      symbol: "BNB"
+      symbol: "BTC"
+      symbol: "DOGE"
+      symbol: "DOT"
+      symbol: "ETH"
+      symbol: "HBAR"
+      symbol: "LINK"
+      symbol: "LTC"
+      symbol: "OM"
+      symbol: "S"
+      symbol: "SOL"
+      symbol: "SUI"
+      symbol: "TAO"
+      symbol: "TRUMP"
+      symbol: "USDC"
+      symbol: "XRP"
+    }
+    list {
+      key: "OkxSpots"
+      symbol: "ADA"
+      symbol: "APT"
+      symbol: "AVAX"
+      symbol: "BCH"
+      symbol: "BETH"
+      symbol: "BNB"
+      symbol: "BTC"
+      symbol: "DOGE"
+      symbol: "DOT"
+      symbol: "ETH"
+      symbol: "HBAR"
+      symbol: "LINK"
+      symbol: "LTC"
+      symbol: "OM"
+      symbol: "PEPE"
+      symbol: "S"
+      symbol: "SHIB"
+      symbol: "SOL"
+      symbol: "SUI"
+      symbol: "TRUMP"
+      symbol: "USDC"
+      symbol: "USDT"
+      symbol: "XRP"
+    }
+    list {
+      key: "OkxFutures"
+      symbol: "ADA"
+      symbol: "APT"
+      symbol: "AVAX"
+      symbol: "BCH"
+      symbol: "BNB"
+      symbol: "BTC"
+      symbol: "DOGE"
+      symbol: "DOT"
+      symbol: "ETH"
+      symbol: "HBAR"
+      symbol: "LINK"
+      symbol: "LTC"
+      symbol: "OM"
+      symbol: "PEPE"
+      symbol: "S"
+      symbol: "SHIB"
+      symbol: "SOL"
+      symbol: "SUI"
+      symbol: "TAO"
+      symbol: "TRUMP"
+      symbol: "USDC"
+      symbol: "XRP"
+    }
+  }
+}
+
+Symbols {
+  value {
+    venues {
+      name: "VENUE_BINANCE"
+      spot: @SymbolLists.list[Binance].symbol
+    }
+    venues {
+      name: "VENUE_BINANCE_USDS_FUTURES"
+      future: @SymbolLists.list[BinanceUsdsFutures].symbol
+    }
+    venues {
+      name: "VENUE_OKX"
+      spot: @SymbolLists.list[OkxSpots].symbol
+      future: @SymbolLists.list[OkxFutures].symbol
+    }
+  }
+}

src/app_sparkd/configurations/alphaworx_zen_hk_live/.env

@@ -0,0 +1,5 @@
+SPARKD__CONFIG_DIR=src/app_sparkd/configurations/alphaworx_zen_hk_live
+SPARKD__SHM_SIZE=8gb
+SPARKD__MEMORY=8gb
+SPARKD__IMAGE=sparkd_symbols
+SPARKD__RUNTIME_CONF_DIR=/home/bts/8000/sparkdata/conf/