Fix: Taifex Time Conversion

FIX: Taifex Crash Fixed, Added Diag Logs
ADD: TBBO & Symbol wise Naming in Recorder
2025-03-26 18:49:17 +00:00 · 2025-03-26 18:48:27 +00:00 · 2025-03-26 18:25:28 +00:00 · 2025-03-26 18:20:39 +00:00 · 2025-03-26 17:59:59 +00:00 · 2025-03-06 05:56:01 +00:00
2418 changed files with 39942 additions and 63455 deletions
@@ -9,7 +9,7 @@
      ]
    },
    "fable": {
-      "version": "4.24.0",
+      "version": "4.0.6",
      "commands": [
        "fable"
      ]
@@ -27,4 +27,4 @@
      ]
    }
  }
-}
+}
@@ -65,7 +65,7 @@ RUN \
   && apt-get install -y cmake

 # Dotnet
-RUN apt-get update && apt-get install -y dotnet8
+RUN apt-get update && apt-get install -y dotnet7

 # Node
 ARG node_major=18
@@ -90,12 +90,6 @@ RUN apt-get update && apt-get install -y libpcap0.8-dev
 # Databento (deps)
 RUN apt-get update && apt-get -y install libzstd-dev

-# Docker in Docker
-RUN apt-get update && apt-get install -y \
-    docker.io \
-    docker-compose \
-    docker-compose-v2
-
 # Set up default executables (update-alternatives)
 RUN \
   update-alternatives --install /usr/bin/clang clang /usr/bin/clang-$clang_major 100; \
@@ -1,10 +1,15 @@
 {
  "dockerFile": "Dockerfile",
+  // Creates local dir to mount to HOME in docker 
+  "initializeCommand": "mkdir -p ${localEnv:HOME}/.bts/devcontainer-mounts/${devcontainerId}",
  "hostRequirements": {
-    "memory": "8gb",
-    "storage": "10gb"
+    "memory": "4gb",
+    "storage": "6gb"
  },
+  "postCreateCommand": "/home/bts/spark/docker/sparkd/dev/devcontainer_post_create.sh",
  "containerUser": "bts",
+  "workspaceMount": "source=${localWorkspaceFolder},target=/home/bts/spark,type=bind",
+  "workspaceFolder": "/home/bts/spark",
  "updateRemoteUserUID": true,
  "shutdownAction": "none",
  // Note: If running multiple containers on the same host, each must use a
@@ -12,20 +17,34 @@
  // example, if 8000-8003 is taken, go with 9000-9003, then 10000-10003, etc.
  "runArgs": [
    "--shm-size",
-    "8gb"
+    "2gb",
+    "-p",
+    "8000:8000",
+    "-p",
+    "8001:8001",
+    "-p",
+    "8002:8002",
+    "-p",
+    "8003:8003",
+    "-p",
+    "12345:12345"
  ],
  "mounts": [
-    "source=/var/run/docker.sock,target=/var/run/docker.sock,type=bind"
+    "source=${localEnv:HOME}/.bts/devcontainer-mounts/${devcontainerId},target=/home/bts,type=bind"
  ],
  "customizations": {
    "vscode": {
+      "settings": {
+        "clangd.arguments": [
+          "--compile-commands-dir=${workspaceFolder}/build/clang/debug"
+        ]
+      },
      "extensions": [
        "eamodio.gitlens",
        "llvm-vs-code-extensions.vscode-clangd",
        "ms-vscode.cmake-tools",
-        "ms-vscode.cpptools",
-        "xaver.clang-format"
+        "ms-vscode.cpptools"
      ]
    }
  }
-}
+}
@@ -1,48 +0,0 @@
-# This file defines the global default configuration for docker deployments.
-# It is tracked in source control. For temporary modifications, please create
-# a separate file called ".env.local" (it's in .gitignore, so won't be tracked).
-# If a variable is defined in both files, the value in .env.local will be used.
-#
-# About the deployment procedure:
-#   We use Docker Compose to build images and launch containers for various services
-#   that make up our software suite:
-#     - sparkd
-#     - TODO: pubsub
-#     - TODO: md-publisher
-#     - TODO: webapp (DSQA on the web)
-#     - TODO: api-server (websocket based)
-#   From inside a devcontainer, you can launch all services in separate containers
-#   on your machine by running:
-#
-#     tools/deploy.sh
-#
-#   This script builds release versions for each service, packaged (generally) in
-#   self-contained directories. Next, it runs a few "docker compose" commands to
-#   build docker images loaded with the self-contained binaries from the previous
-#   step, and start containers with those images to run each service. To view the
-#   commands that deploy.sh runs internally (without actually running them), run:
-#
-#     tools/deploy.sh --dry-run
-#
-#   For additional information/help, run:
-#
-#     tools/deploy.sh --help
-#
-
-### Sparkd ###
-# The internal image for sparkd to use. For more info, view:
-#   src/server/sparkd/Dockerfile
-SPARKD__IMAGE=sparkd
-# The config dir for this service to use
-SPARKD__CONFIG_DIR=src/server/sparkd/configurations/dev
-# Mount a specific volume or dir to "~/sparkdata/data" inside the docker.
-# Defaults to a volume named "sparkd_<config_port>" (e.g., sparkd_8000).
-# SPARKD__DATA_DIR=/home/bts/sparkdata/data
-SPARKD__SOURCE_DIR=install/sparkd-release-gcc
-SPARKD__RUNTIME_CONF_DIR=/home/ubuntu/8000/sparkdata/conf/
-SPARKD__CONFIG_PORT=8000
-SPARKD__QUICKBUS_PORT=8001
-SPARKD__SIM_FIX_PORT=8002
-SPARKD__SHM_SIZE=4gb
-
-# TODO: add config for other services (pubsub, md-publisher, webapp, api-server, etc.)
@@ -59,5 +59,3 @@ trace_data/
 .devcontainer/
 *.Generated.cs
 *.lst
-.env.local
-.secrets
@@ -1,6 +1,6 @@
 [submodule "externals"]
 	path = externals
-	url = https://bitbucket.org/bluetradesys/externals.git
+	url = git@bitbucket.org:bluetradesys/externals.git
 [submodule "vcpkg"]
 	path = vcpkg
 	url = https://github.com/microsoft/vcpkg.git
@@ -1,14 +1,6 @@
 {
  "version": "0.2.0",
  "configurations": [
-    {
-      "name": "Python Debugger: Current File with Arguments",
-      "type": "debugpy",
-      "request": "launch",
-      "program": "${file}",
-      "console": "integratedTerminal",
-      "args": "${command:pickArgs}"
-    },
    {
      "name": ".NET Core Attach",
      "type": "coreclr",
@@ -30,7 +22,7 @@
        "--config",
        "${workspaceFolder}/src/DataStreamSuite/DataStreamPublisher/Configurations/${input:dataStreamPublisherConfig}.json"
      ],
-      "cwd": "build/Debug/bin/net8.0",
+      "cwd": "build/Debug/bin/net7.0",
      "stopAtEntry": false,
      "console": "integratedTerminal"
    },
@@ -39,39 +31,7 @@
      "type": "cppdbg",
      "request": "launch",
      "program": "/home/bts/spark/build/sparkd-debug-clang/server/sparkd",
-      "args": [
-        "run"
-      ],
-      "stopAtEntry": false,
-      "cwd": "/home/bts/spark/build/sparkd-debug-clang/server",
-      "environment": [],
-      "externalConsole": false,
-      "MIMode": "gdb",
-    },
-    {
-      "name": "spreader tests2",
-      "type": "cppdbg",
-      "request": "launch",
-      "program": "/home/bts/spark/build/sparkd-debug-clang/server/sparkd",
-      "args": [
-        "test",
-        "[spreader2]"
-      ],
-      "stopAtEntry": false,
-      "cwd": "/home/bts/spark/build/sparkd-debug-clang/server",
-      "environment": [],
-      "externalConsole": false,
-      "MIMode": "gdb",
-    },
-    {
-      "name": "spreader tests",
-      "type": "cppdbg",
-      "request": "launch",
-      "program": "/home/bts/spark/build/sparkd-debug-clang/server/sparkd",
-      "args": [
-        "test",
-        "[spreader]"
-      ],
+      "args": ["run"],
      "stopAtEntry": false,
      "cwd": "/home/bts/spark/build/sparkd-debug-clang/server",
      "environment": [],
@@ -90,7 +50,7 @@
          "--config",
          "${workspaceFolder}/src/DataStreamSuite/DataStreamQueryAnalyzer/Configurations/DataStreamQueryAnalyzer.json"
        ],
-        "cwd": "build/Debug/bin/net8.0-windows",
+        "cwd": "build/Debug/bin/net7.0-windows",
        "env": {
          "PYTHON_EXE": "${workspaceFolder}/.venv/Scripts/python.exe"
        },
@@ -114,7 +74,7 @@
        "--config",
        "${workspaceFolder}/src/DataStreamSuite/PubSubServer/Configurations/${input:pubSubServerConfig}.json"
      ],
-      "cwd": "build/Debug/bin/net8.0",
+      "cwd": "build/Debug/bin/net7.0",
      "stopAtEntry": false,
      "console": "integratedTerminal"
    },
@@ -126,19 +86,7 @@
      "windows": {
        "preLaunchTask": "Build Spark (Debug)",
        "program": "Spark.exe",
-        "cwd": "build/Debug/bin/net8.0-windows",
-      },
-      "stopAtEntry": false,
-      "console": "integratedTerminal"
-    },
-    {
-      "name": "Spark (Skip Build)",
-      "type": "coreclr",
-      "request": "launch",
-      "program": "",
-      "windows": {
-        "program": "Spark.exe",
-        "cwd": "build/Debug/bin/net8.0-windows",
+        "cwd": "build/Debug/bin/net7.0-windows",
      },
      "stopAtEntry": false,
      "console": "integratedTerminal"
@@ -155,100 +103,27 @@
          "--sparkd",
          "${input:sparkConfigServer}"
        ],
-        "cwd": "build/Debug/bin/net8.0-windows",
-      },
-      "stopAtEntry": false,
-      "console": "integratedTerminal"
-    },
-    {
-      "name": "Spark Test",
-      "type": "coreclr",
-      "request": "launch",
-      "program": "",
-      "windows": {
-        "preLaunchTask": "Build Spark (Debug)",
-        "program": "Spark.Console.exe",
-        "cwd": "build/Debug/bin/net8.0",
-        "args": [
-          "test",
-          "${input:sparkTest}"
-        ]
-      },
-      "stopAtEntry": false,
-      "console": "integratedTerminal"
-    },
-    {
-      "name": "Spark Test (Skip Build)",
-      "type": "coreclr",
-      "request": "launch",
-      "program": "",
-      "windows": {
-        "program": "Spark.Console.exe",
-        "cwd": "build/Debug/bin/net8.0",
-        "args": [
-          "test",
-          "${input:sparkTest}"
-        ]
+        "cwd": "build/Debug/bin/net7.0-windows",
      },
      "stopAtEntry": false,
      "console": "integratedTerminal"
    },
    {
      "name": "Sparkd",
-      "type": "cppdbg",
+      "type": "coreclr",
      "request": "launch",
+      "preLaunchTask": "Build Sparkd (Debug | Clang)",
      "program": "",
      "linux": {
-        "program": "${workspaceFolder}/build/sparkd-debug-clang/src/app_sparkd/sparkd",
+        "program": "sparkd",
      },
      "args": [
        "run"
      ],
-      "cwd": "${workspaceFolder}/build/sparkd-debug-clang/src/app_sparkd",
+      "cwd": "build/sparkd-debug-clang/server",
      "stopAtEntry": false,
-      "externalConsole": false,
-      "MIMode": "gdb",
-      "setupCommands": [
-        {
-          "description": "Enable pretty-printing for gdb",
-          "text": "-enable-pretty-printing",
-          "ignoreFailures": true
-        },
-        {
-          "description": "Set Disassembly Flavor to Intel",
-          "text": "-gdb-set disassembly-flavor intel",
-          "ignoreFailures": true
-        }
-      ]
-    },
-    {
-      "name": "Sparkd (Skip Build)",
-      "type": "cppdbg",
-      "request": "launch",
-      "program": "",
-      "linux": {
-        "program": "${workspaceFolder}/build/sparkd-debug-clang/src/app_sparkd/sparkd",
-      },
-      "args": [
-        "run"
-      ],
-      "cwd": "${workspaceFolder}/build/sparkd-debug-clang/src/app_sparkd",
-      "stopAtEntry": false,
-      "externalConsole": false,
-      "MIMode": "gdb",
-      "setupCommands": [
-        {
-          "description": "Enable pretty-printing for gdb",
-          "text": "-enable-pretty-printing",
-          "ignoreFailures": true
-        },
-        {
-          "description": "Set Disassembly Flavor to Intel",
-          "text": "-gdb-set disassembly-flavor intel",
-          "ignoreFailures": true
-        }
-      ]
-    },
+      "console": "integratedTerminal"
+    }
  ],
  "inputs": [
    {
@@ -279,15 +154,5 @@
      "description": "Config Server URL",
      "default": "http://localhost:8000/config",
    },
-    {
-      "id": "sparkTest",
-      "type": "promptString",
-      "description": "Test to run",
-    },
-    {
-      "id": "sparkdTest",
-      "type": "promptString",
-      "description": "Test to run",
-    },
  ]
-}
+}
@@ -641,21 +641,6 @@
        "color": "terminal.ansiGreen",
      }
    },
-    {
-      "label": "Run Spark (Skip Build)",
-      "type": "process",
-      "command": "",
-      "windows": {
-        "command": "Spark.exe",
-        "options": {
-          "cwd": "${workspaceFolder}/build/${input:buildConfiguration}/bin/net7.0-windows"
-        },
-      },
-      "icon": {
-        "id": "run",
-        "color": "terminal.ansiGreen",
-      }
-    },
    {
      "label": "Run Spark Config Query",
      "dependsOn": "Build Spark",
@@ -699,24 +684,6 @@
        "color": "terminal.ansiGreen",
      },
    },
-    {
-      "label": "Run Sparkd (Skip Build)",
-      "type": "process",
-      "command": "",
-      "linux": {
-        "command": "sparkd",
-      },
-      "args": [
-        "run"
-      ],
-      "options": {
-        "cwd": "${workspaceFolder}/build/${input:sparkdCmakePreset}/server"
-      },
-      "icon": {
-        "id": "run",
-        "color": "terminal.ansiGreen",
-      },
-    },
    {
      "label": "Run Sparkd Config Query",
      "dependsOn": "Build Sparkd",
@@ -738,65 +705,6 @@
        "color": "terminal.ansiGreen",
      },
    },
-    {
-      "label": "Test Spark (Debug)",
-      "dependsOn": "Build Spark (Debug)",
-      "type": "process",
-      "icon": {
-        "id": "coverage",
-        "color": "terminal.ansiYellow",
-      },
-      "command": "",
-      "windows": {
-        "command": "Spark.Console.exe",
-        "args": [
-          "test",
-          "${input:sparkTest}"
-        ],
-        "options": {
-          "cwd": "${workspaceFolder}/build/Debug/bin/net7.0",
-        }
-      },
-    },
-    {
-      "label": "Test Sparkd",
-      "dependsOn": "Build Sparkd",
-      "type": "process",
-      "command": "",
-      "linux": {
-        "command": "sparkd",
-      },
-      "args": [
-        "test",
-        "${input:sparkdTest}",
-      ],
-      "options": {
-        "cwd": "${workspaceFolder}/build/${input:sparkdCmakePreset}/server"
-      },
-      "icon": {
-        "id": "coverage",
-        "color": "terminal.ansiYellow",
-      },
-    },
-    {
-      "label": "Test Sparkd (Skip Build)",
-      "type": "process",
-      "command": "",
-      "linux": {
-        "command": "sparkd",
-      },
-      "args": [
-        "test",
-        "${input:sparkdTest}",
-      ],
-      "options": {
-        "cwd": "${workspaceFolder}/build/${input:sparkdCmakePreset}/server"
-      },
-      "icon": {
-        "id": "coverage",
-        "color": "terminal.ansiYellow",
-      },
-    },
    {
      "label": "View DataStreamQueryAnalyzer Logs (Last Run)",
      "type": "process",
@@ -915,16 +823,6 @@
      "id": "sparkdConfigQuery",
      "type": "promptString",
      "description": "Config Type(s)"
-    },
-    {
-      "id": "sparkTest",
-      "type": "promptString",
-      "description": "Test to run",
-    },
-    {
-      "id": "sparkdTest",
-      "type": "promptString",
-      "description": "Test to run",
-    },
+    }
  ]
 }
@@ -1,10 +1,8 @@
 cmake_minimum_required(VERSION 3.12)
 project(spark)
 include(GNUInstallDirs)
-include(${CMAKE_SOURCE_DIR}/cmake/GitVersion.cmake)
 set(CMAKE_CXX_STANDARD 20)

-get_version_from_git()
 if(WIN32)
  set(platform "windows")
 else()
@@ -22,9 +20,9 @@ set(
 # can run simultaneously without fighting over the same shm files (e.g., sparkd,
 # api server, etc.).
 set(SHARED_MEMORY_PREFIX "bts" CACHE STRING "Prefix to use when creating shared memory files")
-set(EXTERNALS_BIN "${EXTERNALS_ROOT}/bin")
-set(EXTERNALS_INCLUDE "${EXTERNALS_ROOT}/include")
-set(EXTERNALS_LIB "${EXTERNALS_ROOT}/lib/$<CONFIG>")
+set(SPARK_BIN "${EXTERNALS_ROOT}/bin")
+set(SPARK_INCLUDE "${EXTERNALS_ROOT}/include")
+set(SPARK_LIB "${EXTERNALS_ROOT}/lib/$<CONFIG>")
 set(BUILD_DOCS OFF CACHE BOOL "Build documentation")
 set(BUILD_SDK OFF CACHE BOOL "Build the sdk")
 set(EXPORT_MD_CONSUMER OFF CACHE BOOL "Export (extern) C functions for MD consumer interop")
@@ -36,8 +34,8 @@ else()
  set(BUILD_ADMIN_WEB_APP ON CACHE BOOL "Build the Admin WebApp")
 endif()

-include_directories(BEFORE SYSTEM ${EXTERNALS_INCLUDE} ${EXTERNALS_ROOT}/include/zlib)
-link_directories(${EXTERNALS_LIB})
+include_directories(BEFORE SYSTEM ${SPARK_INCLUDE} ${EXTERNALS_ROOT}/include/zlib)
+link_directories(${SPARK_LIB})

 add_compile_definitions(SHARED_MEMORY_PREFIX="${SHARED_MEMORY_PREFIX}")
 add_compile_definitions(APP_DATA_DIR="${APP_DATA_DIR}")
@@ -47,8 +45,12 @@ endif()
 add_compile_definitions(EXPORT_MD_CONSUMER="${EXPORT_MD_CONSUMER}")
 if(WIN32)
  # subdirs
-  add_subdirectory(src/lib_spark)
+  add_subdirectory(src/lib)
 else()
+  # make sparkd look for libspark.so in its directory
+  set(CMAKE_BUILD_WITH_INSTALL_RPATH TRUE)
+  set(CMAKE_INSTALL_RPATH ".")
+
  if(BUILD_SPARKD)
    # Note: All macros that begin w/ 'CCAPI' are ccapi's macros (defined and/or
    # read by their code). We override some below.
@@ -86,15 +88,9 @@ else()
    add_compile_definitions(BINANCE_US_PRICE_STATS_PATH="/api/v3/ticker/24hr")

    # ** Binance USDS Futures **
-    add_compile_definitions(CCAPI_BINANCE_USDS_FUTURES_URL_REST_BASE="https://fapi.binance.com")
-    add_compile_definitions(BINANCE_USDS_FUTURES_INSTRUMENTS_PATH="/fapi/v1/exchangeInfo")
-    add_compile_definitions(BINANCE_USDS_FUTURES_PRICE_STATS_PATH="/fapi/v1/ticker/24hr")
-    add_compile_definitions(BINANCE_USDS_FUTURES_USER_DATA_PATH="/api/v3/userDataStream")
-
-    # ** Binance Coin Futures **
-    add_compile_definitions(CCAPI_BINANCE_COIN_FUTURES_URL_REST_BASE="https://dapi.binance.com")
-    add_compile_definitions(BINANCE_COIN_FUTURES_INSTRUMENTS_PATH="/dapi/v1/exchangeInfo")
-    add_compile_definitions(BINANCE_COIN_FUTURES_PRICE_STATS_PATH="/dapi/v1/ticker/24hr")
+    add_compile_definitions(CCAPI_BINANCE_USDS_FUTURES_URL_REST_BASE="https://dapi.binance.com")
+    add_compile_definitions(BINANCE_USDS_FUTURES_INSTRUMENTS_PATH="/dapi/v1/exchangeInfo")
+    add_compile_definitions(BINANCE_USDS_FUTURES_PRICE_STATS_PATH="/dapi/v1/ticker/24hr")

    # ** Deribit **
    add_compile_definitions(DERIBIT_INSTRUMENT_PATH="/api/v2/public/get_instrument")
@@ -114,14 +110,10 @@ else()

  # subdirs
  if (BUILD_SPARKD)
-    add_subdirectory(src/lib_onixs_fix)
-    add_subdirectory(src/lib_binance_execution)
-    add_subdirectory(src/lib_okx_execution)
-    add_subdirectory(src/lib_python_md)
-    add_subdirectory(src/app_sparkd)
+    add_subdirectory(src/server/sparkd)
    add_subdirectory(src/data_files)
  endif()
-  add_subdirectory(src/lib_spark)
+  add_subdirectory(src/lib)
  if (BUILD_ADMIN_WEB_APP)
    add_subdirectory(src/Spark.Admin.WebApp)
  endif()
@@ -131,4 +123,37 @@ else()
  if(BUILD_SDK)
    add_subdirectory(src/sdk)
  endif()
+
+  if(BUILD_SPARKD)
+    add_custom_command(
+      OUTPUT ${CMAKE_BINARY_DIR}/server
+      COMMAND mkdir -p ${CMAKE_BINARY_DIR}/server
+      COMMAND ln ARGS -sf ${CMAKE_BINARY_DIR}/src/server/sparkd/sparkd ${CMAKE_BINARY_DIR}/server
+      COMMAND ln ARGS -sf ${CMAKE_BINARY_DIR}/data_files ${CMAKE_BINARY_DIR}/server
+      COMMAND ln ARGS -sf ${CMAKE_BINARY_DIR}/src/lib/libspark.so ${CMAKE_BINARY_DIR}/server
+      COMMAND ln ARGS -sf ${EXTERNALS_ROOT}/lib/${CMAKE_BUILD_TYPE}/libonixs-fix-engine.so.4 ${CMAKE_BINARY_DIR}/server
+      
+      COMMAND
+        ln ARGS -sf
+        ${EXTERNALS_ROOT}/lib/Release/libOnixS.IceImpactMulticastPriceFeedHandlerCpp.so.3.16.0.0
+        ${CMAKE_BINARY_DIR}/server
+      COMMAND ln ARGS -sf ${EXTERNALS_ROOT}/lib/Release/libACE-5.7.5.so ${CMAKE_BINARY_DIR}/server
+      COMMAND mkdir ARGS -p ${CMAKE_BINARY_DIR}/server/plugins
+      COMMENT "Creating server runtime layout"
+      DEPENDS sparkd data_files_dir)
+
+    add_custom_target(server_layout ALL DEPENDS ${CMAKE_BINARY_DIR}/server)
+
+    install(
+      DIRECTORY "${CMAKE_BINARY_DIR}/server/plugins"
+      DESTINATION "${CMAKE_INSTALL_BINDIR}")
+    install(
+      FILES
+      "${SPARK_LIB}/libACE-5.7.5.so"
+      "${SPARK_LIB}/libOnixS.IceImpactMulticastPriceFeedHandlerCpp.so.3.16.0.0"
+      "${SPARK_LIB}/libonixs-fix-engine.so.4.10.1"
+      "${SPARK_LIB}/libonixs-fix-engine.so.4"
+      "${SPARK_LIB}/libonixs-fix-engine.so"
+      DESTINATION "${CMAKE_INSTALL_BINDIR}")
+  endif()
 endif()
@@ -26,13 +26,13 @@
        "CMAKE_CXX_FLAGS_DEBUG": "$env{SPARK_CMAKE_CXX_FLAGS} /MTd",
        "CMAKE_CXX_FLAGS_RELEASE": "$env{SPARK_CMAKE_CXX_FLAGS} /MT",
        "CMAKE_CXX_STANDARD": "20",
-        "CMAKE_INSTALL_PREFIX": "${sourceDir}/install/${presetName}",
        "CMAKE_SHARED_LINKER_FLAGS_DEBUG": "/DEBUG",
        "CMAKE_SHARED_LINKER_FLAGS_RELEASE": "/OPT:REF /OPT:ICF /LTCG",
        "CMAKE_TOOLCHAIN_FILE": "${sourceDir}/vcpkg/scripts/buildsystems/vcpkg.cmake",
        "EXTERNALS_ROOT": "${sourceDir}/externals/windows",
-        "CONFIG_DEFAULTS_PATH": "spark/config/profiles/default/*.conf",
-        "DOTNET_BINARY_DIR": "${sourceDir}/build/$<CONFIG>/bin/net8.0;${sourceDir}/build/$<CONFIG>/bin/net8.0-windows",
+        "CONFIG_DEFAULTS_PATH": "config/profiles/default/*.conf",
+        "DOTNET_BINARY_DIR": "${sourceDir}/build/$<CONFIG>/bin/net7.0",
+        "DOTNET_WINDOWS_BINARY_DIR": "${sourceDir}/build/$<CONFIG>/bin/net7.0-windows",
        "EXPORT_MD_CONSUMER": {
          "type": "BOOL",
          "value": "ON"
@@ -57,18 +57,12 @@
        "APP_DATA_DIR_OVERRIDE": "${dollar}{SPARKD_DATA_DIR}",
        "CMAKE_BUILD_WITH_INSTALL_RPATH": {
          "type": "BOOL",
-          "value": "OFF"
+          "value": "ON"
        },
        "CMAKE_CXX_STANDARD": "20",
        "CMAKE_INSTALL_RPATH": ".",
        "CMAKE_INSTALL_PREFIX": "${sourceDir}/install/${presetName}",
-        "CMAKE_PREFIX_PATH": "${sourceDir}/cmake/;${sourceDir}/.venv/lib/python3.10/site-packages/pybind11/share/cmake/pybind11",
        "CMAKE_EXE_LINKER_FLAGS": "-Wno-undef",
-        "CMAKE_TOOLCHAIN_FILE": "${sourceDir}/vcpkg/scripts/buildsystems/vcpkg.cmake",
-        "DATABENTO_USE_EXTERNAL_JSON": {
-          "type": "BOOL",
-          "value": "ON"
-        },
        "EXTERNALS_ROOT": "${sourceDir}/externals/linux",
        "BUILD_SDK": {
          "type": "BOOL",
@@ -93,7 +87,7 @@
          "value": "ON"
        },
        "CCAPI_LOG_LEVEL": "Info",
-        "CONFIG_DEFAULTS_PATH": "spark/config/profiles/md_dev/*.conf;spark/config/profiles/default/*.conf"
+        "CONFIG_DEFAULTS_PATH": "config/profiles/md_dev/*.conf;config/profiles/default/*.conf"
      }
    },

@@ -104,7 +98,7 @@
      "cacheVariables": {
        "CMAKE_BUILD_TYPE": "Release",
        "CCAPI_LOG_LEVEL": "Warn",
-        "CONFIG_DEFAULTS_PATH": "spark/config/profiles/md_prod/*.conf;spark/config/profiles/default/*.conf"
+        "CONFIG_DEFAULTS_PATH": "config/profiles/md_prod/*.conf;config/profiles/default/*.conf"
      }
    },

@@ -186,12 +180,8 @@
        "CMAKE_CXX_FLAGS_DEBUG": "-g -fPIC -Wall -pedantic -Wextra -Werror -Wno-missing-braces -Wno-ignored-qualifiers --std=c++20 -fdiagnostics-color -DDEBUG",
        "CMAKE_CXX_FLAGS_RELEASE": "-fPIC -Wall -pedantic -Wextra -Werror -Wno-missing-braces -Wno-ignored-qualifiers --std=c++20 -fdiagnostics-color -fno-omit-frame-pointer -DNDEBUG -O3",
        "CMAKE_INSTALL_PREFIX": "${sourceDir}/install/${presetName}",
-        "CMAKE_EXPORT_COMPILE_COMMANDS": {
-          "type": "BOOL",
-          "value": "ON"
-        },
-        "CONFIG_DEFAULTS_PATH": "spark/config/profiles/default/*.conf",
-        "DOTNET_BINARY_DIR": "${sourceDir}/build/$<CONFIG>/bin/net8.0",
+        "CONFIG_DEFAULTS_PATH": "config/profiles/default/*.conf",
+        "DOTNET_BINARY_DIR": "${sourceDir}/build/$<CONFIG>/bin/net7.0",
        "EXPORT_MD_CONSUMER": {
          "type": "BOOL",
          "value": "ON"
@@ -231,7 +221,7 @@
      "cacheVariables": {
        "APP_DATA_DIR": "${dollar}{APPDATA}/bts/DataStream",
        "APP_DATA_DIR_OVERRIDE": "${dollar}{DATA_STREAM_DATA_DIR}",
-        "CONFIG_DEFAULTS_PATH": "spark/config/profiles/default/*.conf;spark/config/profiles/data_stream/*.conf",
+        "CONFIG_DEFAULTS_PATH": "config/profiles/default/*.conf;config/profiles/data_stream/*.conf",
        "SHARED_MEMORY_PREFIX": "data_stream"
      }
    },
@@ -266,7 +256,8 @@
        "CMAKE_CXX_COMPILER": "clang++",
        "CMAKE_CXX_FLAGS_DEBUG": "-g -fPIC -Wall -pedantic -Wextra -Werror -Wno-missing-braces -Wno-ignored-qualifiers --std=c++20 -fdiagnostics-color -DDEBUG",
        "CMAKE_CXX_FLAGS_RELEASE": "-fPIC -Wall -pedantic -Wextra -Werror -Wno-missing-braces -Wno-ignored-qualifiers --std=c++20 -fdiagnostics-color -fno-omit-frame-pointer -DNDEBUG -O3",
-        "CONFIG_DEFAULTS_PATH": "spark/config/profiles/default/*.conf;spark/config/profiles/data_stream/*.conf",
+        "CMAKE_INSTALL_PREFIX": "${sourceDir}/install/${presetName}",
+        "CONFIG_DEFAULTS_PATH": "config/profiles/default/*.conf;config/profiles/data_stream/*.conf",
        "DOTNET_BINARY_DIR": "${sourceDir}/build/$<CONFIG>/bin/net7.0",
        "EXPORT_MD_CONSUMER": {
          "type": "BOOL",
@@ -311,7 +302,16 @@
        "lhs": "${hostSystemName}",
        "rhs": "Linux"
      },
-      "hidden": true
+      "hidden": true,
+      "targets": [
+        "data_files_dir",
+        "admin_web_app_bundle",
+        "install",
+        "libspark",
+        "rapidjson_project",
+        "server_layout",
+        "sparkd"
+      ]
    },

    {
@@ -1,7 +0,0 @@
-include_guard(GLOBAL)
-
-macro(export_to_parent)
-  foreach(var IN ITEMS ${ARGN})
-    set(${var} ${${var}} PARENT_SCOPE)
-  endforeach()
-endmacro()
@@ -1,48 +0,0 @@
-include_guard(GLOBAL)
-
-function(get_version_from_git)
-  find_package(Git QUIET)
-  if(NOT Git_FOUND)
-    message(WARNING "Git not found")
-    return()
-  endif()
-
-  execute_process(
-    COMMAND ${GIT_EXECUTABLE} describe --tags --always
-    WORKING_DIRECTORY ${CMAKE_SOURCE_DIR}
-    OUTPUT_VARIABLE GIT_TAG
-    OUTPUT_STRIP_TRAILING_WHITESPACE
-    RESULT_VARIABLE GIT_RESULT
-  )
-
-  if(NOT GIT_RESULT EQUAL 0)
-    message(WARNING "Failed to get git tag")
-    return()
-  endif()
-
-  execute_process(
-    COMMAND ${GIT_EXECUTABLE} rev-parse --short=7 HEAD
-    WORKING_DIRECTORY ${CMAKE_SOURCE_DIR}
-    OUTPUT_VARIABLE GIT_COMMIT_SHORT_HASH
-    OUTPUT_STRIP_TRAILING_WHITESPACE
-  )
-  set(PROJECT_VERSION_BUILD_METADATA ${GIT_COMMIT_SHORT_HASH})
-  set(PROJECT_VERSION_BUILD_METADATA ${GIT_COMMIT_SHORT_HASH} PARENT_SCOPE)
-
-  string(REGEX REPLACE "^v" "" CLEAN_TAG "${GIT_TAG}")
-  if(CLEAN_TAG MATCHES "^([0-9]+)\\.([0-9]+)\\.([0-9]+)(-.*)?$")
-    set(PROJECT_VERSION_MAJOR ${CMAKE_MATCH_1})
-    set(PROJECT_VERSION_MAJOR ${CMAKE_MATCH_1} PARENT_SCOPE)
-    set(PROJECT_VERSION_MINOR ${CMAKE_MATCH_2})
-    set(PROJECT_VERSION_MINOR ${CMAKE_MATCH_2} PARENT_SCOPE)
-    set(PROJECT_VERSION_PATCH ${CMAKE_MATCH_3})
-    set(PROJECT_VERSION_PATCH ${CMAKE_MATCH_3} PARENT_SCOPE)
-
-    set(PROJECT_VERSION "${CMAKE_MATCH_1}.${CMAKE_MATCH_2}.${CMAKE_MATCH_3}") 
-    set(PROJECT_VERSION "${CMAKE_MATCH_1}.${CMAKE_MATCH_2}.${CMAKE_MATCH_3}" PARENT_SCOPE)
-    set(PROJECT_VERSION_FULL "${CMAKE_MATCH_1}.${CMAKE_MATCH_2}.${CMAKE_MATCH_3}+${GIT_COMMIT_SHORT_HASH}")
-    set(PROJECT_VERSION_FULL "${CMAKE_MATCH_1}.${CMAKE_MATCH_2}.${CMAKE_MATCH_3}+${GIT_COMMIT_SHORT_HASH}" PARENT_SCOPE)
-  else()
-    message(WARNING "Tag '${CLEAN_TAG}' does not match semver format")
-  endif()
-endfunction()
@@ -1,142 +0,0 @@
-include_guard(DIRECTORY)
-include(${CMAKE_SOURCE_DIR}/cmake/ExportToParent.cmake)
-include(${CMAKE_SOURCE_DIR}/cmake/InstallBasic.cmake)
-include(${CMAKE_SOURCE_DIR}/cmake/ProtobufGenerateCpp.cmake)
-include(${CMAKE_SOURCE_DIR}/cmake/SetFirstDefined.cmake)
-include(${CMAKE_SOURCE_DIR}/cmake/safe_dep_glob.cmake)
-
-function(init_library)
-  set(_options SHARED STATIC)
-  set(_one_value_args
-    NAME
-    SOURCE_DIR
-    INCLUDE_DIR
-  )
-  set(_multi_value_args)
-  cmake_parse_arguments(_ARGS "${_options}" "${_one_value_args}" "${_multi_value_args}" ${ARGN})
-  if (_ARGS_SHARED AND _ARGS_STATIC)
-    message(FATAL_ERROR "Invalid args passed to init_library: cannot specify STATIC and SHARED")
-  endif()
-  set_first_defined(_name _ARGS_NAME PROJECT_NAME)
-  set_first_defined(_source_dir _ARGS_SOURCE_DIR _name)
-  set_first_defined(_include_dir _ARGS_INCLUDE_DIR _source_dir)
-
-  # Cpp sources
-  file(
-    GLOB_RECURSE ${_name}_SOURCES
-    RELATIVE ${CMAKE_CURRENT_SOURCE_DIR}
-    ${_source_dir}/*.cpp
-  )
-  # Note: cpp files inside 'tests' dir(s) are considered tests.
-  # Remove tests from main sources, and add them to test sources
-  foreach(item ${${_name}_SOURCES})
-    if(${item} MATCHES "tests/.*\.cpp$")
-      list(APPEND ${_name}_TEST_SOURCES ${item})
-    endif()
-  endforeach()
-  foreach(item ${${_name}_TEST_SOURCES})
-    list(REMOVE_ITEM ${_name}_SOURCES ${item})
-  endforeach()
-
-  # Proto sources 
-  file(
-    GLOB_RECURSE ${_name}_PROTO_SOURCES
-    RELATIVE ${CMAKE_CURRENT_SOURCE_DIR}
-    ${_source_dir}/protobufs/*.proto
-  )
-  update_deps_file("${${_name}_PROTO_SOURCES};")
-  protobuf_generate_cpp(
-    CXX_SOURCES ${_name}_CXX_PROTO_SOURCES
-    CXX_HEADERS ${_name}_CXX_PROTO_HEADERS
-    CXX_OUT_DIR ${CMAKE_CURRENT_BINARY_DIR}
-    PROTO_INCLUDE_DIRS
-      ${EXTERNALS_INCLUDE}
-      ${CMAKE_CURRENT_SOURCE_DIR}
-    PROTO_SOURCES
-      ${${_name}_PROTO_SOURCES}
-  )
-
-  export_to_parent(
-    ${_name}_SOURCES
-    ${_name}_TEST_SOURCES
-    ${_name}_PROTO_SOURCES
-    ${_name}_CXX_PROTO_SOURCES
-    ${_name}_CXX_PROTO_HEADERS
-  )
-
-  # Add main lib
-  if (_ARGS_STATIC)
-    add_library(${_name} STATIC ${${_name}_SOURCES} ${${_name}_CXX_PROTO_SOURCES})
-  elseif(_ARGS_SHARED)
-    add_library(${_name} SHARED ${${_name}_SOURCES} ${${_name}_CXX_PROTO_SOURCES})
-  else()
-    add_library(${_name} ${${_name}_SOURCES} ${${_name}_CXX_PROTO_SOURCES})
-  endif()
-  target_include_directories(
-    ${_name}
-    SYSTEM
-    BEFORE
-    PUBLIC
-      $<BUILD_INTERFACE:${EXTERNALS_ROOT}/include>
-      $<BUILD_INTERFACE:${EXTERNALS_ROOT}/include/zlib>
-  )
-  target_include_directories(
-    ${_name}
-    PUBLIC
-      $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}>
-      $<INSTALL_INTERFACE:${CMAKE_INSTALL_INCLUDEDIR}>
-  )
-  # Include protos (if any)
-  if (${_name}_PROTO_SOURCES)
-    target_include_directories(
-      ${_name}
-      PUBLIC
-        $<BUILD_INTERFACE:${CMAKE_CURRENT_BINARY_DIR}>
-    )
-  endif()
-  target_link_directories(
-    ${_name}
-    PUBLIC
-      $<BUILD_INTERFACE:${CMAKE_CURRENT_BINARY_DIR}>
-      $<BUILD_INTERFACE:${EXTERNALS_ROOT}/lib/$<CONFIG>>
-      $<INSTALL_INTERFACE:${CMAKE_INSTALL_LIBDIR}>
-  )
-  # Install main lib
-  foreach(_hdr ${${_name}_CXX_PROTO_HEADERS})
-    file(RELATIVE_PATH _relative_path ${CMAKE_CURRENT_BINARY_DIR} ${_hdr})
-    install(FILES ${_hdr} DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}/${_relative_path})
-  endforeach()
-  install_basic(
-    NAME ${_name}
-    INCLUDE_DIR ${_include_dir}
-  )
-
-  # Add tests executable (if there are any)
-  if (${_name}_TEST_SOURCES AND NOT WIN32)
-    add_executable(${_name}-tests ${${_name}_TEST_SOURCES})
-
-    get_target_property(_lib_type ${_name} TYPE)
-    if (_lib_type STREQUAL "SHARED_LIBRARY")
-      # Set RPATH so the tests exe knows where to find the shared lib.
-      set_target_properties(
-        ${_name}-tests
-        PROPERTIES
-          BUILD_RPATH "$ORIGIN"
-          INSTALL_RPATH "$ORIGIN/../${CMAKE_INSTALL_LIBDIR}"
-      )
-    endif()
-
-    # Link main lib
-    target_link_directories(
-      ${_name}-tests
-      PRIVATE
-        $<BUILD_INTERFACE:${CMAKE_CURRENT_BINARY_DIR}>
-        $<INSTALL_INTERFACE:${CMAKE_INSTALL_LIBDIR}>
-    )
-    target_link_libraries(${_name}-tests PRIVATE ${_name})
-
-    # Catch2
-    find_package(Catch2 CONFIG REQUIRED COMPONENTS Catch2WithMain)
-    target_link_libraries(${_name}-tests PRIVATE Catch2::Catch2WithMain)
-  endif()
-endfunction()
@@ -1,39 +0,0 @@
-include_guard(DIRECTORY)
-include(GNUInstallDirs)
-include(${CMAKE_SOURCE_DIR}/cmake/PackBasic.cmake)
-include(${CMAKE_SOURCE_DIR}/cmake/SetFirstDefined.cmake)
-
-function(install_basic)
-  set(_options)
-  set(_one_value_args NAME INCLUDE_DIR)
-  set(_multi_value_args)
-  cmake_parse_arguments(_ARGS "${_options}" "${_one_value_args}" "${_multi_value_args}" ${ARGN})
-  set_first_defined(_name _ARGS_NAME PROJECT_NAME)
-  set_first_defined(_include_dir _ARGS_INCLUDE_DIR _name)
-
-  file(
-    GLOB_RECURSE ${_name}_HEADERS
-    RELATIVE ${CMAKE_CURRENT_SOURCE_DIR}
-    ${_include_dir}/*.h)
-
-  target_sources(${_name}
-    PUBLIC
-      FILE_SET HEADERS FILES ${${_name}_HEADERS})
-
-  install(
-    TARGETS ${_name}
-    EXPORT ${_name}Targets
-    RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR}
-    LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR}
-    ARCHIVE DESTINATION ${CMAKE_INSTALL_LIBDIR}
-    FILE_SET HEADERS DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}
-  )
-
-  install(
-    EXPORT ${_name}Targets
-    FILE ${_name}Targets.cmake
-    NAMESPACE ${_name}::
-    DESTINATION ${CMAKE_INSTALL_LIBDIR}/cmake/${_name})
-
-  pack_basic()
-endfunction()
@@ -1,44 +0,0 @@
-include_guard(DIRECTORY)
-include(CMakePackageConfigHelpers)
-include(${CMAKE_SOURCE_DIR}/cmake/SetFirstDefined.cmake)
-
-function(pack_basic)
-  set(_options)
-  set(_one_value_args NAME)
-  set(_multi_value_args)
-  cmake_parse_arguments(_ARGS "${_options}" "${_one_value_args}" "${_multi_value_args}" ${ARGN})
-  set_first_defined(_name _ARGS_NAME PROJECT_NAME)
-
-  find_package(Git QUIET)
-
-  if(GIT_FOUND)
-    execute_process(
-      COMMAND ${GIT_EXECUTABLE} describe --tags --always --dirty
-      WORKING_DIRECTORY ${CMAKE_SOURCE_DIR}
-      OUTPUT_VARIABLE PROJECT_VERSION
-      OUTPUT_STRIP_TRAILING_WHITESPACE
-    )
-  endif()
-
-  if(NOT GIT_FOUND OR PROJECT_VERSION STREQUAL "")
-    set(PROJECT_VERSION "0.0.0")
-  endif()
-
-  write_basic_package_version_file(
-    "${CMAKE_CURRENT_BINARY_DIR}/${_name}ConfigVersion.cmake"
-    VERSION ${PROJECT_VERSION}
-    COMPATIBILITY AnyNewerVersion
-  )
-
-  configure_package_config_file(
-    "${CMAKE_CURRENT_SOURCE_DIR}/Config.cmake.in"
-    "${CMAKE_CURRENT_BINARY_DIR}/${_name}Config.cmake"
-    INSTALL_DESTINATION ${CMAKE_INSTALL_LIBDIR}/cmake/${_name}
-  )
-
-  install(
-    FILES
-      "${CMAKE_CURRENT_BINARY_DIR}/${_name}Config.cmake"
-      "${CMAKE_CURRENT_BINARY_DIR}/${_name}ConfigVersion.cmake"
-    DESTINATION ${CMAKE_INSTALL_LIBDIR}/cmake/${_name})
-endfunction()
@@ -1,23 +1,12 @@
-set(Protobuf_PROTOC_EXECUTABLE "${EXTERNALS_BIN}/protoc")
-set(PROTOBUF_INCLUDE_DIRS "${EXTERNALS_INCLUDE}")
+set(Protobuf_PROTOC_EXECUTABLE "${SPARK_BIN}/protoc")
+set(PROTOBUF_INCLUDE_DIRS "${SPARK_INCLUDE}")
 set(PROTOBUF_STATIC_LIB "libprotobuf")
-set(_protobuf_src_dir ${CMAKE_SOURCE_DIR}/src/lib_spark)
-set(_protobuf_bin_dir ${CMAKE_BINARY_DIR}/src/lib_spark)

-function(Protobuf_Generate_Cpp)
-  set(_options)
-  set(_one_value_args CXX_SOURCES CXX_HEADERS CXX_OUT_DIR)
-  set(_multi_value_args PROTO_INCLUDE_DIRS PROTO_SOURCES)
-  cmake_parse_arguments(_ARGS "${_options}" "${_one_value_args}" "${_multi_value_args}" ${ARGN})
-  set(${_ARGS_CXX_SOURCES})
-  set(${_ARGS_CXX_HEADERS})
+function(Protobuf_Generate_Cpp SRCS HDRS)
+  set(${SRCS})
+  set(${HDRS})

-  set(_proto_include_cli_args)
-  foreach(_dir ${_ARGS_PROTO_INCLUDE_DIRS})
-    list(APPEND _proto_include_cli_args -I ${_dir})
-  endforeach()
-
-  foreach(FIL ${_ARGS_PROTO_SOURCES})
+  foreach(FIL ${ARGN})
    get_filename_component(ABS_FIL ${FIL} ABSOLUTE)
    get_filename_component(FIL_WE ${FIL} NAME_WE)
    get_filename_component(FIL_DIR ${FIL} DIRECTORY)
@@ -25,56 +14,48 @@ function(Protobuf_Generate_Cpp)
      set(FIL_WE "${FIL_DIR}/${FIL_WE}")
    endif()

-    list(APPEND ${_ARGS_CXX_SOURCES} "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.pb.cc")
-    list(APPEND ${_ARGS_CXX_HEADERS} "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.pb.h")
+    list(APPEND ${SRCS} "${CMAKE_CURRENT_BINARY_DIR}/${FIL_WE}.pb.cc")
+    list(APPEND ${HDRS} "${CMAKE_CURRENT_BINARY_DIR}/${FIL_WE}.pb.h")

    add_custom_command(
-      OUTPUT "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.pb.cc"
-             "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.pb.h"
+      OUTPUT "${CMAKE_CURRENT_BINARY_DIR}/${FIL_WE}.pb.cc"
+             "${CMAKE_CURRENT_BINARY_DIR}/${FIL_WE}.pb.h"
      COMMAND
-        ${Protobuf_PROTOC_EXECUTABLE} ARGS --cpp_out=${_ARGS_CXX_OUT_DIR} ${_proto_include_cli_args} ${ABS_FIL}
+        ${Protobuf_PROTOC_EXECUTABLE} ARGS --cpp_out=${CMAKE_BINARY_DIR}/src -I
+        ${PROTOBUF_INCLUDE_DIRS} -I ${CMAKE_SOURCE_DIR}/src ${ABS_FIL}
      DEPENDS ${ABS_FIL}
      COMMENT "Running C++ protocol buffer compiler on ${FIL}"
      VERBATIM)
  endforeach()

-  set_source_files_properties(${${_ARGS_CXX_SOURCES}} ${${_ARGS_CXX_HEADERS}} PROPERTIES GENERATED TRUE)
+  set_source_files_properties(${${SRCS}} ${${HDRS}} PROPERTIES GENERATED TRUE)

  # remove some warnings from protobuf generated files
  if(WIN32)
-    set_source_files_properties(${${_ARGS_CXX_SOURCES}} PROPERTIES COMPILE_FLAGS
+    set_source_files_properties(${${SRCS}} PROPERTIES COMPILE_FLAGS
                                                      "/wd4309 /wd4838")
  else()
    set_source_files_properties(
-      ${${_ARGS_CXX_SOURCES}}
+      ${${SRCS}}
      PROPERTIES
        COMPILE_FLAGS
        "-Wno-unused-parameter -Wno-effc++ -Wno-pedantic -Wno-sign-compare -Wno-uninitialized"
    )
  endif()

-  set(${_ARGS_CXX_SOURCES}
-      ${${_ARGS_CXX_SOURCES}}
+  set(${SRCS}
+      ${${SRCS}}
      PARENT_SCOPE)
-  set(${_ARGS_CXX_HEADERS}
-      ${${_ARGS_CXX_HEADERS}}
+  set(${HDRS}
+      ${${HDRS}}
      PARENT_SCOPE)
 endfunction()

-function(Service_Compiler_Generate_Cpp)
-  set(_options)
-  set(_one_value_args CXX_SOURCES CXX_HEADERS CXX_OUT_DIR)
-  set(_multi_value_args PROTO_INCLUDE_DIRS PROTO_SOURCES)
-  cmake_parse_arguments(_ARGS "${_options}" "${_one_value_args}" "${_multi_value_args}" ${ARGN})
-  set(${_ARGS_CXX_SOURCES})
-  set(${_ARGS_CXX_HEADERS})
+function(Service_Compiler_Generate_Cpp SRCS HDRS)
+  set(${SRCS})
+  set(${HDRS})

-  set(_proto_include_cli_args)
-  foreach(_dir ${_ARGS_PROTO_INCLUDE_DIRS})
-    list(APPEND _proto_include_cli_args -I ${_dir})
-  endforeach()
-
-  foreach(FIL ${_ARGS_PROTO_SOURCES})
+  foreach(FIL ${ARGN})
    get_filename_component(ABS_FIL ${FIL} ABSOLUTE)
    get_filename_component(FIL_WE ${FIL} NAME_WE)
    get_filename_component(FIL_DIR ${FIL} DIRECTORY)
@@ -82,58 +63,58 @@ function(Service_Compiler_Generate_Cpp)
      set(FIL_WE "${FIL_DIR}/${FIL_WE}")
    endif()

-    list(APPEND ${_ARGS_CXX_SOURCES} "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.pb.cc")
-    list(APPEND ${_ARGS_CXX_HEADERS} "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.pb.h")
+    list(APPEND ${SRCS} "${CMAKE_CURRENT_BINARY_DIR}/${FIL_WE}.pb.cc")
+    list(APPEND ${HDRS} "${CMAKE_CURRENT_BINARY_DIR}/${FIL_WE}.pb.h")

-    list(APPEND ${_ARGS_CXX_SOURCES} "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.client.pb.cc")
-    list(APPEND ${_ARGS_CXX_HEADERS} "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.client.pb.h")
+    list(APPEND ${SRCS} "${CMAKE_CURRENT_BINARY_DIR}/${FIL_WE}.client.pb.cc")
+    list(APPEND ${HDRS} "${CMAKE_CURRENT_BINARY_DIR}/${FIL_WE}.client.pb.h")

-    list(APPEND ${_ARGS_CXX_SOURCES} "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.server.pb.cc")
-    list(APPEND ${_ARGS_CXX_HEADERS} "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.server.pb.h")
+    list(APPEND ${SRCS} "${CMAKE_CURRENT_BINARY_DIR}/${FIL_WE}.server.pb.cc")
+    list(APPEND ${HDRS} "${CMAKE_CURRENT_BINARY_DIR}/${FIL_WE}.server.pb.h")

    set(TEMPLATES
-        ${CMAKE_SOURCE_DIR}/src/lib_spark/spark/api/service_compiler/templates/cpp/client.pb.cc
-        ${CMAKE_SOURCE_DIR}/src/lib_spark/spark/api/service_compiler/templates/cpp/client.pb.h
-        ${CMAKE_SOURCE_DIR}/src/lib_spark/spark/api/service_compiler/templates/cpp/server.pb.cc
-        ${CMAKE_SOURCE_DIR}/src/lib_spark/spark/api/service_compiler/templates/cpp/server.pb.h
+        ${CMAKE_CURRENT_SOURCE_DIR}/api/service_compiler/templates/cpp/client.pb.cc
+        ${CMAKE_CURRENT_SOURCE_DIR}/api/service_compiler/templates/cpp/client.pb.h
+        ${CMAKE_CURRENT_SOURCE_DIR}/api/service_compiler/templates/cpp/server.pb.cc
+        ${CMAKE_CURRENT_SOURCE_DIR}/api/service_compiler/templates/cpp/server.pb.h
    )

-    set(_plugin_py ${CMAKE_SOURCE_DIR}/src/lib_spark/spark/api/service_compiler/plugin.py)
    if(WIN32)
-      set(_plugin_cmd ${CMAKE_SOURCE_DIR}/src/lib_spark/spark/api/service_compiler/plugin.cmd)
      add_custom_command(
-        OUTPUT "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.pb.cc"
-               "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.pb.h"
-               "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.client.pb.cc"
-               "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.client.pb.h"
-               "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.server.pb.cc"
-               "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.server.pb.h"
+        OUTPUT "${CMAKE_CURRENT_BINARY_DIR}/${FIL_WE}.pb.cc"
+               "${CMAKE_CURRENT_BINARY_DIR}/${FIL_WE}.pb.h"
+               "${CMAKE_CURRENT_BINARY_DIR}/${FIL_WE}.client.pb.cc"
+               "${CMAKE_CURRENT_BINARY_DIR}/${FIL_WE}.client.pb.h"
+               "${CMAKE_CURRENT_BINARY_DIR}/${FIL_WE}.server.pb.cc"
+               "${CMAKE_CURRENT_BINARY_DIR}/${FIL_WE}.server.pb.h"
        COMMAND SET PYTHONPATH=${SPARK_PYTHON}
        COMMAND ${CMAKE_SOURCE_DIR}/.venv/Scripts/activate.bat
        COMMAND
-          ${Protobuf_PROTOC_EXECUTABLE} ARGS --cpp_out=${_ARGS_CXX_OUT_DIR}
-          --plugin=protoc-gen-service=${_plugin_cmd}
-          --service_out=lang=cpp:${_ARGS_CXX_OUT_DIR} -I
-          ${PROTOBUF_INCLUDE_DIRS} -I ${_protobuf_src_dir} ${ABS_FIL}
-        DEPENDS ${ABS_FIL} ${TEMPLATES} ${_plugin_cmd} ${_plugin_py}
+          ${Protobuf_PROTOC_EXECUTABLE} ARGS --cpp_out=${CMAKE_BINARY_DIR}/src
+          --plugin=protoc-gen-service=${CMAKE_SOURCE_DIR}/src/lib/api/service_compiler/plugin.cmd
+          --service_out=lang=cpp:${CMAKE_BINARY_DIR}/src -I
+          ${PROTOBUF_INCLUDE_DIRS} -I ${CMAKE_SOURCE_DIR}/src ${ABS_FIL}
+        DEPENDS ${ABS_FIL} ${TEMPLATES}
+                ${CMAKE_CURRENT_SOURCE_DIR}/api/service_compiler/plugin.py
        COMMENT "Running C++ service compiler on ${FIL}"
        VERBATIM)
    else()
      add_custom_command(
-        OUTPUT "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.pb.cc"
-               "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.pb.h"
-               "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.client.pb.cc"
-               "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.client.pb.h"
-               "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.server.pb.cc"
-               "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.server.pb.h"
+        OUTPUT "${CMAKE_CURRENT_BINARY_DIR}/${FIL_WE}.pb.cc"
+               "${CMAKE_CURRENT_BINARY_DIR}/${FIL_WE}.pb.h"
+               "${CMAKE_CURRENT_BINARY_DIR}/${FIL_WE}.client.pb.cc"
+               "${CMAKE_CURRENT_BINARY_DIR}/${FIL_WE}.client.pb.h"
+               "${CMAKE_CURRENT_BINARY_DIR}/${FIL_WE}.server.pb.cc"
+               "${CMAKE_CURRENT_BINARY_DIR}/${FIL_WE}.server.pb.h"
        COMMAND ${CMAKE_SOURCE_DIR}/tools/venv_exec.sh
        ARGS
          ${Protobuf_PROTOC_EXECUTABLE}
-          --cpp_out=${_ARGS_CXX_OUT_DIR}
-          --plugin=protoc-gen-service=${_plugin_py}
-          --service_out=lang=cpp,make_file_name=${ABS_FIL}:${_ARGS_CXX_OUT_DIR}
-          ${_proto_include_cli_args} ${ABS_FIL}
-        DEPENDS ${ABS_FIL} ${TEMPLATES} ${_plugin_py}
+          --cpp_out=${CMAKE_BINARY_DIR}/src
+          --plugin=protoc-gen-service=${CMAKE_SOURCE_DIR}/src/lib/api/service_compiler/plugin.py
+          --service_out=lang=cpp,make_file_name=${ABS_FIL}:${CMAKE_BINARY_DIR}/src
+          -I ${PROTOBUF_INCLUDE_DIRS} -I ${CMAKE_SOURCE_DIR}/src ${ABS_FIL}
+        DEPENDS ${ABS_FIL} ${TEMPLATES}
+                ${CMAKE_CURRENT_SOURCE_DIR}/api/service_compiler/plugin.py
        COMMENT "Running C++ service compiler on ${FIL}"
        VERBATIM)
    endif()
@@ -142,22 +123,22 @@ function(Service_Compiler_Generate_Cpp)

  # remove some warnings from protobuf generated files
  if(WIN32)
-    set_source_files_properties(${${_ARGS_CXX_SOURCES}} PROPERTIES COMPILE_FLAGS
+    set_source_files_properties(${${SRCS}} PROPERTIES COMPILE_FLAGS
                                                      "/wd4309 /wd4838")
  else()
    set_source_files_properties(
-      ${${_ARGS_CXX_SOURCES}}
+      ${${SRCS}}
      PROPERTIES
        COMPILE_FLAGS
        "-Wno-unused-parameter -Wno-effc++ -Wno-pedantic -Wno-sign-compare -Wno-uninitialized"
    )
  endif()

-  set(${_ARGS_CXX_SOURCES}
-      ${${_ARGS_CXX_SOURCES}}
+  set(${SRCS}
+      ${${SRCS}}
      PARENT_SCOPE)
-  set(${_ARGS_CXX_HEADERS}
-      ${${_ARGS_CXX_HEADERS}}
+  set(${HDRS}
+      ${${HDRS}}
      PARENT_SCOPE)

 endfunction()
@@ -1,83 +0,0 @@
-include_guard(GLOBAL)
-
-function(protobuf_generate_cpp)
-  set(_options)
-  set(_one_value_args
-    CXX_SOURCES
-    CXX_HEADERS
-    CXX_OUT_DIR
-    PROTOC
-  )
-  set(_multi_value_args
-    PROTO_INCLUDE_DIRS
-    PROTO_SOURCES
-  )
-  cmake_parse_arguments(_ARGS "${_options}" "${_one_value_args}" "${_multi_value_args}" ${ARGN})
-
-  if (NOT DEFINED _ARGS_PROTOC)
-    if (WIN32)
-      set(_ARGS_PROTOC "${EXTERNALS_BIN}/protoc.exe")
-    else()
-      set(_ARGS_PROTOC "${EXTERNALS_BIN}/protoc")
-    endif()
-  endif()
-  set(${_ARGS_CXX_SOURCES})
-  set(${_ARGS_CXX_HEADERS})
-
-  set(_proto_include_cli_args)
-  foreach(_dir ${_ARGS_PROTO_INCLUDE_DIRS})
-    list(APPEND _proto_include_cli_args -I ${_dir})
-  endforeach()
-
-  foreach(_fil ${_ARGS_PROTO_SOURCES})
-    get_filename_component(_abs_fil ${_fil} ABSOLUTE)
-    get_filename_component(_fil_we ${_fil} NAME_WE)
-    get_filename_component(_fil_dir ${_fil} DIRECTORY)
-    if(_fil_dir)
-      set(_fil_we "${_fil_dir}/${_fil_we}")
-    endif()
-    list(APPEND ${_ARGS_CXX_SOURCES} "${_ARGS_CXX_OUT_DIR}/${_fil_we}.pb.cc")
-    list(APPEND ${_ARGS_CXX_HEADERS} "${_ARGS_CXX_OUT_DIR}/${_fil_we}.pb.h")
-    message(STATUS "add_custom_command for ${_fil} -> ${_ARGS_CXX_OUT_DIR}/${_fil_we}.pb.h")
-    # TODO: Pass proto sources by relative path (so this fn can be called from
-    # arbitrary subdirs).
-    add_custom_command(
-      OUTPUT
-        "${_ARGS_CXX_OUT_DIR}/${_fil_we}.pb.cc"
-        "${_ARGS_CXX_OUT_DIR}/${_fil_we}.pb.h"
-      COMMAND
-        ${_ARGS_PROTOC}
-      ARGS
-        --cpp_out=${_ARGS_CXX_OUT_DIR} ${_proto_include_cli_args} ${_abs_fil}
-      DEPENDS ${_abs_fil}
-      COMMENT "Running C++ protocol buffer compiler on ${_fil}"
-      VERBATIM
-    )
-  endforeach()
-
-  set_source_files_properties(
-    ${${_ARGS_CXX_SOURCES}}
-    ${${_ARGS_CXX_HEADERS}}
-    PROPERTIES
-      GENERATED TRUE
-  )
-
-  # remove some warnings from protobuf generated files
-  if(WIN32)
-    set_source_files_properties(
-      ${${_ARGS_CXX_SOURCES}}
-      PROPERTIES
-        COMPILE_FLAGS "/wd4309 /wd4838"
-    )
-  else()
-    set_source_files_properties(
-      ${${_ARGS_CXX_SOURCES}}
-      PROPERTIES
-        COMPILE_FLAGS "-Wno-unused-parameter -Wno-effc++ -Wno-pedantic -Wno-sign-compare -Wno-uninitialized"
-    )
-  endif()
-
-  set(${_ARGS_CXX_SOURCES} ${${_ARGS_CXX_SOURCES}} PARENT_SCOPE)
-  set(${_ARGS_CXX_HEADERS} ${${_ARGS_CXX_HEADERS}} PARENT_SCOPE)
-endfunction()
-
@@ -1,61 +0,0 @@
-include_guard(GLOBAL)
-
-function(protobuf_generate_cpp_config)
-  set(_options)
-  set(_one_value_args
-    PROTO_SOURCES_HEADER
-    PROTO_DEFAULTS_HEADER
-  )
-  set(_multi_value_args
-    PROTO_SOURCES
-    PROTO_DEFAULTS
-  )
-  cmake_parse_arguments(_ARGS "${_options}" "${_one_value_args}" "${_multi_value_args}" ${ARGN})
-
-  # Configure the 'config_proto_files' header. It provides an array of strings populated with the
-  # files in PROTO_SOURCES. At runtime, we use this array to generate a special 'config' proto
-  # message with fields for each message defined in said files. A global instance of this 'config'
-  # proto is then constructed, and populated with [compile-time] defaults, and [runtime] overrides
-  # (i.e., in spark.conf). Calls to `bts::config::get<T>()` retrieve the value from the field
-  # associated with type `T` in this global instance.
-  message(STATUS "PROTO_SOURCES: ${_ARGS_PROTO_SOURCES}")
-  message(STATUS "PROTO_SOURCES_HEADER: ${_ARGS_PROTO_SOURCES_HEADER}")
-  message(STATUS "PROTO_DEFAULTS_HEADER: ${_ARGS_PROTO_DEFAULTS_HEADER}")
-  if (_ARGS_PROTO_SOURCES_HEADER)
-    set(PROTO_SOURCES)
-    set(PROTO_SOURCES_CXX_ARRAY_ITEMS "")
-    foreach(_proto_path ${_ARGS_PROTO_SOURCES})
-      list(APPEND PROTO_SOURCES ${_proto_path})
-      string(APPEND PROTO_SOURCES_CXX_ARRAY_ITEMS "\"${_proto_path}\",\n")
-    endforeach()
-    list(LENGTH PROTO_SOURCES PROTO_SOURCES_LENGTH)
-    message(STATUS "CONFIGURING PROTO_SOURCES_HEADER): ${_ARGS_PROTO_SOURCES_HEADER}")
-    configure_file(${CMAKE_SOURCE_DIR}/cmake/config_proto_files.h.in ${_ARGS_PROTO_SOURCES_HEADER})
-  endif()
-
-  # Generate static defaults. Ideally, this would be a single string, but we use an array because
-  # the compiler complains if the string is longer than 65k.
-  set(_cfg_defaults_str "inline const char * config_static_defaults[] = { ")
-
-  foreach(_cfg_file_path ${_ARGS_PROTO_DEFAULTS})
-    file(READ ${_cfg_file_path} _cfg_file_content)
-    string(REPLACE "\"" "\\\"" _cfg_file_content_escaped ${_cfg_file_content})
-    string(REPLACE "\n" ";" _cfg_lines ${_cfg_file_content_escaped})
-
-    foreach(_line ${_cfg_lines})
-      string(APPEND _cfg_defaults_str "\n  \"${_line}\",")
-    endforeach(_line)
-  endforeach(_cfg_file_path)
-
-  string(APPEND _cfg_defaults_str "};")
-  set(_old_cfg_defaults_str "")
-
-  if(EXISTS ${_ARGS_PROTO_DEFAULTS_HEADER})
-    file(READ ${_ARGS_PROTO_DEFAULTS_HEADER} _old_cfg_defaults_str)
-  endif()
-
-  if(NOT _cfg_defaults_str STREQUAL _old_cfg_defaults_str)
-    file(WRITE "${_ARGS_PROTO_DEFAULTS_HEADER}" "${_cfg_defaults_str}")
-  endif()
-endfunction()
-
@@ -1,49 +0,0 @@
-include_guard(DIRECTORY)
-
-function(protobuf_generate_fs)
-  set(_options)
-  set(_one_value_args TARGET_NAME OUT_DIR PROTOC)
-  set(_multi_value_args PROTO_INCLUDE_DIRS PROTO_SOURCES)
-  cmake_parse_arguments(_ARGS "${_options}" "${_one_value_args}" "${_multi_value_args}" ${ARGN})
-
-  if (NOT DEFINED _ARGS_TARGET_NAME)
-    set(_ARGS_TARGET_NAME "${PROJECT_NAME}_generate_fs_protos")
-  endif()
-  if (NOT DEFINED _ARGS_PROTOC)
-    set(_ARGS_PROTOC "${EXTERNALS_BIN}/protoc")
-  endif()
-
-  set(_proto_include_cli_args)
-  foreach(_dir ${_ARGS_PROTO_INCLUDE_DIRS})
-    list(APPEND _proto_include_cli_args -I ${_dir})
-  endforeach()
-
-  set(_outputs)
-  foreach(_proto_source ${_ARGS_PROTO_SOURCES})
-    # get_filename_component(_abs_proto_source ${_proto_source} ABSOLUTE)
-    get_filename_component(_proto_source_we ${_proto_source} NAME_WE)
-    # get_filename_component(_proto_source_dir ${_proto_source} DIRECTORY)
-    list(APPEND _outputs ${_ARGS_OUT_DIR}/${_proto_source_we}.fable.fs)
-  endforeach()
-
-  set(_venv_exec ${CMAKE_SOURCE_DIR}/tools/venv_exec.sh)
-  set(_plugin_py ${CMAKE_SOURCE_DIR}/src/lib_spark/spark/api/service_compiler/plugin.py)
-  set(_template_fs ${CMAKE_SOURCE_DIR}/src/lib_spark/spark/api/service_compiler/templates/fs/fable.fs)
-  add_custom_command(
-    OUTPUT ${_outputs}
-    WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}
-    COMMAND ${_venv_exec}
-      ${_ARGS_PROTOC}
-      --plugin=protoc-gen-service=${_plugin_py}
-      --service_out=lang=fs,strip_out_path=1:${_ARGS_OUT_DIR}
-      ${_proto_include_cli_args}
-      ${_ARGS_PROTO_SOURCES}
-    DEPENDS
-      ${_venv_exec}
-      ${_plugin_py}
-      ${_template_fs}
-      ${CMAKE_CURRENT_SOURCE_DIR}/package-lock.json
-      ${_ARGS_PROTO_SOURCES}
-  )
-  add_custom_target(${_ARGS_TARGET_NAME} ALL DEPENDS "${_outputs}")
-endfunction()
@@ -1,33 +0,0 @@
-include_guard(DIRECTORY)
-
-function(protobuf_generate_js)
-  set(_options)
-  set(_one_value_args TARGET_NAME OUTPUT)
-  set(_multi_value_args PROTO_INCLUDE_DIRS PROTO_SOURCES)
-  cmake_parse_arguments(_ARGS "${_options}" "${_one_value_args}" "${_multi_value_args}" ${ARGN})
-
-  if (NOT DEFINED _ARGS_TARGET_NAME)
-    set(_ARGS_TARGET_NAME "${PROJECT_NAME}_generate_js_protos")
-  endif()
-
-  set(_proto_include_cli_args)
-  foreach(_dir ${_ARGS_PROTO_INCLUDE_DIRS})
-    list(APPEND _proto_include_cli_args -p ${_dir})
-  endforeach()
-
-  add_custom_command(
-    OUTPUT ${_ARGS_OUTPUT}
-    WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}
-    COMMAND npx
-      --no-install pbjs
-      -t json-module
-      -w commonjs
-      -o ${_ARGS_OUTPUT}
-      ${_proto_include_cli_args}
-      ${_ARGS_PROTO_SOURCES}
-    DEPENDS
-      ${CMAKE_CURRENT_SOURCE_DIR}/package-lock.json
-      ${_ARGS_PROTO_SOURCES}
-  )
-  add_custom_target(${_ARGS_TARGET_NAME} ALL DEPENDS ${_ARGS_OUTPUT})
-endfunction()
@@ -1,10 +0,0 @@
-include_guard(GLOBAL)
-
-macro(set_first_defined RESULT)
-  foreach(_arg ${ARGN})
-    if (DEFINED ${_arg})
-      set(${RESULT} ${${_arg}})
-      break()
-    endif()
-  endforeach()
-endmacro()
@@ -1,49 +0,0 @@
-include_guard(DIRECTORY)
-include(${CMAKE_SOURCE_DIR}/cmake/TernaryOperator.cmake)
-
-function(sym_link_dir)
-  set(_options)
-  set(_one_value_args TARGET_NAME SOURCE DEST COMMENT)
-  set(_multi_value_args)
-  cmake_parse_arguments(_ARGS "${_options}" "${_one_value_args}" "${_multi_value_args}" ${ARGN})
-  add_custom_command(
-    OUTPUT ${_ARGS_DEST}
-    COMMAND ln ARGS -sf ${_ARGS_SOURCE} ${_ARGS_DEST}
-  )
-  set_ternary(_comment _ARGS_COMMENT "${_ARGS_COMMENT}" "linking dir ${_ARGS_SOURCE} -> ${_ARGS_DEST}")
-  add_custom_target(
-    ${_ARGS_TARGET_NAME} ALL
-    DEPENDS ${_ARGS_DEST}
-    COMMENT "${_comment}"
-  )
-endfunction()
-
-function(sym_link_file)
-  set(_options)
-  set(_one_value_args SOURCE DEST INSTALL_DEST)
-  set(_multi_value_args)
-  cmake_parse_arguments(_ARGS "${_options}" "${_one_value_args}" "${_multi_value_args}" ${ARGN})
-  file(
-    CREATE_LINK
-      ${_ARGS_SOURCE}
-      ${_ARGS_DEST}
-    SYMBOLIC
-  )
-  if (DEFINED _ARGS_INSTALL_DEST)
-    install(
-      FILES
-      ${_ARGS_SOURCE}
-      DESTINATION ${_ARGS_INSTALL_DEST}
-    )
-  endif()
-endfunction()
-
-function(sym_link_local_lib LIB_NAME)
-  set(_lib_file_name "${CMAKE_SHARED_LIBRARY_PREFIX}${LIB_NAME}.so")
-  file(
-    CREATE_LINK
-      "${CMAKE_BINARY_DIR}/src/lib_${LIB_NAME}/${_lib_file_name}"
-      "${CMAKE_CURRENT_BINARY_DIR}/${_lib_file_name}"
-    SYMBOLIC
-  )
-endfunction()
@@ -1,9 +0,0 @@
-include_guard(GLOBAL)
-
-macro(set_ternary RESULT CONDITION TRUE_VALUE FALSE_VALUE)
-  if(${CONDITION})
-    set(${RESULT} ${TRUE_VALUE})
-  else()
-    set(${RESULT} ${FALSE_VALUE})
-  endif()
-endmacro()
@@ -1,11 +0,0 @@
-#pragma once
-
-#include <array>
-
-namespace bts::@PROJECT_NAME@::protobufs {
-
-inline std::array<const char *, @PROTO_SOURCES_LENGTH@> config_proto_files = {
-@PROTO_SOURCES_CXX_ARRAY_ITEMS@
-};
-
-} // namespace bts::@PROJECT_NAME@::protobufs
@@ -1,11 +0,0 @@
-#ifndef BTS_SPARK_VERSION_H
-#define BTS_SPARK_VERSION_H
-
-#define BTS_SPARK_VERSION_MAJOR @PROJECT_VERSION_MAJOR@
-#define BTS_SPARK_VERSION_MINOR @PROJECT_VERSION_MINOR@
-#define BTS_SPARK_VERSION_PATCH @PROJECT_VERSION_PATCH@
-#define BTS_SPARK_VERSION_BUILD_METADATA @PROJECT_VERSION_BUILD_METADATA@
-#define BTS_SPARK_VERSION "@PROJECT_VERSION@"
-#define BTS_SPARK_VERSION_FULL "@PROJECT_VERSION_FULL@"
-
-#endif
@@ -0,0 +1,24 @@
+cmake_minimum_required(VERSION 3.5)
+
+# Explicitly set policy to convert relative paths to absolute (although this is a default policy,
+# setting it explicitly prevents a compile-warning)
+cmake_policy(SET CMP0076 NEW)
+
+# add protos to the library to facilitate source_group
+target_sources(libspark PUBLIC ${Config_Protos} ${NonConfig_Protos} ${Service_Protos})
+
+function(assign_source_group)
+  foreach(_source IN ITEMS ${ARGN})
+    if(IS_ABSOLUTE "${_source}")
+      file(RELATIVE_PATH _source_rel "${CMAKE_CURRENT_SOURCE_DIR}" "${_source}")
+    else()
+      set(_source_rel "${_source}")
+    endif()
+    get_filename_component(_source_path "${_source_rel}" PATH)
+    string(REPLACE "/" "\\" _source_path_msvc "${_source_path}")
+    source_group("${_source_path_msvc}" FILES "${_source}")
+    set(SOURCE_FILES "${_source}")
+  endforeach()
+endfunction(assign_source_group)
+
+assign_source_group(${SparkLib_Sources} ${Config_Protos} ${NonConfig_Protos} ${Service_Protos})
@@ -1,104 +0,0 @@
-services:
-  sparkd:
-    volumes:
-      - ${SPARKD__DATA_DIR:-sparkd_data}:/home/bts/sparkdata/data
-    build:
-      dockerfile: src/app_sparkd/Dockerfile
-      args:
-        - IMAGE=${SPARKD__IMAGE}
-        - SOURCE_DIR=${SPARKD__SOURCE_DIR}
-        - CONFIG_DIR=${SPARKD__CONFIG_DIR}
-    shm_size: "${SPARKD__SHM_SIZE:-512mb}"
-    command: >
-      sh -c "
-        echo bts | sudo -S chown -R bts:bts /home/bts/sparkdata/data
-        if [ -e "/home/bts/sparkdata/data/logging.sparkd/log" ]; then
-          mv "/home/bts/sparkdata/data/logging.sparkd/log" "/home/bts/sparkdata/data/logging.sparkd/log_$(date '+%Y%m%d-%H%M')"
-        fi
-        if [ -e "/home/bts/sparkdata/conf/.env.local" ]; then
-          echo 'Sourcing /home/bts/sparkdata/conf/.env.local'
-          set -a
-          . /home/bts/sparkdata/conf/.env.local
-          set +a
-        fi
-        ./sparkd run"
-    env_file:
-      - .env
-      - ${SPARKD__CONFIG_DIR}/.env
-    ports:
-      - "${SPARKD__CONFIG_PORT}:${SPARKD__CONFIG_PORT}"
-      - "${SPARKD__QUICKBUS_PORT}:${SPARKD__QUICKBUS_PORT}"
-      - "${SPARKD__SIM_FIX_PORT}:${SPARKD__SIM_FIX_PORT}"
-    deploy:
-      resources:
-        limits:
-          memory: "${SPARKD__MEMORY:-4gb}"
-
-  sparkd-symbols:
-    volumes:
-      - ${SPARKD__DATA_DIR:-sparkd_data}:/home/bts/sparkdata/data
-      - ${SPARKD__RUNTIME_CONF_DIR:-sparkd_conf}:/home/bts/sparkdata/conf
-    build:
-      dockerfile: src/server/sparkd/Dockerfile
-      args:
-        - IMAGE=${SPARKD__IMAGE}
-        - SOURCE_DIR=${SPARKD__SOURCE_DIR}
-        - CONFIG_DIR=${SPARKD__CONFIG_DIR}
-    shm_size: "${SPARKD__SHM_SIZE:-512mb}"
-    command: >
-      sh -c "
-        echo bts | sudo -S chown -R bts:bts /home/bts/sparkdata
-        if [ -e "/home/bts/sparkdata/data/logging.sparkd/log" ]; then
-          mv "/home/bts/sparkdata/data/logging.sparkd/log" "/home/bts/sparkdata/data/logging.sparkd/log_$(date '+%Y%m%d-%H%M')"
-        fi
-        ls -alh /home/bts/sparkdata/conf/
-        cat /home/bts/sparkdata/conf/.env.local
-        if [ -e "/home/bts/sparkdata/conf/.env.local" ]; then
-          echo 'Sourcing /home/bts/sparkdata/conf/.env.local'
-          set -a
-          . /home/bts/sparkdata/conf/.env.local
-          set +a
-        fi
-        ./sparkd run"
-    env_file:
-      - .env
-      - ${SPARKD__CONFIG_DIR}/.env
-    ports:
-      - "${SPARKD__CONFIG_PORT}:${SPARKD__CONFIG_PORT}"
-      - "${SPARKD__QUICKBUS_PORT}:${SPARKD__QUICKBUS_PORT}"
-      - "${SPARKD__SIM_FIX_PORT}:${SPARKD__SIM_FIX_PORT}"
-    deploy:
-      restart_policy:
-        condition: any
-        delay: 3s
-      resources:
-        limits:
-          memory: "${SPARKD__MEMORY:-4gb}"
-
-  sparkd-tail:
-    image: bluetradingsystems/linux_dotnet7
-    volumes:
-      - type: volume
-        read_only: true
-        source: sparkd_data
-        target: /home/bts/sparkdata/data/logging.sparkd
-        volume:
-          subpath: logging.sparkd
-    command: ["tail", "-f", "/home/bts/sparkdata/data/logging.sparkd/log"]
-
-  sparkd-ping-test:
-    depends_on:
-      - sparkd
-    profiles:
-      - ping-test
-    build:
-      context: .
-      dockerfile_inline: |
-        FROM bluetradingsystems/linux_dotnet7
-        USER "root"
-        RUN apt-get update && apt-get -y install iputils-ping
-    command: [ "ping", "sparkd" ]
-
-volumes:
-  sparkd_data:
-    name: "sparkd_${SPARKD__CONFIG_PORT:-8000}"
@@ -124,8 +124,8 @@ service OrderService {
 ```
 ### C++ Example
 ```cpp
-#include "spark/api/public/ServiceContext.h"
-#include "spark/api/services/OrderService.client.pb.h"
+#include "lib/api/public/ServiceContext.h"
+#include "lib/api/services/OrderService.client.pb.h"
 using namespace bts::api;
 ServiceContext ctx;
 auto orders = ctx.locate<orderman::OrderService>();
@@ -29,9 +29,9 @@ namespace sample {
 ```

 ```cpp
-#include "spark/api/public/ServiceContext.h"
-#include "spark/domain_model/Security.h"
-#include "spark/api/services/MarketDataService.client.pb.h"
+#include "lib/api/public/ServiceContext.h"
+#include "spark/lib/domain_model/Security.h"
+#include "lib/api/services/MarketDataService.client.pb.h"
 #include <thread>

 using namespace bts::api;
@@ -1,38 +0,0 @@
-FROM ubuntu:22.04
-
-ARG DEBIAN_FRONTEND=noninteractive
-
-RUN apt-get update && apt-get install -y pkg-config
-RUN apt-get update && apt-get install -y locales
-RUN apt-get update && apt-get install -y ca-certificates
-RUN apt-get update && apt-get install -y libssh2-1-dev
-RUN apt-get update && apt-get install -y software-properties-common
-RUN apt-get update && apt-get install -y sudo
-
-# GCC
-ARG gcc_major=13
-RUN add-apt-repository ppa:ubuntu-toolchain-r/test \
-  && apt-get update \
-  && apt-get install -y g++-$gcc_major
-
-RUN apt-get update \
-  && apt-get install -y dotnet7
-
-RUN locale-gen en_US.UTF-8
-
-ARG docker_user="bts"
-ARG docker_uid="1000"
-ARG docker_pass="bts"
-
-RUN \
-  groupadd -g "$docker_uid" "$docker_user" \
-  && useradd \
-  -m \
-  -u "$docker_uid" \
-  -g "$docker_uid" \
-  -G sudo \
-  --password "$(openssl passwd -1 "$docker_pass")" \
-  "$docker_user"
-
-USER bts
-WORKDIR /home/bts
@@ -1,20 +1,11 @@
 astroid==2.7.3
 autopep8==1.5.7
-blinker==1.9.0
-boto3==1.37.9
-botocore==1.37.9
 certifi==2023.5.7
 charset-normalizer==3.1.0
-click==8.1.8
 colorama==0.4.4
-Flask==3.1.0
 idna==3.4
 isort==5.9.3
-itsdangerous==2.2.0
-Jinja2==3.1.6
-jmespath==1.0.1
 lazy-object-proxy==1.6.0
-MarkupSafe==3.0.2
 mccabe==0.6.1
 platformdirs==2.3.0
 protobuf==4.21.12
@@ -23,14 +14,9 @@ pycodestyle==2.7.0
 pycparser==2.20
 pylint==2.10.2
 PyMeta3==0.5.1
-python-dateutil==2.9.0.post0
-python-dotenv==1.0.1
 requests==2.31.0
-s3transfer==0.11.4
-six==1.17.0
 termcolor==1.1.0
 toml==0.10.2
 urllib3==2.0.3
-Werkzeug==3.1.3
 wrapt==1.12.1
 yapf==0.31.0
@@ -1,7 +1,7 @@
 <Project Sdk="Microsoft.NET.Sdk">

  <PropertyGroup>
-    <TargetFramework>net8.0</TargetFramework>
+    <TargetFramework>net7.0</TargetFramework>
    <ImplicitUsings>enable</ImplicitUsings>
    <Nullable>enable</Nullable>
  </PropertyGroup>
@@ -1,7 +1,7 @@
 <Project Sdk="Microsoft.NET.Sdk">

  <PropertyGroup>
-    <TargetFramework>net8.0</TargetFramework>
+    <TargetFramework>net7.0</TargetFramework>
    <ImplicitUsings>enable</ImplicitUsings>
    <Nullable>enable</Nullable>
  </PropertyGroup>
@@ -1,7 +1,7 @@
 <Project Sdk="Microsoft.NET.Sdk">

  <PropertyGroup>
-    <TargetFramework>net8.0</TargetFramework>
+    <TargetFramework>net7.0</TargetFramework>
    <ImplicitUsings>enable</ImplicitUsings>
    <Nullable>enable</Nullable>
  </PropertyGroup>
@@ -1,14 +1,14 @@
 <Project Sdk="Microsoft.NET.Sdk">

  <PropertyGroup>
-    <TargetFramework>net8.0</TargetFramework>
+    <TargetFramework>net7.0</TargetFramework>
    <ImplicitUsings>enable</ImplicitUsings>
    <Nullable>enable</Nullable>
  </PropertyGroup>

  <ItemGroup>
    <PackageReference Include="Google.Protobuf" Version="3.24.3" />
-    <PackageReference Include="YaccLexTools" Version="1.2.3" />
+    <PackageReference Include="YaccLexTools" Version="1.2.0" />
  </ItemGroup>

  <ItemGroup>
@@ -24,6 +24,8 @@ namespace QUT.GplexBuffers
        public BufferException(string message) : base(message) { }
        public BufferException(string message, Exception innerException)
            : base(message, innerException) { }
+        protected BufferException(SerializationInfo info, StreamingContext context)
+            : base(info, context) { }
    }

    internal abstract class ScanBuff
@@ -58,10 +60,12 @@ namespace QUT.GplexBuffers
            return new BuildBuffer(source);
        }

+#if (!BYTEMODE)
        public static ScanBuff GetBuffer(Stream source, int fallbackCodePage)
        {
            return new BuildBuffer(source, fallbackCodePage);
        }
+#endif // !BYTEMODE
 #endif // !NOFILES
    }

@@ -349,15 +353,17 @@ namespace QUT.GplexBuffers
        {
            FileStream fStrm = (stream as FileStream);
            if (fStrm != null) FileName = fStrm.Name;
-            NextBlk = BlockReaderFactory.Get(stream);
+            NextBlk = BlockReaderFactory.Raw(stream);
        }

+#if (!BYTEMODE)
        public BuildBuffer(Stream stream, int fallbackCodePage)
        {
            FileStream fStrm = (stream as FileStream);
            if (fStrm != null) FileName = fStrm.Name;
            NextBlk = BlockReaderFactory.Get(stream, fallbackCodePage);
        }
+#endif

        /// <summary>
        /// Marks a conservative lower bound for the buffer,
@@ -428,19 +434,43 @@ namespace QUT.GplexBuffers
    //
    internal static class BlockReaderFactory
    {
+        public static BlockReader Raw(Stream stream)
+        {
+            return delegate(char[] block, int index, int number)
+            {
+                byte[] b = new byte[number];
+                int count = stream.Read(b, 0, number);
+                int i = 0;
+                int j = index;
+                for (; i < count; i++, j++)
+                    block[j] = (char)b[i];
+                return count;
+            };
+        }

-        public static BlockReader Get(Stream stream, int fallbackCodePage = -2)
+#if (!BYTEMODE)
+        public static BlockReader Get(Stream stream, int fallbackCodePage)
        {
            Encoding encoding;
            int preamble = Preamble(stream);

            if (preamble != 0)  // There is a valid BOM here!
                encoding = Encoding.GetEncoding(preamble);
-            else if (fallbackCodePage >= 0) // Anything but "guess"
+            else if (fallbackCodePage == -1) // Fallback is "raw" bytes
+                return Raw(stream);
+            else if (fallbackCodePage != -2) // Anything but "guess"
                encoding = Encoding.GetEncoding(fallbackCodePage);
            else // This is the "guess" option
-                encoding = GuessEncoding(stream);
-
+            {
+                int guess = new Guesser(stream).GuessCodePage();
+                stream.Seek(0, SeekOrigin.Begin);
+                if (guess == -1) // ==> this is a 7-bit file
+                    encoding = Encoding.ASCII;
+                else if (guess == 65001)
+                    encoding = Encoding.UTF8;
+                else             // ==> use the machine default
+                    encoding = Encoding.Default;
+            }
            StreamReader reader = new StreamReader(stream, encoding);
            return reader.Read;
        }
@@ -458,14 +488,6 @@ namespace QUT.GplexBuffers
            int b2 = stream.ReadByte();
            if (b0 == 0xef && b1 == 0xbb && b2 == 0xbf)
                return 65001; // UTF8
-
-            int b3 = stream.ReadByte();
-            // Check for UTF-32 BOM (Byte Order Mark)
-            if (b0 == 0x00 && b1 == 0x00 && b2 == 0xFE && b3 == 0xFF)
-                return 12001; // UTF-32BE
-            if (b0 == 0xFF && b1 == 0xFE && b2 == 0x00 && b3 == 0x00)
-                return 12000; // UTF-32LE
-
            //
            // There is no unicode preamble, so we
            // return denoter for the machine default.
@@ -473,150 +495,7 @@ namespace QUT.GplexBuffers
            stream.Seek(0, SeekOrigin.Begin);
            return 0;
        }
-
-        private static Encoding GuessEncoding(Stream stream)
-        {
-            stream.Seek(0, SeekOrigin.Begin);
-            if (IsUtf16(stream, out Encoding encoding))
-            {
-                stream.Seek(0, SeekOrigin.Begin);
-                return encoding;
-            }
-
-            stream.Seek(0, SeekOrigin.Begin);
-            if (IsUtf8(stream))
-            {
-                stream.Seek(0, SeekOrigin.Begin);
-                return Encoding.UTF8;
-            }
-
-            stream.Seek(0, SeekOrigin.Begin);
-            if (IsAscii(stream))
-            {
-                stream.Seek(0, SeekOrigin.Begin);
-                return Encoding.ASCII;
-            }
-
-#if !NET462_OR_GREATER
-            stream.Seek(0, SeekOrigin.Begin);
-            if (IsLatin1(stream))
-            {
-                stream.Seek(0, SeekOrigin.Begin);
-                return Encoding.Latin1;
-            }
-#endif
-            stream.Seek(0, SeekOrigin.Begin);
-            return Encoding.Default;
-        }
-
-        private static bool IsUtf8(Stream stream)
-        {
-            int b;
-            while ((b = stream.ReadByte()) != -1)
-            {
-                if ((b & 0x80) == 0) // 0xxxxxxx
-                {
-                    continue;
-                }
-                else if ((b & 0xE0) == 0xC0) // 110xxxxx 10xxxxxx
-                {
-                    if (!IsValidUtf8ContinuationByte(stream)) return false;
-                }
-                else if ((b & 0xF0) == 0xE0) // 1110xxxx 10xxxxxx 10xxxxxx
-                {
-                    if (!IsValidUtf8ContinuationByte(stream) || !IsValidUtf8ContinuationByte(stream)) return false;
-                }
-                else if ((b & 0xF8) == 0xF0) // 11110xxx 10xxxxxx 10xxxxxx 10xxxxxx
-                {
-                    if (!IsValidUtf8ContinuationByte(stream) || !IsValidUtf8ContinuationByte(stream) || !IsValidUtf8ContinuationByte(stream)) return false;
-                }
-                else
-                {
-                    return false;
-                }
-            }
-            return true;
-        }
-
-        private static bool IsValidUtf8ContinuationByte(Stream stream)
-        {
-            int b = stream.ReadByte();
-            return b != -1 && (b & 0xC0) == 0x80; // 10xxxxxx
-        }
-
-        private static bool IsUtf16(Stream stream, out Encoding encoding)
-        {
-            encoding = Encoding.Default;
-
-            if (stream.Length >= 2)
-            {
-                // Read the first two bytes
-                int b0 = stream.ReadByte();
-                int b1 = stream.ReadByte();
-
-                // Check for UTF-16 BOM (Byte Order Mark)
-                if (b0 == 0xFF && b1 == 0xFE)
-                {
-                    encoding = Encoding.Unicode;
-                    return true; // UTF-16LE
-                }
-                if (b0 == 0xFE && b1 == 0xFF)
-                {
-                    encoding = Encoding.BigEndianUnicode;
-                    return true; // UTF-16BE
-                }
-
-                // Reset the stream position
-                stream.Seek(0, SeekOrigin.Begin);
-
-                // Check for common UTF-16 patterns
-                byte[] buffer = new byte[2];
-                while (stream.Read(buffer, 0, 2) == 2)
-                {
-                    if (buffer[0] == 0x00 && buffer[1] != 0x00)
-                    {
-                        encoding = Encoding.BigEndianUnicode;
-                        return true; // Likely UTF-16BE
-                    }
-                    if (buffer[0] != 0x00 && buffer[1] == 0x00)
-                    {
-                        encoding = Encoding.Unicode;
-                        return true; // Likely UTF-16LE
-                    }
-                }
-            }
-
-            return false;
-        }
-
-        private static bool IsAscii(Stream stream)
-        {
-            int b;
-            while ((b = stream.ReadByte()) != -1)
-            {
-                // Latin1 includes byte values from 0x00 to 0xFF
-                if (b > 0x7F)
-                {
-                    return false;
-                }
-            }
-            return true;
-        }
-
-        private static bool IsLatin1(Stream stream)
-        {
-            int b;
-            while ((b = stream.ReadByte()) != -1)
-            {
-                // Latin1 includes byte values from 0x00 to 0xFF
-                if (b > 0xFF)
-                {
-                    return false;
-                }
-            }
-            return true;
-        }
-
+#endif // !BYTEMODE
    }
 #endif // !NOFILES
    #endregion Buffer classes
@@ -661,6 +540,180 @@ namespace QUT.GplexBuffers
            return 0;
        }
    }
+#region guesser
+#if (!BYTEMODE)
+    // ==============================================================
+    // ============          Encoding Guesser           =============
+    // ==============================================================
+
+    /// <summary>
+    /// This class provides a simple finite state automaton that
+    /// scans the file looking for (1) valid UTF-8 byte patterns,
+    /// (2) bytes >= 0x80 which are not part of a UTF-8 sequence.
+    /// The method then guesses whether it is UTF-8 or maybe some 
+    /// local machine default encoding.  This works well for the
+    /// various Latin encodings.
+    /// </summary>
+    internal class Guesser
+    {
+        ScanBuff buffer;
+
+        public int GuessCodePage() { return Scan(); }
+
+        const int maxAccept = 10;
+        const int initial = 0;
+        const int eofNum = 0;
+        const int goStart = -1;
+        const int INITIAL = 0;
+        const int EndToken = 0;
+
+        #region user code
+        /* 
+         *  Reads the bytes of a file to determine if it is 
+         *  UTF-8 or a single-byte code page file.
+         */
+        public long utfX;
+        public long uppr;
+        #endregion user code
+
+        int state;
+        int currentStart = startState[0];
+        int code;
+
+        #region ScannerTables
+        static int[] startState = new int[] { 11, 0 };
+
+        #region CharacterMap
+        static sbyte[] map = new sbyte[256] {
+/*     '\0' */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 
+/*   '\x10' */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 
+/*   '\x20' */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 
+/*      '0' */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 
+/*      '@' */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 
+/*      'P' */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 
+/*      '`' */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 
+/*      'p' */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 
+/*   '\x80' */ 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 
+/*   '\x90' */ 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 
+/*   '\xA0' */ 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 
+/*   '\xB0' */ 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 
+/*   '\xC0' */ 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 
+/*   '\xD0' */ 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 
+/*   '\xE0' */ 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 
+/*   '\xF0' */ 4, 4, 4, 4, 4, 4, 4, 4, 5, 5, 5, 5, 5, 5, 5, 5 };
+        #endregion
+
+        static sbyte[][] nextState = new sbyte[][] {
+            new sbyte[] {0, 0, 0, 0, 0, 0},
+            new sbyte[] {-1, -1, 10, -1, -1, -1},
+            new sbyte[] {-1, -1, -1, -1, -1, -1},
+            new sbyte[] {-1, -1, 8, -1, -1, -1},
+            new sbyte[] {-1, -1, 5, -1, -1, -1},
+            new sbyte[] {-1, -1, 6, -1, -1, -1},
+            new sbyte[] {-1, -1, 7, -1, -1, -1},
+            null,
+            new sbyte[] {-1, -1, 9, -1, -1, -1},
+            null,
+            null,
+            new sbyte[] {-1, 1, 2, 3, 4, 2}
+        };
+
+
+        [SuppressMessage("Microsoft.Performance", "CA1810:InitializeReferenceTypeStaticFieldsInline")]
+        // Reason for suppression: cannot have self-reference in array initializer.
+        static Guesser()
+        {
+            nextState[7] = nextState[2];
+            nextState[9] = nextState[2];
+            nextState[10] = nextState[2];
+        }
+
+        int NextState()
+        {
+            if (code == ScanBuff.EndOfFile)
+                return eofNum;
+            else
+                return nextState[state][map[code]];
+        }
+        #endregion
+
+        public Guesser(System.IO.Stream file) { SetSource(file); }
+
+        public void SetSource(System.IO.Stream source)
+        {
+            this.buffer = new BuildBuffer(source);
+            code = buffer.Read();
+        }
+
+        int Scan()
+        {
+            for (; ; )
+            {
+                int next;
+                state = currentStart;
+                while ((next = NextState()) == goStart)
+                    code = buffer.Read();
+
+                state = next;
+                code = buffer.Read();
+
+                while ((next = NextState()) > eofNum)
+                {
+                    state = next;
+                    code = buffer.Read();
+                }
+                if (state <= maxAccept)
+                {
+                    #region ActionSwitch
+#pragma warning disable 162
+                    switch (state)
+                    {
+                        case eofNum:
+                            switch (currentStart)
+                            {
+                                case 11:
+                                    if (utfX == 0 && uppr == 0) return -1; /* raw ascii */
+                                    else if (uppr * 10 > utfX) return 0;   /* default code page */
+                                    else return 65001;                     /* UTF-8 encoding */
+                                    break;
+                            }
+                            return EndToken;
+                        case 1: // Recognized '{Upper128}',	Shortest string "\xC0"
+                        case 2: // Recognized '{Upper128}',	Shortest string "\x80"
+                        case 3: // Recognized '{Upper128}',	Shortest string "\xE0"
+                        case 4: // Recognized '{Upper128}',	Shortest string "\xF0"
+                            uppr++;
+                            break;
+                        case 5: // Recognized '{Utf8pfx4}{Utf8cont}',	Shortest string "\xF0\x80"
+                            uppr += 2;
+                            break;
+                        case 6: // Recognized '{Utf8pfx4}{Utf8cont}{2}',	Shortest string "\xF0\x80\x80"
+                            uppr += 3;
+                            break;
+                        case 7: // Recognized '{Utf8pfx4}{Utf8cont}{3}',	Shortest string "\xF0\x80\x80\x80"
+                            utfX += 3;
+                            break;
+                        case 8: // Recognized '{Utf8pfx3}{Utf8cont}',	Shortest string "\xE0\x80"
+                            uppr += 2;
+                            break;
+                        case 9: // Recognized '{Utf8pfx3}{Utf8cont}{2}',	Shortest string "\xE0\x80\x80"
+                            utfX += 2;
+                            break;
+                        case 10: // Recognized '{Utf8pfx2}{Utf8cont}',	Shortest string "\xC0\x80"
+                            utfX++;
+                            break;
+                        default:
+                            break;
+                    }
+#pragma warning restore 162
+                    #endregion
+                }
+            }
+        }
+    } // end class Guesser
+    
+#endif // !BYTEMODE
+#endregion
 #endif // !NOFILES

 // End of code copied from embedded resource
@@ -1,7 +1,7 @@
 <Project Sdk="Microsoft.NET.Sdk">

  <PropertyGroup>
-    <TargetFramework>net8.0</TargetFramework>
+    <TargetFramework>net7.0</TargetFramework>
    <ImplicitUsings>enable</ImplicitUsings>
    <Nullable>enable</Nullable>
  </PropertyGroup>
@@ -1,7 +1,7 @@
 <Project Sdk="Microsoft.NET.Sdk">

  <PropertyGroup>
-    <TargetFramework>net8.0</TargetFramework>
+    <TargetFramework>net7.0</TargetFramework>
    <ImplicitUsings>enable</ImplicitUsings>
    <Nullable>enable</Nullable>
  </PropertyGroup>
@@ -1,7 +1,7 @@
 <Project Sdk="Microsoft.NET.Sdk">

  <PropertyGroup>
-    <TargetFramework>net8.0</TargetFramework>
+    <TargetFramework>net7.0</TargetFramework>
    <ImplicitUsings>enable</ImplicitUsings>
    <Nullable>enable</Nullable>
  </PropertyGroup>
@@ -1,7 +1,7 @@
 <Project Sdk="Microsoft.NET.Sdk">

  <PropertyGroup>
-    <TargetFramework>net8.0</TargetFramework>
+    <TargetFramework>net7.0</TargetFramework>
    <Nullable>enable</Nullable>
  </PropertyGroup>

@@ -74,7 +74,7 @@ namespace BCG {
      }

      public void _on_stream_cb(object dp_obj) {
-         //Log.Gen.info($"NonIncJoinBookkeeper._on_stream_cb dp_obj: {dp_obj}");
+         Log.Gen.info($"NonIncJoinBookkeeper._on_stream_cb dp_obj: {dp_obj}");
         var timer = Stopwatch.StartNew();
         DataPacket dp = (DataPacket)dp_obj;
         _fill_missing_data(dp);
@@ -50,7 +50,7 @@ namespace BCG {
         int data_col_no = 1;
         for (int jp = 0; jp < _join_specs.Length; jp++) {
            NonIncJoinBookkeeper s = new NonIncJoinBookkeeper(_join_specs[jp], jp, data_col_no, _row_hasher);
-            //Log.Gen.info($"NonIncJoinStream.myOnFirstSubscriber jp: {jp} spec: {_join_specs[jp]}");
+            Log.Gen.info($"NonIncJoinStream.myOnFirstSubscriber jp: {jp} spec: {_join_specs[jp]}");
            data_col_no += _join_specs[jp].data_col_names.Length;
            s.on_new_data += _stream_callback;
            _book_keeping_data.Add(s);
@@ -71,7 +71,7 @@ namespace BCG {
      }

      private void _stream_callback() {
-         //Log.Gen.info("NonIncJoinStream._stream_callback received callback");
+         Log.Gen.info("NonIncJoinStream._stream_callback received callback");
         _is_dirty = true;
         doWork();
      }
@@ -99,7 +99,7 @@ namespace BCG {

      private int _n_total_fires = 0, _n_expensive_fires = 0;
      private void _form_and_fire() {
-         //Log.Gen.info($"NonIncJoinStream._form_and_fire _is_ready(): {_is_ready()}");
+         Log.Gen.info($"NonIncJoinStream._form_and_fire _is_ready(): {_is_ready()}");
         if (_is_ready()) {
            _construct_schema();
            var timer = Stopwatch.StartNew();
@@ -126,11 +126,11 @@ namespace BCG {
            _n_total_fires++;
            //if (timer.ElapsedMilliseconds >= 250) {
            _n_expensive_fires++;
-            //string log_msg = $"Join and map time for {_book_keeping_data.Count} "
-            //   + $"tables {timer.Elapsed.TotalMilliseconds} msecs, ratio {_n_expensive_fires}/{_n_total_fires}";
-            //Log.Gen.info(log_msg);
+            string log_msg = $"Join and map time for {_book_keeping_data.Count} "
+               + $"tables {timer.Elapsed.TotalMilliseconds} msecs, ratio {_n_expensive_fires}/{_n_total_fires}";
+            Log.Gen.info(log_msg);
            //}
-            var dp_out = BlueDataUtility.build_dp_inc(dp_output_rows, myLastValue as DataPacket, true);
+            var dp_out = BlueDataUtility.build_dp_inc(dp_output_rows, myLastValue as DataPacket);
            myFireEvent(dp_out);
            //myFireEvent(new DataPacket(dp_output_rows, dp_output_rows, myLastValue == null ? new BlueTable(_output_schema) : (myLastValue as DataPacket).newRows));
         }
@@ -152,7 +152,7 @@ namespace BCG {
      }

      public void doWork() {
-         //Log.Gen.info($"NonIncJoinStream.doWork _last_fire_time: {_last_fire_time} _is_dirty: {_is_dirty}");
+         Log.Gen.info($"NonIncJoinStream.doWork _last_fire_time: {_last_fire_time} _is_dirty: {_is_dirty}");
         if (DateTime.Now - _last_fire_time >= _throttle_period && _is_dirty) {
            _is_dirty = false;
            _last_fire_time = DateTime.Now;
@@ -4,7 +4,7 @@ namespace bts.SilverPine {
   using BCG;

   using System;
-   [DataStreamUserRespresentation("DateTimeExpanderStream")]
+
   public class DateTimeExpanstionStream : ViewStream {
      public DateTimeExpanstionStream(ViewStream parent_stream, string inp_col) {
         Func<IBlueTableSchema, IBlueTableSchema> get_schema = (s) => {
@@ -24,11 +24,11 @@ namespace bts.SilverPine {
            IBlueTableSchema s = get_schema(r.schema);
            MutableBlueRow rv = new MutableBlueRow(s, r);
            rv.setValue($"{inp_col}_date", dt.Date);
-            rv.setValue($"{inp_col}_day_of_week", dt.DayOfWeek.ToString());
+            rv.setValue($"{inp_col}_day", dt.DayOfWeek.ToString());
            rv.setValue($"{inp_col}_day_of_month", dt.Day);
            rv.setValue($"{inp_col}_month", dt.Date.Month);
            rv.setValue($"{inp_col}_hour", dt.Hour);
-            rv.setValue($"{inp_col}_minute", dt.Minute);
+            rv.setValue($"{inp_col}_min", dt.Minute);
            rv.setValue($"{inp_col}_seconds_of_day", (dt - dt.Date).TotalSeconds);
            return rv;
         });
@@ -37,42 +37,4 @@ namespace BCG
      }
         
   }
-
-   [DataStreamUserRespresentation("throttle")]
-   public class ThrottleStream : ViewStream{
-      private readonly ViewStream _main_stream;
-      private readonly TimeSpan _ts;
-      private readonly ViewStreamDatabase _vsd;
-      private DateTime _last_fire_time = DateTime.MinValue;
-
-      public ThrottleStream (ViewStream main_stream, int throttle_seconds)
-      :
-      this(main_stream, TimeSpan.FromSeconds(throttle_seconds), ViewStreamDatabase.instance)
-      {
-      }
-
-      public ThrottleStream (ViewStream main_stream, TimeSpan ts, ViewStreamDatabase vsd){
-         _main_stream = main_stream;
-         _ts = ts;
-         _vsd = vsd;
-      }
-
-       protected override void myOnFirstSubscriber() {
-         _main_stream.onChange += _handle_base_stream_update;
-      }
-
-      protected override void myOnLastSubscriber() {
-         _main_stream.onChange -= _handle_base_stream_update;
-      }
-
-      private void _handle_base_stream_update(object dp_obj){
-         if (DateTime.Now - _last_fire_time >= _ts){
-            DataPacket dp_in = dp_obj as DataPacket;
-            _last_fire_time = DateTime.Now;
-            DataPacket dp = BlueDataUtility.build_dp_inc(dp_in.newRows, myLastValue as DataPacket);
-            myFireEvent(dp);
-         }
-      }
- }
-
 }
@@ -1,40 +0,0 @@
-namespace bts.SilverPine{
-   using BGC;
-   using System;
-   [DataStreamUserRespresentation("throttle")]
-   public class ThrottleStream : ViewStream{
-      private readonly ViewStream _main_stream;
-      private readonly TimeSpan _ts;
-      private readonly ViewStreamDatabase _vsd;
-      private DateTime _last_fire_time = DateTime.MinValue;
-
-      public ThrottleStream (ViewStream main_stream, int throttle_seconds)
-      :
-      this(main_stream, TimeSpan.FromSeconds(throttle_seconds), ViewStreamDatabase.instance)
-      {
-      }
-
-      public ThrottleStream (ViewStream main_stream, TimeSpan ts, ViewStreamDatabase vsd){
-         _main_stream = main_stream;
-         _ts = ts;
-         _vsd;
-      }
-
-       protected override void myOnFirstSubscriber() {
-         _main_stream.onChange += _handle_base_stream_update;
-      }
-
-      protected override void myOnLastSubscriber() {
-         _main_stream.onChange -= _handle_base_stream_update;
-      }
-
-      private void _handle_base_stream_update(object dp_obj){
-         if (DateTime.Now - _last_fire_time >= _ts){
-            DataPacket dp_in = dp_obj;
-            _last_fire_time = DateTime.Now;
-            DataPacket dp = BlueDataUtility.build_dp_inc(dp_in.newRow, myLastValue as DataPacket);
-            myFireEvent(dp);
-         }
-      }
- }
-}
@@ -2,7 +2,7 @@

  <PropertyGroup>
    <AllowUnsafeBlocks>true</AllowUnsafeBlocks>
-    <TargetFramework>net8.0</TargetFramework>
+    <TargetFramework>net7.0</TargetFramework>
    <Nullable>disable</Nullable>
  </PropertyGroup>

@@ -19,8 +19,8 @@
  </ItemGroup>

  <ItemGroup>
-    <Proto Include="../../lib_spark/spark/config/protos/data_stream_config.proto" CSName="DataStreamConfig">
-      <OutDir>$(ProtoOutputPath)/lib_spark/spark/config/protos</OutDir>
+    <Proto Include="../../lib/config/protos/data_stream_config.proto" CSName="DataStreamConfig">
+      <OutDir>$(ProtoOutputPath)/lib/config/protos</OutDir>
    </Proto>
  </ItemGroup>

@@ -1,7 +1,7 @@
 <Project Sdk="Microsoft.NET.Sdk">

  <PropertyGroup>
-    <TargetFramework>net8.0</TargetFramework>
+    <TargetFramework>net7.0</TargetFramework>
    <ImplicitUsings>enable</ImplicitUsings>
    <Nullable>enable</Nullable>
  </PropertyGroup>
@@ -2,7 +2,7 @@

  <PropertyGroup>
    <OutputType>Exe</OutputType>
-    <TargetFramework>net8.0</TargetFramework>
+    <TargetFramework>net7.0</TargetFramework>
    <Nullable>enable</Nullable>
    <AllowUnsafeBlocks>true</AllowUnsafeBlocks>
  </PropertyGroup>
@@ -2,7 +2,7 @@

  <PropertyGroup>
    <OutputType>Exe</OutputType>
-    <TargetFramework>net8.0-windows</TargetFramework>
+    <TargetFramework>net7.0-windows</TargetFramework>
    <Nullable>enable</Nullable>
    <UseWindowsForms>true</UseWindowsForms>
  </PropertyGroup>
@@ -36,8 +36,6 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Bts.ColumnExpressions", "Bt
 EndProject
 Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Bts.ColumnSortExpressions", "Bts.ColumnSortExpressions\Bts.ColumnSortExpressions.csproj", "{B038CB72-4E81-4AF1-976F-ED8655C2C3D6}"
 EndProject
-Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "DataStreamQueryAnalyzer", "DataStreamQueryAnalyzer\DataStreamQueryAnalyzer.csproj", "{BFA5BA5C-927A-41A9-A696-1E2F4B5FFD15}"
-EndProject
 Global
 	GlobalSection(SolutionConfigurationPlatforms) = preSolution
 		Debug|Any CPU = Debug|Any CPU
@@ -115,9 +113,5 @@ Global
 		{B038CB72-4E81-4AF1-976F-ED8655C2C3D6}.Debug|Any CPU.Build.0 = Debug|Any CPU
 		{B038CB72-4E81-4AF1-976F-ED8655C2C3D6}.Release|Any CPU.ActiveCfg = Release|Any CPU
 		{B038CB72-4E81-4AF1-976F-ED8655C2C3D6}.Release|Any CPU.Build.0 = Release|Any CPU
-		{BFA5BA5C-927A-41A9-A696-1E2F4B5FFD15}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
-		{BFA5BA5C-927A-41A9-A696-1E2F4B5FFD15}.Debug|Any CPU.Build.0 = Debug|Any CPU
-		{BFA5BA5C-927A-41A9-A696-1E2F4B5FFD15}.Release|Any CPU.ActiveCfg = Release|Any CPU
-		{BFA5BA5C-927A-41A9-A696-1E2F4B5FFD15}.Release|Any CPU.Build.0 = Release|Any CPU
 	EndGlobalSection
 EndGlobal
@@ -1,7 +1,7 @@
 <Project Sdk="Microsoft.NET.Sdk">

  <PropertyGroup>
-    <TargetFramework>net8.0</TargetFramework>
+    <TargetFramework>net7.0</TargetFramework>
    <ImplicitUsings>enable</ImplicitUsings>
    <Nullable>enable</Nullable>

@@ -1,7 +1,7 @@
 <Project Sdk="Microsoft.NET.Sdk">

  <PropertyGroup>
-    <TargetFramework>net8.0</TargetFramework>
+    <TargetFramework>net7.0</TargetFramework>
    <ImplicitUsings>enable</ImplicitUsings>
    <Nullable>enable</Nullable>
  </PropertyGroup>
@@ -1,7 +1,7 @@
 <Project Sdk="Microsoft.NET.Sdk">

  <PropertyGroup>
-    <TargetFramework>net8.0-windows</TargetFramework>
+    <TargetFramework>net7.0-windows</TargetFramework>
    <ImplicitUsings>enable</ImplicitUsings>
    <Nullable>enable</Nullable>
    <UseWindowsForms>true</UseWindowsForms>
@@ -6,18 +6,12 @@ namespace BCG
 	public interface IPublisher
 	{
      void publish( IPubItem pi );
-#pragma warning disable SYSLIB0011 // Type or member is obsolete
      void publish( IPubItem pi, IFormatter f );
-#pragma warning restore SYSLIB0011 // Type or member is obsolete
      void publish( IPubItem[] pi );
-#pragma warning disable SYSLIB0011 // Type or member is obsolete
      void publish( IPubItem[] pi, IFormatter f );
-#pragma warning restore SYSLIB0011 // Type or member is obsolete
   }

-#pragma warning disable SYSLIB0011 // Type or member is obsolete
   public interface IPublisher<T> where T : IFormatter
-#pragma warning restore SYSLIB0011 // Type or member is obsolete
   {
      void publish (IPubItem pi);
      void publish (IPubItem[] pi);
@@ -5,21 +5,23 @@ using System.Runtime.Serialization.Formatters.Binary;

 namespace BCG
 {
-#pragma warning disable SYSLIB0011 // Type or member is obsolete
   public class GenericFormatter<F> : IGenericFormatter where F : IFormatter, new ()
   {
      IFormatter myFormatter = new F ();

      public T Deserialize<T> (Stream serializationStream)
      {
+         #pragma warning disable 
         return (T) myFormatter.Deserialize (serializationStream);
+         #pragma warning restore SYSLIB0011
      }
      public void Serialize<T> (Stream serializationStream, T graph)
      {
+         #pragma warning disable 
         myFormatter.Serialize (serializationStream, graph);
+         #pragma warning restore SYSLIB0011
      }
   }

   public class GenericBinaryFormatter : GenericFormatter<BinaryFormatter> { }
-#pragma warning restore SYSLIB0011 // Type or member is obsolete
 }
@@ -6,7 +6,7 @@

  <PropertyGroup>
      <AllowUnsafeBlocks>true</AllowUnsafeBlocks>
-    <TargetFramework>net8.0</TargetFramework>
+    <TargetFramework>net7.0</TargetFramework>
    <Nullable>enable</Nullable>
  </PropertyGroup>

@@ -2,7 +2,7 @@

  <PropertyGroup>
    <OutputType>Exe</OutputType>
-    <TargetFramework>net8.0</TargetFramework>
+    <TargetFramework>net7.0</TargetFramework>
    <Nullable>enable</Nullable>
  </PropertyGroup>

@@ -1,17 +0,0 @@
-namespace bts.SilverPine {
-   using BCG;
-   using System;
-
-   [DataStreamUserRespresentation("SpatialArbScoreStream")]
-   public class SpatialArbScoreStream : ViewStream {
-      public SpatialArbScoreStream(int window_sz_secs, int sample_period_sz)
-         : this(TimeSpan.FromSeconds(window_sz_secs), TimeSpan.FromSeconds(sample_period_sz), ViewStreamDatabase.instance) { }
-
-      public SpatialArbScoreStream(TimeSpan window_sz, TimeSpan sample_period, ViewStreamDatabase vsd) {
-         ViewStream rs = new SpatialArbStatsStream(window_sz, sample_period, vsd);
-         rs = new ThrottleStream(rs, TimeSpan.FromSeconds(1), vsd);
-         rs = new SortStream(rs, "-n_trade_ticks");
-         mySetRealSource(rs);
-      }
-   }
-}
@@ -1,113 +0,0 @@
-namespace bts.SilverPine {
-   using BCG;
-
-   using SharpCompress.Common.Rar;
-
-   using System;
-   using System.Collections.Generic;
-   using System.Linq;
-   using System.Text;
-   using System.Threading.Tasks;
-
-   // For all the pairs generated by SpatialFuturePairsStream, this stream 
-   // generates the statistics, by appending together a SpreadStatsStream
-   [DataStreamUserRespresentation("SpatialArbStatsStream")]
-   public class SpatialArbStatsStream : ViewStream{
-      private ViewStream _spatial_pairs_stream;
-      private ViewStreamDatabase _vsd;
-      private ViewStream _tot_str = null;
-      private HashSet<string> _key_map = new HashSet<string>();
-      private ViewStream _bbo_stream;
-      private readonly TimeSpan _window_sz, _sampling_period_sz;
-
-      public SpatialArbStatsStream(int window_sz_seconds, int sampling_period_sz_seconds)
-         : this(TimeSpan.FromSeconds(window_sz_seconds), TimeSpan.FromSeconds(sampling_period_sz_seconds), ViewStreamDatabase.instance) { }
-
-      public SpatialArbStatsStream(TimeSpan window_sz, TimeSpan sampling_period_sz, ViewStreamDatabase vsd) {
-         _vsd = vsd;
-         _window_sz = window_sz;
-         _sampling_period_sz = sampling_period_sz;
-         _spatial_pairs_stream = new SpatialFuturePairsStream(vsd);
-         _bbo_stream = new BBOStream(vsd);
-      }
-
-      protected override void myOnFirstSubscriber() {
-         _spatial_pairs_stream.onChange += _spatial_pairs_stream_onchange;
-      }
-
-      protected override void myOnLastSubscriber() {
-         _spatial_pairs_stream.onChange -= _spatial_pairs_stream_onchange;
-      }
-
-      private ViewStream _build_mapped_stream(ViewStream parent_str) {
-         Func<IBlueTableSchema, IBlueTableSchema> build_schema = (inp_schema) => {
-            IBlueTableSchema rv = new BlueTableSchema("SpreaddStats", new ColumnInfo("id", 0, TypeCode.Int32));
-            rv = rv.add_column(TypeCode.String, "spread");
-            rv = rv.add_column(TypeCode.String, "window");
-            // bids
-            rv = rv.add_column(TypeCode.Double, "bid_mean");
-            rv = rv.add_column(TypeCode.Double, "bid_stddev");
-            rv = rv.add_column(TypeCode.Double, "bid_skewness");
-            rv = rv.add_column(TypeCode.Double, "bid_kurtosis");
-            rv = rv.add_column(TypeCode.Double, "bid_max");
-            rv = rv.add_column(TypeCode.Double, "bid_min");
-            // asks
-            rv = rv.add_column(TypeCode.Double, "ask_mean");
-            rv = rv.add_column(TypeCode.Double, "ask_stddev");
-            rv = rv.add_column(TypeCode.Double, "ask_skewness");
-            rv = rv.add_column(TypeCode.Double, "ask_kurtosis");
-            rv = rv.add_column(TypeCode.Double, "ask_max");
-            rv = rv.add_column(TypeCode.Double, "ask_min");
-            // tail distribution
-            rv = rv.add_column(TypeCode.Double, "bid Z ≥ 2σ");
-            rv = rv.add_column(TypeCode.Double, "ask Z < -2σ");
-            // trading ticks
-            rv = rv.add_column(TypeCode.Int32, "n_buy_ticks");
-            rv = rv.add_column(TypeCode.Double, "cumm_buy_price");
-            rv = rv.add_column(TypeCode.Int32, "n_sell_ticks");
-            rv = rv.add_column(TypeCode.Double, "cumm_sell_price");
-            rv = rv.add_column(TypeCode.Int32, "n_trade_ticks");
-            return rv;
-         };
-         Func<IBlueRow, IBlueRow> build_row = (r) => {
-            IBlueTableSchema s = build_schema(r.schema);
-            return new BlueRow(s, r["id"], r["spread"], r["window"],
-                           r["bid_mean"], r["bid_stddev"], r["bid_skewness"], r["bid_kurtosis"], r["bid_max"], r["bid_min"],
-                           r["ask_mean"], r["ask_stddev"], r["ask_skewness"], r["ask_kurtosis"], r["ask_max"], r["ask_min"],
-                           r["bid Z ≥ 2σ"], r["ask Z < -2σ"],
-                           r["n_buy_ticks"], r["cumm_buy_price"], r["n_sell_ticks"], r["cumm_sell_price"], r["n_trade_ticks"]);
-         };
-         MapStream rs = new MapStream(parent_str, inp_schema => build_schema(inp_schema),
-            inp_r => build_row(inp_r));
-         return rs;
-      }
-
-      private void _spatial_pairs_stream_onchange(object dp_obj) {
-         if (_tot_str != null)
-            return;
-         MutableAppendStream append_str = new MutableAppendStream();
-         _tot_str = _build_mapped_stream(append_str);
-         //_tot_str = new ThrottleStream(_tot_str, 1000);
-         //_tot_str = new SortStream(_tot_str, "-n_trade_ticks");
-         _tot_str.onChange += _handle_tot_stream;
-
-         DataPacket dp = dp_obj as DataPacket;         
-         foreach (IBlueRow r in dp.newRows) {
-            string s1 = r.get<string>("s1");
-            string s2 = r.get<string>("s2");
-            string key = SpatialFuturePairsStream.get_key(s1, s2);
-            if (!_key_map.Contains(key)) {
-               _key_map.Add(key);
-               ViewStream spread_stats_stream = new SpreadStatsStream(s1, s2, 1, 1, Convert.ToInt32(_window_sz.TotalSeconds), 
-                                                                     Convert.ToInt32(_sampling_period_sz.TotalSeconds), _bbo_stream, _vsd);
-               append_str.addStream(spread_stats_stream);
-            }
-         }
-      }
-
-      private void _handle_tot_stream(object dp_obj) {
-         myFireEvent(dp_obj);
-      }
-   }
-
-}
@@ -1,89 +0,0 @@
-namespace bts.SilverPine {
-   using BCG;
-
-   using bts.api;
-
-   using System;
-   using System.Collections.Generic;
-   using System.Linq;
-   using System.Text;
-   using System.Threading.Tasks;
-
-   [DataStreamUserRespresentation("SpatialFuturePairsStream")]
-   public class SpatialFuturePairsStream : ViewStream {
-      private ViewStream _main_stream;
-      private IBlueTableSchema _schema;
-      private HashSet<string> _key_map = new HashSet<string>();
-      public SpatialFuturePairsStream()
-         : this(ViewStreamDatabase.instance) { }
-
-      public SpatialFuturePairsStream(ViewStreamDatabase vsd) {
-         _main_stream = new InstrumentsStream(vsd);
-         _main_stream = new FilterStream(_main_stream, "sec_type='Future'");
-         _schema = new BlueTableSchema("SpatialFutures", new ColumnInfo("id", 0, TypeCode.Int32));
-         _schema = _schema.add_column(TypeCode.String, "s1");
-         _schema = _schema.add_column(TypeCode.String, "venue_1");
-         _schema = _schema.add_column(TypeCode.String, "s2");
-         _schema = _schema.add_column(TypeCode.String, "venue_2");
-
-      }
-
-      protected override void myOnFirstSubscriber() {
-         _main_stream.onChange += _main_stream_onchange;
-      }
-
-      protected override void myOnLastSubscriber() {
-         _main_stream.onChange -= _main_stream_onchange;
-      }
-
-      public static string get_key(string s1, string s2) {
-         string key = string.Compare(s1, s2, StringComparison.Ordinal) <= 0 ? s1 + s2 : s2 + s1;
-         return key;
-      }
-
-      public static string get_key(IBlueRow r1, IBlueRow r2) {
-         string s1 = r1.get<string>("sec");
-         string s2 = r2.get<string>("sec");
-         return get_key(s1, s2);
-      }
-
-      private bool _combine(IBlueRow r1, IBlueRow r2) {
-         string key = get_key(r1, r2);
-         if (_key_map.Contains(key))
-            return false;
-         if (r1.get<string>("symbol") != r2.get<string>("symbol"))
-            return false;
-         if (r1.get_enum<Venue>("venue") == r2.get_enum<Venue>("venue"))
-            return false;
-         if (r1.get<int>("sec_key") == r2.get<int>("sec_key")) {
-            return false;
-         }
-         DateTime e1 = r1.get_datetime("expiration");
-         DateTime e2 = r2.get_datetime("expiration");
-         bool exps_eq = e1 == e2 || (e1.Year >= 2100 && e2.Year >= 2100);
-         if (!exps_eq)
-            return false;         
-         return true;
-      }
-
-      private void _main_stream_onchange(object dp_obj) {
-         _key_map.Clear();
-         DataPacket dp = dp_obj as DataPacket;
-         BlueTable table = new BlueTable(_schema);
-         var rows = dp.newRows.to_list();
-         foreach (var r1 in rows) {
-            foreach (var r2 in rows) {
-               if (_combine(r1, r2)) {
-                  IBlueRow nr = new BlueRow(_schema, table.Count, r1.get<string>("sec"), r1.get<string>("venue"),
-                     r2.get<string>("sec"), r2.get<string>("venue"));
-                  table.addRow(nr);
-                  string key = get_key(r1, r2);
-                  _key_map.Add(key);
-               }
-            }
-         }
-         var dp_out = BlueDataUtility.build_dp(table, myLastValue as DataPacket);
-         myFireEvent(dp_out);
-      }
-   }
-}
@@ -1,656 +0,0 @@
-using System;
-using System.Collections.Generic;
-using System.Linq;
-using MathNet.Numerics.Statistics;
-using MathNet.Numerics.LinearRegression;
-using System.Collections.Generic;
-using System.Linq;
-using MathNet.Numerics.Statistics;
-using MathNet.Numerics.LinearAlgebra;
-using Amazon.Runtime;
-using System;
-using System.Collections.Generic;
-using System.Linq;
-using MathNet.Numerics.Statistics;
-using System;
-using System.Collections.Generic;
-using System.Linq;
-using MathNet.Numerics.Statistics;
-using System;
-using System.Collections.Generic;
-using System.Linq;
-using MathNet.Numerics.Statistics;
-// stats calculator
-using System;
-using System.Collections.Generic;
-using System.Linq;
-using MathNet.Numerics.Statistics;
-
-/// <summary>
-/// Provides evaluation metrics to assess the quality of a synthetic spread for mean-reversion trading.
-/// Assumes the input spread data is a trailing, time-causal window of recent ticks (e.g., last 120 seconds).
-/// </summary>
-public static class SpreadPerformanceEvaluator
-{
-
-   public static class SpreadStatistics {
-      public class SpreadStatsResult {
-         public Dictionary<string, double> bid_zscore_distribution { get; set; }
-         public Dictionary<string, double> ask_zscore_distribution { get; set; }
-         public Dictionary<string, double> mid_zscore_distribution { get; set; }
-
-         public DescriptiveStatistics bid_stats { get; set; }
-         public DescriptiveStatistics ask_stats { get; set; }
-         public DescriptiveStatistics mid_stats { get; set; }
-      }
-
-      private static readonly string[] _zscore_labels = new[]
-      {
-        "Z < -2σ",
-        "-2σ ≤ Z < -1σ",
-        "-1σ ≤ Z < 0σ",
-        "0σ ≤ Z < 1σ",
-        "1σ ≤ Z < 2σ",
-        "Z ≥ 2σ"
-    };
-
-      public static SpreadStatsResult analyze_spread_quotes(List<(DateTime timestamp, double bid, double bid_qty, double ask, double ask_qty)> quotes) {
-         if (quotes == null || quotes.Count == 0)
-            throw new ArgumentException("Spread quote list cannot be null or empty.");
-
-         var bid_prices = quotes.Select(q => q.bid).ToList();
-         var ask_prices = quotes.Select(q => q.ask).ToList();
-         var mid_prices = quotes.Select(q => (q.bid + q.ask) / 2.0).ToList();
-
-         var bid_stats = new DescriptiveStatistics(bid_prices);
-         var ask_stats = new DescriptiveStatistics(ask_prices);
-         var mid_stats = new DescriptiveStatistics(mid_prices);
-
-         var bid_zscores = bid_prices.Select(x => zscore(x, bid_stats.Mean, bid_stats.StandardDeviation)).ToList();
-         var ask_zscores = ask_prices.Select(x => zscore(x, ask_stats.Mean, ask_stats.StandardDeviation)).ToList();
-         var mid_zscores = mid_prices.Select(x => zscore(x, mid_stats.Mean, mid_stats.StandardDeviation)).ToList();
-
-         var bid_distribution = bin_zscores_to_percentages(bid_zscores);
-         var ask_distribution = bin_zscores_to_percentages(ask_zscores);
-         var mid_distribution = bin_zscores_to_percentages(mid_zscores);
-
-         return new SpreadStatsResult {
-            bid_zscore_distribution = bid_distribution,
-            ask_zscore_distribution = ask_distribution,
-            mid_zscore_distribution = mid_distribution,
-            bid_stats = bid_stats,
-            ask_stats = ask_stats,
-            mid_stats = mid_stats
-         };
-      }
-
-      private static double zscore(double x, double mean, double std_dev) {
-         return std_dev == 0 ? 0 : (x - mean) / std_dev;
-      }
-
-      private static Dictionary<string, double> bin_zscores_to_percentages(List<double> zscores) {
-         var bins = _zscore_labels.ToDictionary(label => label, _ => 0);
-
-         foreach (var z in zscores) {
-            if (z < -2) bins["Z < -2σ"]++;
-            else if (z < -1) bins["-2σ ≤ Z < -1σ"]++;
-            else if (z < 0) bins["-1σ ≤ Z < 0σ"]++;
-            else if (z < 1) bins["0σ ≤ Z < 1σ"]++;
-            else if (z < 2) bins["1σ ≤ Z < 2σ"]++;
-            else bins["Z ≥ 2σ"]++;
-         }
-
-         double total = zscores.Count;
-         return bins.ToDictionary(kvp => kvp.Key, kvp => total == 0 ? 0 : (kvp.Value / total) * 100.0);
-      }
-
-      public static double score_spread(SpreadStatsResult result) {
-         var ask_bins = result.ask_zscore_distribution;
-         var bid_bins = result.bid_zscore_distribution;
-         var mid_bins = result.mid_zscore_distribution;
-
-         double buy_opportunity = ask_bins["Z < -2σ"] + ask_bins["-2σ ≤ Z < -1σ"];
-         double sell_opportunity = bid_bins["Z ≥ 2σ"] + bid_bins["1σ ≤ Z < 2σ"];
-         double opportunity_score = buy_opportunity + sell_opportunity;
-
-         double below_mean = mid_bins["Z < -2σ"] + mid_bins["-2σ ≤ Z < -1σ"] + mid_bins["-1σ ≤ Z < 0σ"];
-         double above_mean = mid_bins["Z ≥ 2σ"] + mid_bins["1σ ≤ Z < 2σ"] + mid_bins["0σ ≤ Z < 1σ"];
-         double symmetry_score = 1.0 - Math.Abs(below_mean - above_mean) / (below_mean + above_mean + 1e-6);
-
-         double amplitude_score = result.mid_stats.StandardDeviation;
-         double skew_penalty = 1.0 / (1.0 + Math.Abs(result.mid_stats.Skewness));
-         double kurtosis_bonus = Math.Min(result.mid_stats.Kurtosis, 6.0) / 6.0;
-
-         double score =
-             opportunity_score *
-             symmetry_score *
-             amplitude_score *
-             skew_penalty *
-             (1.0 + kurtosis_bonus * 0.1);
-
-         return score;
-      }
-   }
-
-
-
-
-
-/// <summary>
-/// Represents a single trade action (entry or exit) taken during the simulation.
-/// </summary>
-public struct TradeAction
-    {
-        public bool IsEntry { get; set; }    // True if this action opened a position
-        public bool IsLong { get; set; }     // True = Long, False = Short
-        public double Price { get; set; }    // Executed price
-        public int Index { get; set; }       // Tick index of the action
-
-        public override string ToString()
-        {
-            string side = IsLong ? "Long" : "Short";
-            string type = IsEntry ? "Open" : "Close";
-            return $"{type} {side} @ {Price:F4} (Tick {Index})";
-        }
-    }
-
-    /// <summary>
-/// Simulates a mean-reversion strategy using rolling, causal mid-price stats.
-/// ✅ Entry:
-///   - Buy at ask when ask < rollingMean - entryThreshold × rollingStdDev
-///   - Sell at bid when bid > rollingMean + entryThreshold × rollingStdDev
-/// ✅ Exit:
-///   - Close when mid returns within rollingMean ± exitThreshold × rollingStdDev
-///   - Long exits at bid, short exits at ask
-/// This version uses only past and current data at each step (no forward-looking bias).
-/// </summary>
-public static (double TotalPnL, List<TradeAction> TradeActions, int BuyTurns, int SellTurns) EvaluateSimulatedPnL(
-    List<Tuple<double, double, double, double>> spreadData,
-    int statWindow = 30,
-    double entryThresholdMultiplier = 0.5,
-    double exitThresholdMultiplier = 0.1)
-{
-    var trades = new List<TradeAction>();
-    var mids = spreadData.Select(x => (x.Item1 + x.Item3) / 2.0).ToList();
-
-    if (spreadData.Count < statWindow)
-        return (0.0, trades, 0, 0);
-
-    bool positionOpen = false;
-    bool isLong = false;
-    double entryPrice = 0.0;
-    double pnl = 0.0;
-    int buyTurns = 0;
-    int sellTurns = 0;
-
-    for (int i = statWindow; i < spreadData.Count; i++)
-    {
-        double bid = spreadData[i].Item1;
-        double ask = spreadData[i].Item3;
-        double mid = mids[i];
-
-        // Rolling window
-        var window = mids.Skip(i - statWindow).Take(statWindow).ToList();
-        double mean = window.Average();
-        double stddev = Math.Sqrt(window.Select(x => Math.Pow(x - mean, 2)).Average());
-
-        double lowerThreshold = mean - entryThresholdMultiplier * stddev;
-        double upperThreshold = mean + entryThresholdMultiplier * stddev;
-        double exitBand = exitThresholdMultiplier * stddev;
-
-        if (!positionOpen)
-        {
-            if (ask < lowerThreshold)
-            {
-                positionOpen = true;
-                isLong = true;
-                entryPrice = ask;
-                trades.Add(new TradeAction { IsEntry = true, IsLong = true, Price = entryPrice, Index = i });
-            }
-            else if (bid > upperThreshold)
-            {
-                positionOpen = true;
-                isLong = false;
-                entryPrice = bid;
-                trades.Add(new TradeAction { IsEntry = true, IsLong = false, Price = entryPrice, Index = i });
-            }
-        }
-        else
-        {
-            if (Math.Abs(mid - mean) < exitBand)
-            {
-                double exitPrice = isLong ? bid : ask;
-                pnl += isLong ? (exitPrice - entryPrice) : (entryPrice - exitPrice);
-
-                trades.Add(new TradeAction { IsEntry = false, IsLong = isLong, Price = exitPrice, Index = i });
-
-                if (isLong) buyTurns++;
-                else sellTurns++;
-
-                positionOpen = false;
-            }
-        }
-    }
-
-    return (pnl, trades, buyTurns, sellTurns);
-}
-
-
-
-    /// <summary>
-    /// Estimates the Hurst exponent of the spread mid-prices.
-    /// Assumes the input is a trailing window of recent data.
-    /// Hurst < 0.5 implies mean-reversion; ideal is 0.35–0.45.
-    /// </summary>
-    public static double EvaluateHurstExponent(List<Tuple<double, double, double, double>> spreadData)
-    {
-        const int MinLength = 21;
-
-        if (spreadData == null || spreadData.Count < MinLength)
-            return double.NaN;
-
-        var spreadMids = spreadData.Select(x => (x.Item1 + x.Item3) / 2.0).ToArray();
-        int n = spreadMids.Length;
-
-        var lags = new List<double>();
-        var tau = new List<double>();
-
-        for (int lag = 2; lag <= 20; lag++)
-        {
-            if (n - lag <= 1)
-                continue;
-
-            double[] diffs = new double[n - lag];
-            for (int i = 0; i < diffs.Length; i++)
-                diffs[i] = spreadMids[i + lag] - spreadMids[i];
-
-            double stddev = diffs.StandardDeviation();
-
-            if (stddev > 0)
-            {
-                lags.Add(Math.Log(lag));
-                tau.Add(Math.Log(stddev));
-            }
-        }
-
-        if (lags.Count < 2)
-            return double.NaN;
-
-        var (intercept, slope) = SimpleRegression.Fit(lags.ToArray(), tau.ToArray());
-        double hurst = slope * 0.5 + 0.5;
-        return hurst;
-    }
-
-    /// <summary>
-/// Computes the mean and standard deviation of the spread mid-prices
-/// over a trailing, causal window.
-/// </summary>
-/// <param name="spreadData">List of (bid, bidQty, ask, askQty)</param>
-/// <returns>Tuple of (Mean, StdDev). Returns (NaN, NaN) if input is too short.</returns>
-public static (double Mean, double StdDev) EvaluateMeanAndStdDev(List<Tuple<double, double, double, double>> spreadData)
-{
-    if (spreadData == null || spreadData.Count < 2)
-        return (double.NaN, double.NaN);
-
-    var mids = spreadData.Select(x => (x.Item1 + x.Item3) / 2.0).ToList();
-    double mean = mids.Average();
-    double stddev = Math.Sqrt(mids.Select(x => Math.Pow(x - mean, 2)).Average());
-
-    return (mean, stddev);
-}
-
-
-   /// <summary>
-   /// Counts how many times the spread ask dropped below (mean - threshold) [buy signal]
-   /// and how many times the spread bid rose above (mean + threshold) [sell signal].
-   /// Assumes spreadData is a trailing, causal window.
-   /// </summary>
-   /// <returns>(BuyExcursions, SellExcursions)</returns>
-   public static (uint BuyExcursions, double cumm_buy_price, uint SellExcursions, double cumm_sell_price) EvaluateExcursionCount(
-       List<Tuple<double, double, double, double>> spreadData,
-       double entryThresholdMultiplier = 0.5) {
-      if (spreadData == null || spreadData.Count < 2)
-         return (0, Double.NaN, 0, Double.NaN);
-
-      var midPrices = spreadData.Select(x => (x.Item1 + x.Item3) / 2.0).ToList();
-      double mean = midPrices.Average();
-      double stddev = midPrices.StandardDeviation();
-      double buy_threshold = mean - entryThresholdMultiplier * stddev;
-      double sell_threshold = mean + entryThresholdMultiplier * stddev;
-
-      int buyCount = 0;
-      int sellCount = 0;
-      //bool in_buy_zone = false;
-      //bool in_sell_zone = false;
-
-      double cumm_buy_price = 0.0, cumm_sell_price = 0.0;
-      foreach (var (bid, _, ask, _) in spreadData) {
-         if (ask <= buy_threshold) {
-            //if (!in_buy_zone) {
-            cumm_buy_price += ask;
-            buyCount++;
-            //in_buy_zone = true;
-         }
-         //} else {
-         //   in_buy_zone = false;
-         //}
-
-         if (bid >= sell_threshold) {
-            //if (!in_sell_zone) {
-            cumm_sell_price += bid;
-            sellCount++;
-            //in_sell_zone = true;
-         }
-         //} else {
-         //   in_sell_zone = false;
-         //}
-      }
-
-      return ((uint)buyCount, cumm_buy_price / buyCount, (uint)sellCount, cumm_sell_price / sellCount);
-   }
-
-/// <summary>
-/// Counts how many times the spread mid-price crosses its mean level.
-/// Each crossing indicates a potential oscillation turn (cycle midpoint).
-/// Assumes the input is a trailing, causal window of spread ticks.
-/// </summary>
-/// <param name="spreadData">List of (bid, bidQty, ask, askQty)</param>
-/// <returns>Number of zero crossings in the mid-price series</returns>
-public static UInt32 EvaluateZeroCrossingCount(List<Tuple<double, double, double, double>> spreadData)
-{
-    if (spreadData == null || spreadData.Count < 2)
-        return 0;
-
-    var mids = spreadData.Select(x => (x.Item1 + x.Item3) / 2.0).ToList();
-    double mean = mids.Average();
-    int zeroCrossings = 0;
-
-    for (int i = 1; i < mids.Count; i++)
-    {
-        double prevOffset = mids[i - 1] - mean;
-        double currOffset = mids[i] - mean;
-
-        if (prevOffset * currOffset < 0) // Sign changed
-            zeroCrossings++;
-    }
-
-    return (UInt32)zeroCrossings;
-}
-
-   /// <summary>
-   /// Computes the half-life of mean reversion.
-   /// Lower half-life means the spread reverts to its mean more quickly.
-   /// </summary>
-   public static double ComputeHalfLife(List<Tuple<double, double, double, double>> spreadData) {
-      var prices = spreadData.Select(x => (x.Item1 + x.Item3) / 2.0).ToList();
-      var lagged = prices.Take(prices.Count - 1).ToArray();
-      var delta = prices.Skip(1).Zip(lagged, (cur, prev) => cur - prev).ToArray();
-
-      if (lagged.Length < 2) return double.NaN;
-
-      var x = Vector<double>.Build.Dense(lagged);
-      var y = Vector<double>.Build.Dense(delta);
-      var beta = x.DotProduct(y) / x.DotProduct(x);
-
-      return -Math.Log(2) / beta;
-   }
-
-   /// <summary>
-   /// Computes the lag-1 autocorrelation of the spread.
-   /// Lower values indicate quicker mean-reversion.
-   /// </summary>
-   public static double ComputeLag1Autocorrelation(List<Tuple<double, double, double, double>> spreadData) {
-      var prices = spreadData.Select(x => (x.Item1 + x.Item3) / 2.0).ToList();
-      var mean = prices.Average();
-      var numerator = 0.0;
-      var denominator = 0.0;
-
-      for (int i = 1; i < prices.Count; i++) {
-         numerator += (prices[i] - mean) * (prices[i - 1] - mean);
-         denominator += Math.Pow(prices[i - 1] - mean, 2);
-      }
-
-      return denominator == 0 ? 0 : numerator / denominator;
-   }
-
-   /// <summary>
-   /// Estimates how quickly extreme Z-score values decay back toward the mean.
-   /// Shorter average decay time indicates stronger mean-reversion.
-   /// </summary>
-   public static double ComputeAverageZScoreDecay(List<Tuple<double, double, double, double>> spreadData, int zScoreWindow = 50) {
-      var prices = spreadData.Select(x => (x.Item1 + x.Item3) / 2.0).ToList();
-      if (prices.Count < zScoreWindow + 2) return double.NaN;
-
-      var decayTimes = new List<int>();
-      for (int i = zScoreWindow; i < prices.Count - 1; i++) {
-         var window = prices.Skip(i - zScoreWindow).Take(zScoreWindow).ToList();
-         double mean = window.Average();
-         double stdDev = Statistics.StandardDeviation(window);
-         if (stdDev == 0) continue;
-
-         double z = (prices[i] - mean) / stdDev;
-         if (Math.Abs(z) > 2) {
-            for (int j = 1; j < prices.Count - i; j++) {
-               double zNext = (prices[i + j] - mean) / stdDev;
-               if (Math.Abs(zNext) < 1) {
-                  decayTimes.Add(j);
-                  break;
-               }
-            }
-         }
-      }
-
-      return decayTimes.Count == 0 ? double.NaN : decayTimes.Average();
-   }
-
-   /// <summary>
-   /// Computes the variance ratio: short-term variance divided by long-term variance.
-   /// Lower ratio suggests stronger mean-reversion.
-   /// </summary>
-   public static double ComputeVarianceRatio(List<Tuple<double, double, double, double>> spreadData, int shortWindow, int longWindow) {
-      var prices = spreadData.Select(x => (x.Item1 + x.Item3) / 2.0).ToList();
-      if (prices.Count < longWindow) return double.NaN;
-
-      double shortVar = Statistics.Variance(prices.Skip(prices.Count - shortWindow).Take(shortWindow));
-      double longVar = Statistics.Variance(prices.Skip(prices.Count - longWindow).Take(longWindow));
-
-      return longVar == 0 ? double.NaN : shortVar / longVar;
-   }
-
-   /// <summary>
-   /// Computes the highest bid and the lowest ask from a spread tick series.
-   /// Useful for evaluating the best market opportunity range.
-   /// </summary>
-   /// <param name="spreadData">List of (bid, bidQty, ask, askQty)</param>
-   /// <returns>Tuple of (MaxBid, MinAsk)</returns>
-   public static (double MaxBid, double MinAsk) EvaluateMaxBidMinAsk(List<Tuple<double, double, double, double>> spreadData)
-{
-    if (spreadData == null || spreadData.Count == 0)
-        return (double.NaN, double.NaN);
-
-    double maxBid = double.NegativeInfinity;
-    double minAsk = double.PositiveInfinity;
-
-    foreach (var (bid, _, ask, _) in spreadData)
-    {
-        if (bid > maxBid) maxBid = bid;
-        if (ask < minAsk) minAsk = ask;
-    }
-
-    return (maxBid, minAsk);
-}
-
-/// <summary>
-/// Computes the distribution of spread mid-prices around the mean,
-/// grouped by caller-defined distance bands (in standard deviations).
-///
-/// This helps answer:
-/// - Is the spread tightly hugging the mean?
-/// - Are there wide excursions or heavy tails?
-///
-/// You define breakpoints like: [0.5, 1.0, 2.0] → it will return:
-/// - "≤ 0.5σ"
-/// - "0.5σ – 1.0σ"
-/// - "1.0σ – 2.0σ"
-/// - "> 2.0σ"
-///
-/// How to Interpret It:
-/// - ≤ 0.5σ        → Tight clustering near the mean (strong mean-reversion)
-/// - 0.5σ – 1.0σ   → Mild oscillation (common trade region)
-/// - 1.0σ – 2.0σ   → Moderate deviation (potential trade signals)
-/// - > 2.0σ        → Rare, extreme moves (tail events, potential entries/exits)
-///
-/// Use this to tune entry thresholds or detect regime shifts.
-/// </summary>
-/// <param name="spreadData">List of (bid, bidQty, ask, askQty)</param>
-/// <param name="regions">List of positive breakpoints in std deviations (e.g., [0.5, 1.0, 2.0])</param>
-/// <returns>Dictionary mapping label to % of mid-prices in that region</returns>
-public static Dictionary<string, double> EvaluateDistributionAroundMean(
-    List<Tuple<double, double, double, double>> spreadData,
-    List<double> regions)
-{
-    var result = new Dictionary<string, double>();
-
-    if (spreadData == null || spreadData.Count < 2 || regions == null || regions.Count == 0)
-        return result;
-
-    var mids = spreadData.Select(x => (x.Item1 + x.Item3) / 2.0).ToList();
-    double mean = mids.Average();
-    double stddev = Math.Sqrt(mids.Select(x => Math.Pow(x - mean, 2)).Average());
-
-    if (stddev == 0)
-    {
-        result["Zero Volatility"] = 100.0;
-        return result;
-    }
-
-    var buckets = new int[regions.Count + 1];
-    foreach (var mid in mids)
-    {
-        double distance = Math.Abs(mid - mean);
-        bool placed = false;
-
-        for (int i = 0; i < regions.Count; i++)
-        {
-            if (distance <= regions[i])
-            {
-                buckets[i]++;
-                placed = true;
-                break;
-            }
-        }
-
-        if (!placed)
-            buckets[regions.Count]++;
-    }
-
-    int total = mids.Count;
-    for (int i = 0; i < regions.Count; i++)
-    {
-        string label = i == 0
-            ? $"≤ {regions[i]:0.##}σ"
-            : $"{regions[i - 1]:0.##}σ – {regions[i]:0.##}σ";
-        result[label] = 100.0 * buckets[i] / total;
-    }
-
-    result[$"> {regions.Last():0.##}σ"] = 100.0 * buckets.Last() / total;
-
-    return result;
-}
-
-/// <summary>
-/// Computes two drift metrics indicating how much the spread's mean changes over time,
-/// based on the standard deviation of rolling mean values.
-///
-/// ✅ Absolute Drift: Standard deviation of the rolling mean
-/// ✅ Relative Drift: Drift normalized by the overall mean (unitless ratio)
-///
-/// This shows whether the center of the spread is stable — critical for mean-reversion.
-///
-/// How to Interpret the Output:
-/// - ✅ Relative Drift < 1% → Very stable mean — ideal
-/// - ⚠️ 1–5% → Mild drift — consider shorter holds or adaptive mean
-/// - ❌ >5% → Drifting mean — fixed mean may no longer work
-/// </summary>
-/// <param name="spreadData">List of (bid, bidQty, ask, askQty)</param>
-/// <param name="window">Window size for rolling mean (e.g., 20 ticks)</param>
-/// <returns>
-/// Tuple of (Absolute Drift, Relative Drift) — where:
-/// - Absolute Drift = StdDev of rolling means
-/// - Relative Drift = Drift / global mean
-/// </returns>
-public static (double AbsoluteDrift, double RelativeDrift) EvaluateMeanDriftScore(
-    List<Tuple<double, double, double, double>> spreadData,
-    int window = 20)
-{
-    if (spreadData == null || spreadData.Count < window)
-        return (double.NaN, double.NaN);
-
-    var mids = spreadData.Select(x => (x.Item1 + x.Item3) / 2.0).ToArray();
-    var rollingMeans = new List<double>();
-
-    for (int i = window; i <= mids.Length; i++)
-    {
-        double rollingMean = mids.Skip(i - window).Take(window).Average();
-        rollingMeans.Add(rollingMean);
-    }
-
-
-    double overallMean = mids.Average(); // use full mean for normalization
-    //double drift = Math.Sqrt(rollingMeans.Select(m => Math.Pow(m - rollingMeans.Average(), 2)).Average());
-    double drift = Math.Sqrt(rollingMeans.Select(m => Math.Pow(m - rollingMeans.Average(), 2)).Average());
-
-
-double relativeDrift = Math.Abs(overallMean) > 1e-8 ? drift / Math.Abs(overallMean) : double.NaN;
-    return (drift, relativeDrift);
-}
-
-/// <summary>
-/// Returns the latest Kalman Filter estimate of the true mean of the spread mid-prices.
-/// This tracks the spread's central value while allowing for gradual drift and suppressing noise.
-///
-/// Internally, this applies a recursive 1D Kalman filter over the input mid-prices.
-/// 
-/// Tuning Parameters:
-/// - Q: Process noise variance (higher = faster adaptation to drift)
-/// - R: Measurement noise variance (higher = more smoothing)
-/// 
-/// Use this instead of SMA if your mean drifts or if price series is noisy.
-///
-/// </summary>
-/// <param name="spreadData">List of (bid, bidQty, ask, askQty)</param>
-/// <param name="Q">Process noise variance (default = 0.0001)</param>
-/// <param name="R">Measurement noise variance (default = 0.01)</param>
-/// <returns>Latest estimated Kalman-filtered mean, or NaN if input is invalid</returns>
-public static double GetLatestKalmanMean(
-    List<Tuple<double, double, double, double>> spreadData,
-    double Q = 0.0001,
-    double R = 0.01)
-{
-    if (spreadData == null || spreadData.Count == 0)
-        return double.NaN;
-
-    var mids = spreadData.Select(x => (x.Item1 + x.Item3) / 2.0).ToList();
-
-    double x = mids[0];  // initial estimate of mean
-    double P = 1.0;       // initial uncertainty
-
-    for (int i = 1; i < mids.Count; i++)
-    {
-        // Predict step
-        double x_prior = x;
-        double P_prior = P + Q;
-
-        // Update step
-        double K = P_prior / (P_prior + R);
-        x = x_prior + K * (mids[i] - x_prior);
-        P = (1 - K) * P_prior;
-    }
-
-    return x; // the latest estimated mean
-}
-
-
-
-
-}
@@ -1,137 +0,0 @@
-namespace bts.SilverPine {
-   using BCG;
-
-   using System.Diagnostics;
-
-   public class SpreadPerformanceStream : ViewStream {
-      ViewStream _spread_price_stream = null;
-      private readonly IBlueTableSchema _schema;
-      private readonly int _window_secs;
-      private readonly double _buy_threshold, _sell_threshold, _flat_threshold;
-      private ViewStream _bbo_stream;
-      public SpreadPerformanceStream(string s1, string s2, double w1, double w2, int window_secs,
-                                       double buy_threshold, double sell_threshold, double flat_threshold)
-      : this(s1, s2, w1, w2, window_secs, buy_threshold, sell_threshold, flat_threshold, ViewStreamDatabase.instance) { }
-
-      public SpreadPerformanceStream(string s1, string s2, double w1, double w2, int window_secs,
-                                      double buy_threshold, double sell_threshold, double flat_threshold,
-                                      ViewStreamDatabase vsd) {
-         _bbo_stream = new BBOStream(vsd);
-         _schema = BlueDataUtility.form_schema("SpreadPerformance", Tuple.Create("id", TypeCode.Int32),                                                
-                                                Tuple.Create("t0", TypeCode.DateTime),
-                                                Tuple.Create("t1", TypeCode.DateTime),
-                                                Tuple.Create("n_ticks", TypeCode.UInt32),
-                                                Tuple.Create("curr_windows_seconds", TypeCode.Double),
-                                                // means / std dev                                          
-                                                Tuple.Create("stddev", TypeCode.Double),
-                                                Tuple.Create("mean", TypeCode.Double),
-                                                Tuple.Create("kalman_filter_mean", TypeCode.Double),
-                                                Tuple.Create("absolute_mean_drift", TypeCode.Double),
-                                                Tuple.Create("relative_mean_drift", TypeCode.Double),
-                                                // statistical measures
-                                                Tuple.Create("hurst_exponent", TypeCode.Double),
-                                                Tuple.Create("half_life", TypeCode.Double),
-                                                Tuple.Create("lag1_autocorrelation", TypeCode.Double),
-                                                Tuple.Create("average_zscore_decay", TypeCode.Double),
-                                                Tuple.Create("variance_ratio", TypeCode.Double),
-                                                Tuple.Create("n_zero_crossings", TypeCode.UInt32),                                                
-                                                // distributions
-                                                Tuple.Create("Z ≤ 0.5σ", TypeCode.Double),
-                                                Tuple.Create("0.5σ – 1.0σ", TypeCode.Double),
-                                                Tuple.Create("1.0σ – 2.0σ", TypeCode.Double),
-                                                Tuple.Create("> 2.0σ", TypeCode.Double),
-                                                ////////////////////////////////////////                                                   
-                                                Tuple.Create("max_bid", TypeCode.Double),
-                                                Tuple.Create("min_ask", TypeCode.Double),
-                                                Tuple.Create("n_buy_signals", TypeCode.UInt32),
-                                                Tuple.Create("cumm_buy_price", TypeCode.Double),
-                                                Tuple.Create("n_sell_signals", TypeCode.UInt32),
-                                                Tuple.Create("cumm_sell_price", TypeCode.Double)
-                                                //Tuple.Create("pnl", TypeCode.Double),
-                                                //Tuple.Create("buy_turns", TypeCode.UInt32),
-                                                //Tuple.Create("sell_turns", TypeCode.UInt32),
-                                                //Tuple.Create("trades", TypeCode.String)
-                                                );
-         _window_secs = window_secs;
-         _buy_threshold = buy_threshold;
-         _sell_threshold = sell_threshold;
-         _flat_threshold = flat_threshold;
-         _spread_price_stream = new SyntheticSpreadStream(s1, s2, w1, w2, _bbo_stream, vsd);
-         _spread_price_stream = new TimeSeriesStream(_spread_price_stream);
-         _spread_price_stream = new ThrottleStream(_spread_price_stream, TimeSpan.FromSeconds(5.0), vsd);
-      }
-
-      protected override void myOnFirstSubscriber() {
-         _spread_price_stream.onChange += _handle_spread_prices;
-      }
-
-      protected override void myOnLastSubscriber() {
-         _spread_price_stream.onChange -= _handle_spread_prices;
-      }
-
-      private void _handle_spread_prices(object dp_obj) {
-         DataPacket dp = dp_obj as DataPacket;
-         List<Tuple<DateTime, double, double, double, double>> spread_ts_prices = new List<Tuple<DateTime, double, double, double, double>>();
-         DateTime min_time = DateTime.MaxValue; DateTime max_time = DateTime.MinValue;
-         var sw = Stopwatch.StartNew();
-         sw.Start();
-         foreach (IBlueRow r in dp.newRows) {
-            DateTime ts = r.get_datetime("timestamp");
-            TimeSpan elapsed_time = DateTime.Now - ts;
-            if (elapsed_time.TotalSeconds >= _window_secs)
-               continue;
-            spread_ts_prices.Add(Tuple.Create(
-                                          r.get_datetime("timestamp"),
-                                          r.get<double>("spread_bid_price"),
-                                          r.get<double>("spread_bid_sz"),
-                                          r.get<double>("spread_ask_price"),
-                                          r.get<double>("spread_ask_sz")));
-            if (ts <= min_time) {
-               min_time = ts;
-            }
-            if (ts >= max_time) {
-               max_time = ts;
-            }
-         }
-         List<Tuple<double, double, double, double>> spread_prices = spread_ts_prices
-                                                                                             .OrderBy(t => t.Item1)
-                                                                                             .Select(t => Tuple.Create(t.Item2, t.Item3, t.Item4, t.Item5))
-                                                                                             .ToList();
-
-         uint n_ticks = (uint)spread_prices.Count;
-         double curr_windows_seconds = (max_time - min_time).TotalSeconds;
-         double hurst_exponent = SpreadPerformanceEvaluator.EvaluateHurstExponent(spread_prices);
-         double half_life = SpreadPerformanceEvaluator.ComputeHalfLife (spread_prices);
-         double lag1_autocorrelation = SpreadPerformanceEvaluator.ComputeLag1Autocorrelation(spread_prices);
-         double average_zscore_decay = SpreadPerformanceEvaluator.ComputeAverageZScoreDecay(spread_prices);
-         double variance_ratio = SpreadPerformanceEvaluator.ComputeVarianceRatio(spread_prices, 60, 360);
-
-         (double mean, double stddev) = SpreadPerformanceEvaluator.EvaluateMeanAndStdDev(spread_prices);
-         double kalman_filter_mean = SpreadPerformanceEvaluator.GetLatestKalmanMean(spread_prices);
-         uint n_zero_crossings = SpreadPerformanceEvaluator.EvaluateZeroCrossingCount(spread_prices);
-         (double max_bid, double min_ask) = SpreadPerformanceEvaluator.EvaluateMaxBidMinAsk(spread_prices);
-         (uint n_buy_signals, double cumm_buy_price, uint n_sell_signals, double cumm_sell_price) = SpreadPerformanceEvaluator.EvaluateExcursionCount(spread_prices, _buy_threshold);
-         //(double pnl, List<SpreadPerformanceEvaluator.TradeAction> trades, int buy_turns, int sell_turns) = SpreadPerformanceEvaluator.EvaluateSimulatedPnL(spread_prices, 30, _buy_threshold, _flat_threshold);
-         //string trade_str = String.Empty;
-         //foreach (var tr in trades) {
-         //   trade_str += tr.ToString() + " , ";
-         //}
-         Dictionary<string, double> dist_map = SpreadPerformanceEvaluator.EvaluateDistributionAroundMean(spread_prices,
-                                                                       new List<double>(new double[] { 0.5, 1.0, 2.0 }));
-         var dist_vals = dist_map.Count < 4 ? new double[] { Double.NaN, Double.NaN, Double.NaN, Double.NaN } : dist_map.Values.ToArray();
-         (double absolute_drift, double relative_drift) = SpreadPerformanceEvaluator.EvaluateMeanDriftScore(spread_prices);
-         IBlueRow nr = new BlueRow(_schema, 0, min_time, max_time, n_ticks, curr_windows_seconds, stddev, mean, kalman_filter_mean, absolute_drift, relative_drift,
-            hurst_exponent, half_life, lag1_autocorrelation, average_zscore_decay, variance_ratio, n_zero_crossings,
-                                    dist_vals[0], dist_vals[1], dist_vals[2], dist_vals[3],
-                                    max_bid, min_ask, n_buy_signals, cumm_buy_price, n_sell_signals, cumm_sell_price);
-         BlueTable t = new BlueTable(_schema);
-         t.addRow(nr);
-         var dp_out = BlueDataUtility.build_dp_inc(t, myLastValue as DataPacket, true);
-         var perf_msecs = sw.ElapsedMilliseconds;
-         sw.Stop();
-         Log.Gen.info($"SpreadPerformanceStream._handle_spread_prices comp_time: {perf_msecs} msecs");
-         myFireEvent(dp_out);
-
-      }
-   }
-}
@@ -1,68 +0,0 @@
-namespace bts.SilverPine{
-   using BCG;
-   public class SpreadStatArbsWeightLinearRegressionStream : ViewStream {
-      private IBlueTableSchema _schema;
-      private ViewStream _leg_price_stream;
-      private readonly double _window_secs;
-      private readonly string _s1, _s2;
-
-      public SpreadStatArbsWeightLinearRegressionStream(string s1, string s2, double window_secs, double sample_period_secs)
-         : this(s1, s2, window_secs, sample_period_secs, ViewStreamDatabase.instance) { }
-
-
-      public SpreadStatArbsWeightLinearRegressionStream(string s1, string s2, double window_secs, double sample_period_secs, ViewStreamDatabase vsd) {
-         _window_secs = window_secs;
-         _s1 = s1;
-         _s2 = s2;
-         _schema = BlueDataUtility.form_schema("LinearRegressionWeights",
-                                                Tuple.Create("id", TypeCode.Int32),
-                                                Tuple.Create("t0", TypeCode.DateTime),
-                                                Tuple.Create("t1", TypeCode.DateTime),
-                                                Tuple.Create("curr_windows_seconds", TypeCode.Double),
-                                                Tuple.Create("n_ticks", TypeCode.UInt32),
-                                                Tuple.Create("s1", TypeCode.String),
-                                                Tuple.Create("w1", TypeCode.Double),
-                                                Tuple.Create("s2", TypeCode.String),
-                                                Tuple.Create("w2", TypeCode.String));
-
-         _leg_price_stream = new TimeSeriesStream(new SyntheticSpreadLegsStream(s1, s2));
-         _leg_price_stream = new ThrottleStream(_leg_price_stream, TimeSpan.FromSeconds(1), vsd);
-      }
-
-      protected override void myOnFirstSubscriber() {
-         _leg_price_stream.onChange += _handle_leg_prices;
-      }
-
-      protected override void myOnLastSubscriber() {
-         _leg_price_stream.onChange -= _handle_leg_prices;
-      }
-
-      private void _handle_leg_prices(object dp_obj){
-         DataPacket dp = dp_obj as DataPacket;
-         List<Tuple<double, double, DateTime>> leg_prices = new List<Tuple<double, double, DateTime>>();
-         foreach (IBlueRow r in dp.newRows){
-            double s1_mid = (r.get<double>("s1_bid_price") + r.get<double>("s1_ask_price")) /2.0;
-            double s2_mid = (r.get<double>("s2_bid_price") + r.get<double>("s2_ask_price")) /2.0;
-            DateTime ts = r.get_datetime("timestamp");
-            leg_prices.Add(new Tuple<double, double, DateTime>(s1_mid, s2_mid, ts));
-         }
-         if (leg_prices.Count <= 2)
-            return;
-         leg_prices = leg_prices.Where(t => (DateTime.Now - t.Item3).TotalSeconds <= _window_secs).ToList();
-         DateTime t0 = leg_prices.Select(t => t.Item3).Min();
-         DateTime t1 = leg_prices.Select(t => t.Item3).Max();
-         double curr_windows_seconds = (t1 - t0).TotalSeconds;
-         int n_ticks = leg_prices.Count;
-         List<double> s1_mids = leg_prices.Select(t => t.Item1).ToList();
-         List<double> s2_mids = leg_prices.Select(t => t.Item2).ToList();
-         (double w1, double w2) = SpreadWeightCalculator.CalculateWeightsLinearRegression(s1_mids, s2_mids);
-         BlueTable state = new BlueTable(_schema);
-         IBlueRow nr = new BlueRow(_schema, 0, t0, t1, curr_windows_seconds, n_ticks,_s1, w1, _s2, w2);
-         state.addRow(nr);
-         var dp_out = BlueDataUtility.build_dp_inc(state, myLastValue as DataPacket, true);
-         myFireEvent(dp_out);
-
-      }
-   }
-}
-
@@ -1,143 +0,0 @@
-namespace bts.SilverPine {
-   using BCG;
-
-   using MathNet.Numerics.LinearAlgebra.Factorization;
-
-   using System;
-   using System.Collections.Generic;
-   using System.Diagnostics;
-   using System.Linq;
-   using System.Text;
-   using System.Threading.Tasks;
-   [DataStreamUserRespresentation("SpreadStatsStream")]
-   // This is the fundamental stream that computes the stats (including buy and sell zones) 
-   // of a spread. the legs and the weights are passed as args
-   public class SpreadStatsStream : ViewStream {
-      private ViewStream _spread_price_stream;
-      private readonly IBlueTableSchema _schema;
-      private readonly TimeSpan _window_sz;
-      private readonly string _spread;
-      public SpreadStatsStream(string s1, string s2, double w1, double w2, int rolling_window_sz_seconds, int window_sampling_sz_secs)
-         : this(s1, s2, w1, w2, rolling_window_sz_seconds, window_sampling_sz_secs, new BBOStream(ViewStreamDatabase.instance), ViewStreamDatabase.instance) { }
-      public SpreadStatsStream(string s1, string s2, double w1, double w2, int rolling_window_sz_seconds, int window_sampling_sz_secs, ViewStream bbo_stream, ViewStreamDatabase vsd) {
-         _window_sz = TimeSpan.FromSeconds(rolling_window_sz_seconds);
-         _spread = $"{w1} X {s1} - {w2} X {s2}";
-         _schema = new BlueTableSchema("SpreaddStats", new ColumnInfo ("id", 0, TypeCode.Int32));
-         _schema = _schema.add_column(TypeCode.String, "spread");
-         _schema = _schema.add_column(TypeCode.String, "window");
-         // basic stats
-         _schema = _schema.add_column(TypeCode.Double, "bid_mean");
-         _schema = _schema.add_column(TypeCode.Double, "ask_mean");
-         _schema = _schema.add_column(TypeCode.Double, "bid_stddev");
-         _schema = _schema.add_column(TypeCode.Double, "ask_stddev");
-         _schema = _schema.add_column(TypeCode.Double, "bid_skewness");
-         _schema = _schema.add_column(TypeCode.Double, "ask_skewness");
-         _schema = _schema.add_column(TypeCode.Double, "bid_kurtosis");
-         _schema = _schema.add_column(TypeCode.Double, "ask_kurtosis");
-         _schema = _schema.add_column(TypeCode.Double, "bid_max");
-         _schema = _schema.add_column(TypeCode.Double, "ask_max");
-         _schema = _schema.add_column(TypeCode.Double, "bid_min");
-         _schema = _schema.add_column(TypeCode.Double, "ask_min");
-         // bid bins
-         _schema = _schema.add_column(TypeCode.Double, "bid Z < -2σ");
-         _schema = _schema.add_column(TypeCode.Double, "bid -2σ ≤ Z < -1σ");
-         _schema = _schema.add_column(TypeCode.Double, "bid -1σ ≤ Z < 0σ");
-         // positive z's
-         _schema = _schema.add_column(TypeCode.Double, "bid 0σ ≤ Z < 1σ");
-         _schema = _schema.add_column(TypeCode.Double, "bid 1σ ≤ Z < 2σ");
-         _schema = _schema.add_column(TypeCode.Double, "bid Z ≥ 2σ");
-         // ask bins
-         _schema = _schema.add_column(TypeCode.Double, "ask Z < -2σ");
-         _schema = _schema.add_column(TypeCode.Double, "ask -2σ ≤ Z < -1σ");
-         _schema = _schema.add_column(TypeCode.Double, "ask -1σ ≤ Z < 0σ");
-         // positive z's
-         _schema = _schema.add_column(TypeCode.Double, "ask 0σ ≤ Z < 1σ");
-         _schema = _schema.add_column(TypeCode.Double, "ask 1σ ≤ Z < 2σ");
-         _schema = _schema.add_column(TypeCode.Double, "ask Z ≥ 2σ");
-         // opps
-         _schema = _schema.add_column(TypeCode.Int32, "n_buy_ticks");
-         _schema = _schema.add_column(TypeCode.Double, "cumm_buy_price");
-         _schema = _schema.add_column(TypeCode.Int32, "n_sell_ticks");
-         _schema = _schema.add_column(TypeCode.Double, "cumm_sell_price");
-         _schema = _schema.add_column(TypeCode.Int32, "n_trade_ticks");
-         _spread_price_stream = new SyntheticSpreadStream(s1, s2, w1, w2, bbo_stream, vsd);
-         _spread_price_stream = new TimeSeriesStream(_spread_price_stream);
-         double window_sampling_sz_smsecs = window_sampling_sz_secs * 1000.0 + new Random(DateTime.Now.Second).Next(-100, 100);
-         TimeSpan sampling_period = TimeSpan.FromMilliseconds(window_sampling_sz_smsecs);
-         _spread_price_stream = new ThrottleStream(_spread_price_stream, sampling_period, vsd);
-      }
-
-      protected override void myOnFirstSubscriber() {
-         _spread_price_stream.onChange += _handle_spread_prices;
-      }
-
-      protected override void myOnLastSubscriber() {
-         _spread_price_stream.onChange -= _handle_spread_prices;
-      }
-
-      private void _handle_spread_prices(object dp_obj) {
-         List<IBlueRow> rows = (dp_obj as DataPacket).newRows.to_list();
-         Func<IBlueRow, bool> row_filter = (r) => {
-            DateTime ts = r.get_datetime("timestamp");
-            DateTime start_time = DateTime.Now.Subtract(_window_sz);
-            return ts >= start_time;
-         };
-         var rolling_win_rows = rows.Where(r => row_filter(r));
-         var spread_prices = rolling_win_rows.Select(r => (
-                                          r.get_datetime("timestamp"),
-                                          r.get<double>("spread_bid_price"),
-                                          r.get<double>("spread_bid_sz"),
-                                          r.get<double>("spread_ask_price"),
-                                          r.get<double>("spread_ask_sz"))).ToList();
-         spread_prices.Sort((a,b) => a.Item1.CompareTo(b.Item1));
-         DateTime t0 = spread_prices.First().Item1, t1 = spread_prices.Last().Item1;
-         var tot_secs = (t1 - t0).TotalSeconds;
-         var stopwatch = Stopwatch.StartNew();
-         Log.Gen.info($"SpreadStatsStream.analyze_spread_quotes executed in {stopwatch.ElapsedMilliseconds} ms");
-         var result = SpreadPerformanceEvaluator.SpreadStatistics.analyze_spread_quotes(spread_prices);
-         (uint n_buy_signals, double cumm_buy_price, uint n_sell_signals, double cumm_sell_price) = 
-              SpreadPerformanceEvaluator.EvaluateExcursionCount(spread_prices.Select(t => Tuple.Create(t.Item2, t.Item3, t.Item4, t.Item5)).ToList(), 2.0);
-         IBlueRow nr = new BlueRow(_schema,
-            0,
-            _spread,
-            $"n_ticks: {spread_prices.Count} t0: {t0.ToLongTimeString()} -> t1: {t1.ToLongTimeString()} ({tot_secs}) secs",
-            result.bid_stats.Mean,
-            result.ask_stats.Mean,
-            result.bid_stats.StandardDeviation,
-            result.ask_stats.StandardDeviation,
-            result.bid_stats.Skewness,
-            result.ask_stats.Skewness,
-            result.bid_stats.Kurtosis,
-            result.ask_stats.Kurtosis,
-            result.bid_stats.Maximum,
-            result.ask_stats.Maximum,
-            result.bid_stats.Minimum,
-            result.ask_stats.Minimum,
-            // bid distribution
-            result.bid_zscore_distribution["Z < -2σ"],
-            result.bid_zscore_distribution["-2σ ≤ Z < -1σ"],
-            result.bid_zscore_distribution["-1σ ≤ Z < 0σ"],
-            result.bid_zscore_distribution["0σ ≤ Z < 1σ"],
-            result.bid_zscore_distribution["1σ ≤ Z < 2σ"],
-            result.bid_zscore_distribution["Z ≥ 2σ"],
-            // ask distributions
-            result.ask_zscore_distribution["Z < -2σ"],
-            result.ask_zscore_distribution["-2σ ≤ Z < -1σ"],
-            result.ask_zscore_distribution["-1σ ≤ Z < 0σ"],
-            result.ask_zscore_distribution["0σ ≤ Z < 1σ"],
-            result.ask_zscore_distribution["1σ ≤ Z < 2σ"],
-            result.ask_zscore_distribution["Z ≥ 2σ"],
-            n_buy_signals,
-            cumm_buy_price,
-            n_sell_signals,
-            cumm_sell_price,
-            n_buy_signals + n_sell_signals
-            );
-         stopwatch.Stop();
-         BlueTable t = new BlueTable(_schema);
-         t.addRow(nr);
-         var dp_out = BlueDataUtility.build_dp(t, myLastValue as DataPacket);
-         myFireEvent(dp_out);
-      }
-   }
-}
@@ -1,49 +0,0 @@
-using System;
-using System.Collections.Generic;
-using MathNet.Numerics.LinearRegression;
-
-namespace bts.SilverPine {
-
-   public static class SpreadWeightCalculator {
-      /// <summary>
-      /// Estimates hedge weights (w1, w2) such that the spread s = w1 * s1 - w2 * s2 
-      /// is potentially mean-reverting. Uses linear regression: s1 = beta * s2 + ε.
-      /// 
-      /// Assumes mid-prices are precomputed and aligned over the same time window.
-      /// </summary>
-      /// <param name="s1MidPrices">List of mid-prices for asset 1 (dependent variable)</param>
-      /// <param name="s2MidPrices">List of mid-prices for asset 2 (independent variable)</param>
-      /// <param name="includeIntercept">
-      /// Whether to include an intercept in the regression model (default: true).
-      /// </param>
-      /// <returns>(w1, w2): Tuple of hedge weights</returns>
-      public static (double w1, double w2) CalculateWeightsLinearRegression(
-          List<double> s1MidPrices,
-          List<double> s2MidPrices,
-          bool includeIntercept = true) {
-         if (s1MidPrices == null || s2MidPrices == null)
-            throw new ArgumentNullException("Price series cannot be null.");
-         if (s1MidPrices.Count != s2MidPrices.Count)
-            throw new ArgumentException("Price series must be of equal length.");
-         if (s1MidPrices.Count < 2)
-            throw new ArgumentException("Need at least two data points for regression.");
-
-         double[] x = s2MidPrices.ToArray(); // Independent variable
-         double[] y = s1MidPrices.ToArray(); // Dependent variable
-
-         double beta;
-
-         if (includeIntercept) {
-            // Returns (intercept, slope)
-            (double intercept, double slope) = SimpleRegression.Fit(x, y);
-            beta = slope;
-         } else {
-            // Returns only slope when forcing regression through origin
-            beta = SimpleRegression.FitThroughOrigin(x, y);
-         }
-
-         // Return fixed w1 = 1.0 and w2 = beta
-         return (1.0, beta);
-      }
-   }
-}
@@ -13,6 +13,7 @@ namespace bts.SilverPine {
   using Spark.Api.MarketData;

   using SecurityKey = System.UInt32;
+   using System.Data.Entity.Infrastructure;

   public class FakeBBOStream : ViewStream, IWorker {
      private string _symbol;
@@ -104,14 +105,14 @@ namespace bts.SilverPine {
   public class BBOStream : ViewStream, IWorker {
      private readonly TimeSpan _poll_period = TimeSpan.FromMilliseconds(500);
      private DateTime _last_poll_time = DateTime.MinValue;
-      private ViewStreamDatabase? _vsd;
-      IReadOnlyDictionary<SecurityKey, BookTuple>? _cached_state = null;
-      MarketDataClient? _md_client;
+      private ViewStreamDatabase _vsd;
+      private int _id = 0;
+      IReadOnlyDictionary<SecurityKey, BookTuple> _cached_state = null;
+      MarketDataClient _md_client;
      internal static IBlueTableSchema _bbo_schema = BlueDataUtility.form_schema("bbo",
                                                Tuple.Create("id", TypeCode.Int32),
                                                Tuple.Create("sec_key", TypeCode.UInt32),
                                                Tuple.Create("sec", TypeCode.String),
-                                                Tuple.Create("exchange", TypeCode.String),
                                                Tuple.Create("bid_price", TypeCode.Double),
                                                Tuple.Create("bid_sz", TypeCode.Single),
                                                Tuple.Create("ask_price", TypeCode.Double),
@@ -123,18 +124,17 @@ namespace bts.SilverPine {
      }

      public BBOStream(ViewStreamDatabase vsd) {
-         bts.Log.Gen.info("BBOStream.cons");
-         _vsd = vsd;
-         _md_client = new MarketDataClient(new MDSource(TimeSpan.FromSeconds(0.5)));
-         //FakeBBOStream rs = new FakeBBOStream("Future", "CME", "ES", 5000, 0.25, vsd);
-         //mySetRealSource(rs);
+         // bts.Log.Gen.info("BBOStream.cons");
+         // _vsd = vsd;
+         // _md_client = new MarketDataClient(new MDSource(TimeSpan.FromSeconds(0.5)));
+         FakeBBOStream rs = new FakeBBOStream("Future", "CME", "ES", 5000, 0.25, vsd);
+         mySetRealSource(rs);
      }

      protected override void myOnFirstSubscriber() {
         _vsd.addWorker(this);
         _md_client.books().Subscribe(_on_books);
-         _vsd?.addWorker(this);
-         _md_client?.books().Subscribe(_on_books);
+
      }

      private void _log_updates(TableUpdate<BookTuple, uint> update) {
@@ -144,24 +144,24 @@ namespace bts.SilverPine {
         }
         foreach (Change<BookTuple> c in update.changes) {
            var bt = c.New;
-            //bts.Log.Gen.info($"BBOStream changes: idx: {i} bid: {bt.book.bids[0].price} / {bt.book.bids[0].size} ask: {bt.book.asks[0].price} / {bt.book.asks[0].size}");
+            bts.Log.Gen.info($"BBOStream changes: idx: {i} bid: {bt.book.bids[0].price} / {bt.book.bids[0].size} ask: {bt.book.asks[0].price} / {bt.book.asks[0].size}");
         }
      }

      private void _on_books(TableUpdate<BookTuple, uint> update) {
         _log_updates(update);
         // just cacching in here, will populate the rows in _poll_scan and fire the datapacket
-         //bts.Log.Gen.info($"BBOStream._on_books state: {update.state.Count()} changes: {update.changes.Count()} added: {update.added.Count()}");
+         bts.Log.Gen.info($"BBOStream._on_books state: {update.state.Count()} changes: {update.changes.Count()} added: {update.added.Count()}");
         _cached_state = update.state;
      }

      protected override void myOnLastSubscriber() {
-         _vsd?.removeWorker(this);
+         _vsd.removeWorker(this);
      }

      private void _scan_bbo() {
         // this shouldn be made incremental using the TableUpdate
-         //bts.Log.Gen.info("BBOStream._scan_bbo");
+         bts.Log.Gen.info("BBOStream._scan_bbo");
         if (_cached_state == null)
            return;
         BlueTable new_state = new BlueTable(_bbo_schema);
@@ -170,23 +170,22 @@ namespace bts.SilverPine {
            var sec = kv.Value.security;
            int id = Convert.ToInt32(skey);
            BookTuple bt = kv.Value;
-            var venue = sec.Exchange;
-            IBlueRow r = new BlueRow(_bbo_schema, id, skey, sec.ToString(), venue.ToString(), bt.book.bids[0].price, bt.book.bids[0].size,
+            IBlueRow r = new BlueRow(_bbo_schema, id, skey, sec.ToString(), bt.book.bids[0].price, bt.book.bids[0].size,
                                     bt.book.asks[0].price, bt.book.asks[0].size);
            new_state.addRow(r);
         }
-         var dp = accumulate(myLastValue as DataPacket, new_state);
+         DataPacket dp = null;
+         if (myLastValue == null) {
+            dp = new DataPacket(new_state, new_state, new BlueTable(_bbo_schema));
+         } else {
+            DataPacket old_dp = myLastValue as DataPacket;
+            IBlueRowCollection added_rows, del_rows;
+            BlueDataUtility.compute_differences_fast(new_state, old_dp.newRows, out del_rows, out added_rows);
+            dp = new DataPacket(new_state, added_rows, del_rows);
+         }
         if (dp.addedRows.Count == 0 && dp.deletedRows.Count == 0)
            return;
         myFireEvent(dp);
-
-         static DataPacket accumulate(DataPacket? old_dp, BlueTable new_state) {
-            if (old_dp is null) {
-               return new(new_state, new_state, new BlueTable(_bbo_schema));
-            }
-            BlueDataUtility.compute_differences_fast(new_state, old_dp.newRows, out var del_rows, out var added_rows);
-            return new(new_state, added_rows, del_rows);
-         }
      }

      public void doWork() {
@@ -196,4 +195,4 @@ namespace bts.SilverPine {
         }
      }
   }
-}
+}
@@ -148,22 +148,27 @@ namespace bts.SilverPine {

   [DataStreamUserRespresentation("FutureHistoricalBars")]
   public class FutureHistoricalBarsStream : ViewStream {
+      public FutureHistoricalBarsStream(string fut_contract, string bar_size_txt, string range_beg, string range_end) {
+         DateTime dt_range_beg = DateTime.Parse(range_beg);
+         DateTime dt_range_end = DateTime.Parse(range_end);
+         TimeSpan diff = dt_range_end - dt_range_beg;
+         BarSize bar_size = Enum.Parse<BarSize>(bar_size_txt, true);
+         ViewStream rs = new FutureHistoricalBarsStream(fut_contract, bar_size.ToString(), dt_range_beg, dt_range_end);
+         mySetRealSource(rs);
+      }
+
      public FutureHistoricalBarsStream(string fut_contract, string range_beg, string range_end) {
         DateTime dt_range_beg = DateTime.Parse(range_beg);
         DateTime dt_range_end = DateTime.Parse(range_end);
         TimeSpan diff = dt_range_end - dt_range_beg;
-         BarSize bar_size = diff < TimeSpan.FromDays(1.0) ? BarSize.ONE_MIN : BarSize.ONE_DAY;
+         BarSize bar_size = diff < TimeSpan.FromDays(1.0) ? BarSize.ONE_DAY : BarSize.ONE_MIN;
         ViewStream rs = new FutureHistoricalBarsStream(fut_contract, bar_size.ToString(), dt_range_beg, dt_range_end);
         mySetRealSource(rs);
      }
+
      public FutureHistoricalBarsStream(string fut_contract, string bar_size)
      : this(fut_contract, bar_size, DateTime.MinValue, DateTime.MaxValue) { }

-      public FutureHistoricalBarsStream(string fut_contract, string bar_size,
-                                        string dt_range_beg, string dt_range_end)
-                                        : this(fut_contract, bar_size, DateTime.Parse(dt_range_beg),
-                                        DateTime.Parse(dt_range_end)) { }
-
      public FutureHistoricalBarsStream(string fut_contract, string bar_size,
                                        DateTime dt_range_beg, DateTime dt_range_end)
                                        :
@@ -21,7 +21,7 @@ namespace BCG {
      private DateTime _last_poll_time = DateTime.MinValue;
      private ViewStreamDatabase _vsd;
      private int _id = 0;
-      IReadOnlyDictionary<SecurityKey, BookTuple>? _cached_state = null;
+      IReadOnlyDictionary<SecurityKey, BookTuple> _cached_state = null;
      MarketDataClient _md_client;
      readonly Stopwatch _timer = new();
      private readonly IBlueTableSchema _bbo_schema = BlueDataUtility.form_schema("books",
@@ -73,8 +73,14 @@ namespace BCG {

      private Dictionary<SecurityKey, Dictionary<int, int>> _id_map = new Dictionary<SecurityKey, Dictionary<int, int>>();
      private int _get_id(SecurityKey skey, int depth) {
-         var d_map = _id_map.GetOrAdd(skey, _ => new() { {depth, _id++} });
-         if (!d_map.TryGetValue(depth, out int rv)) {
+         Dictionary<int, int> d_map = null;
+         if (!_id_map.TryGetValue(skey, out d_map)) {
+            d_map = new Dictionary<int, int>();
+            d_map.Add(depth, _id++);
+            _id_map.Add(skey, d_map);
+         }
+         int rv = 0;
+         if (!d_map.TryGetValue(depth, out rv)) {
            rv = _id++;
            d_map.Add(depth, rv);
         }
@@ -1,9 +1,8 @@
-namespace bts.SilverPine {
+namespace BCG {
   using BCG.Utilities.General;

   using bts.api;
   using bts.md;
-   using BCG;

   using Spark.Core.Reactive;

@@ -32,7 +31,6 @@ namespace bts.SilverPine {
                                                Tuple.Create("sec_type", TypeCode.String),
                                                Tuple.Create("venue", TypeCode.String),
                                                Tuple.Create("symbol", TypeCode.String),
-                                                Tuple.Create("currency", TypeCode.String),
                                                Tuple.Create("expiration", TypeCode.DateTime),
                                                Tuple.Create("strike", TypeCode.Double),
                                                Tuple.Create("callput", TypeCode.String)
@@ -83,7 +81,7 @@ namespace bts.SilverPine {
         foreach (var kv in _cached_state) {
            SecurityKey skey = kv.Key;
            int id = Convert.ToInt32(skey);
-            SecurityImmutable sec = kv.Value;            
+            SecurityImmutable sec = kv.Value;
            bts.api.SecurityType sec_type = sec.security_type();
            object exp = DBNull.Value;
            if (sec_type == bts.api.SecurityType.Future || sec_type == bts.api.SecurityType.Option)
@@ -94,7 +92,7 @@ namespace bts.SilverPine {
            object cp = DBNull.Value;
            if (sec_type == bts.api.SecurityType.Option)
               cp = sec.Option.CallPut.ToString();
-            IBlueRow r = new BlueRow(_instrument_schema, id, skey, sec.security_desc(), sec.security_type().ToString(), sec.display_exchange().ToString(), sec.Symbol, sec.Currency, exp, strike, cp);
+            IBlueRow r = new BlueRow(_instrument_schema, id, skey, sec.security_desc(), sec.security_type().ToString(), sec.display_exchange().ToString(), sec.Symbol, exp, strike, cp);
            new_state.addRow(r);
         }
         DataPacket dp = null;
@@ -119,4 +117,4 @@ namespace bts.SilverPine {
         }
      }
   }
-}
+}
@@ -174,10 +174,10 @@ namespace bts.SilverPine {
               // TODO: assuming 1 day only for historic               
               string table_name = get_table_from_barsize(ohlcv_params.bar_size, symbol);// $"{symbol}_OHLCV_1DAY";
                                                                                         //string table_name = $"{symbol}_{ohlcv_params.bar_size}";
-               string db_name = _db_name;// $"Data Source={_location}{_db_name};Version=3";
+               string db_str = $"Data Source={_location}{_db_name};Version=3";
               string sql_str = $"select * from {table_name}";
               // "Data Source=C:\\Users\\m_tah\\development\\data\\stream_data.db;Version=3;"
-               ViewStream rs = new SQLiteStream(db_name, sql_str);
+               ViewStream rs = new SQLiteStream(db_str, sql_str);
               rs = new SchemaGuesserStream(rs);
               if (ohlcv_params.bar_size == OHLCVBarSize.ONE_DAY)
                  rs = new DateStandardizationStream(rs, "ts_event", DateConversion.ChopTime);
@@ -207,4 +207,4 @@ namespace bts.SilverPine {


   }
-}
+}
@@ -1,125 +0,0 @@
-using System;
-using BCG;
-
-namespace bts.SilverPine
-{
-/*
-   ----------------------------------------------------------------------
-                               *Expression*
-   ----------------------------------------------------------------------
-       s1 = filter(bbo(), "sec = 'ETH-USDT-SWAP'")
-       s1 = addColumns(s1, "ljc:0")
-       s2 = filter(bbo(), "sec = 'ETH-USDT-SWAP'")
-       s2 = addColumns(s2, "rjc:0")
-       
-       s = fastJoin(s1, "ljc", "sec:s1 bid_price:s1_bid_price bid_sz:s1_bid_sz ask_price:s1_ask_price ask_sz:s1_ask_sz", 
-                    s2, "rjc", "sec:s2 bid_price:s2_bid_price bid_sz:s2_bid_sz ask_price:s2_ask_price ask_sz:s2_ask_sz")
-       
-
-   ----------------------------------------------------------------------
-                                 *Schema*
-   ----------------------------------------------------------------------
-       id                              System.Int32
-       s1                              System.String
-       s1_bid_price                    System.Double
-       s1_bid_sz                       System.Single
-       s1_ask_price                    System.Double
-       s1_ask_sz                       System.Single
-       s2                              System.String
-       s2_bid_price                    System.Double
-       s2_bid_sz                       System.Single
-       s2_ask_price                    System.Double
-       s2_ask_sz                       System.Single
-*/
-
-
-   // Combines the market data of spread legs into one stream, lined up nicely in a row.
-   // this is useful for operations like finding weights, that need leg market data organized together succintly.
-   public class SyntheticSpreadLegsStream : ViewStream 
-   {
-      public SyntheticSpreadLegsStream(string s1, string s2) : 
-         this(s1, s2, new BBOStream(ViewStreamDatabase.instance), ViewStreamDatabase.instance) {}
-
-      public SyntheticSpreadLegsStream(string s1, string s2, ViewStream bbo_stream, ViewStreamDatabase vsb)
-      {
-         ViewStream vs1;
-         ViewStream vs2;
-         ViewStream rs;
-
-         vs1 = new FilterStream (new BBOStream (),
-                                $"sec = '{s1}'");
-         vs1 = new AddColumnsStream (vs1,
-                                    "ljc:0");
-         vs2 = new FilterStream (bbo_stream,
-                                $"sec = '{s2}'");
-         vs2 = new AddColumnsStream (vs2,
-                                    "rjc:0");
-         rs = new NonIncJoinStream (vs1,
-                                   "ljc",
-                                   "sec:s1 bid_price:s1_bid_price bid_sz:s1_bid_sz ask_price:s1_ask_price ask_sz:s1_ask_sz",
-                                   vs2,
-                                   "rjc",
-                                   "sec:s2 bid_price:s2_bid_price bid_sz:s2_bid_sz ask_price:s2_ask_price ask_sz:s2_ask_sz");
-         mySetRealSource(rs);
-      }
-   }
-   public class SyntheticSpreadStream : ViewStream{
-      public SyntheticSpreadStream (string s1, string s2, double w1, double w2)
-      : this(s1, s2, w1, w2, new BBOStream(ViewStreamDatabase.instance), ViewStreamDatabase.instance){}
-      public SyntheticSpreadStream (string s1, string s2, double w1, double w2, ViewStream bbo_stream, ViewStreamDatabase vsd){
-         ViewStream rs = new SyntheticSpreadLegsStream(s1, s2, bbo_stream, vsd);
-
-         Func<IBlueTableSchema, IBlueTableSchema> get_schema = (inp_schema) => {
-            IBlueTableSchema rv = inp_schema;
-            rv = rv.add_column(TypeCode.Double, "spread_bid_price");
-            rv = rv.add_column(TypeCode.Double, "spread_bid_sz");
-            rv = rv.add_column(TypeCode.Double, "spread_ask_price");
-            rv = rv.add_column(TypeCode.Double, "spread_ask_sz");
-            return rv;
-         };
-
-         rs = new MapStream(rs, (inp_schema) => {
-           return get_schema(inp_schema);
-         }, 
-         (r) => {
-              double s1_bid_price = r.get<double>("s1_bid_price");
-              double s1_bid_sz = r.get<double>("s1_bid_sz");
-              double s1_ask_price = r.get<double>("s1_ask_price");
-              double s1_ask_sz = r.get<double>("s1_ask_sz");
-               
-              double s2_bid_price = r.get<double>("s2_bid_price");
-              double s2_bid_sz = r.get<double>("s2_bid_sz");
-              double s2_ask_price = r.get<double>("s2_ask_price");
-              double s2_ask_sz = r.get<double>("s2_ask_sz");
-               
-              (double spread_bid_price, double spread_ask_price, double spread_bid_sz, double spread_ask_sz) =   
-              ComputeSpreadBidAsk(s1_bid_price, s1_ask_price, s2_bid_price, s2_ask_price, s1_bid_sz, s1_ask_sz, s2_bid_sz, s2_ask_sz, w1, w2);
-              IBlueTableSchema out_schema = get_schema(r.schema);
-              MutableBlueRow rv = new MutableBlueRow(out_schema, r);
-              rv.setValue("spread_bid_price", spread_bid_price);
-              rv.setValue("spread_ask_price", spread_ask_price);
-              rv.setValue("spread_bid_sz", spread_bid_sz);
-              rv.setValue("spread_ask_sz", spread_ask_sz);
-              return rv;
-              }
-           );
-         mySetRealSource(rs);
-
-      }
-      public static (double SpreadBid, double SpreadAsk, double SpreadBidQty, double SpreadAskQty) ComputeSpreadBidAsk(
-        double s1Bid, double s1Ask,
-        double s2Bid, double s2Ask,
-        double s1BidQty, double s1AskQty,
-        double s2BidQty, double s2AskQty,
-        double w1, double w2)
-    {
-        double spreadBid = (w1 * s1Bid) - (w2 * s2Ask);
-        double spreadAsk = (w1 * s1Ask) - (w2 * s2Bid);
-
-        double spreadBidQty = Math.Min(s1BidQty / w1, s2AskQty / w2);
-        double spreadAskQty = Math.Min(s1AskQty / w1, s2BidQty / w2);
-
-        return (spreadBid, spreadAsk, spreadBidQty, spreadAskQty);
-    }
-   }
-}
@@ -1,7 +1,7 @@
 <Project Sdk="Microsoft.NET.Sdk">

  <PropertyGroup>
-    <TargetFramework>net8.0</TargetFramework>
+    <TargetFramework>net7.0</TargetFramework>
    <ImplicitUsings>enable</ImplicitUsings>
    <Nullable>enable</Nullable>
  </PropertyGroup>
@@ -43,7 +43,7 @@
      Condition="Exists($(ComputerSpecificConfig))" />

  <PropertyGroup Label="ProtoBuf">
-    <ProtocPluginBase>$(SparkRoot)\src\lib_spark\spark\api\service_compiler</ProtocPluginBase>
+    <ProtocPluginBase>$(SparkRoot)\src\lib\api\service_compiler</ProtocPluginBase>
  </PropertyGroup>
  <Choose>
    <When Condition="'$(OS)' == 'Windows_NT'">
@@ -24,7 +24,7 @@
    <PropertyGroup>
      <GenProtosCmd>$(protoc) %(Proto.FullPath)</GenProtosCmd>
      <GenProtosCmd>$(GenProtosCmd) --error_format=msvs</GenProtosCmd>
-      <GenProtosCmd>$(GenProtosCmd) --proto_path=$(SourceRoot)/lib_spark</GenProtosCmd>
+      <GenProtosCmd>$(GenProtosCmd) --proto_path=$(SourceRoot)</GenProtosCmd>
      <GenProtosCmd>$(GenProtosCmd) --plugin=&quot;protoc-gen-spark-csharp=$(ProtocPlugin)&quot;</GenProtosCmd>
      <GenProtosCmd>$(GenProtosCmd) --spark-csharp_out=lang=c#,templates=immutable.pb.cs,strip_out_path=1:%(Proto.OutDir)</GenProtosCmd>
      <GenProtosCmd>$(GenProtosCmd) --csharp_out=%(Proto.OutDir)</GenProtosCmd>
@@ -65,7 +65,7 @@
    <PropertyGroup>
      <GenProtosCmd>$(protoc) %(ProtoService.FullPath)</GenProtosCmd>
      <GenProtosCmd>$(GenProtosCmd) --error_format=msvs</GenProtosCmd>
-      <GenProtosCmd>$(GenProtosCmd) --proto_path=$(SourceRoot)/lib_spark</GenProtosCmd>
+      <GenProtosCmd>$(GenProtosCmd) --proto_path=$(SourceRoot)</GenProtosCmd>
      <GenProtosCmd>$(GenProtosCmd) --plugin=&quot;protoc-gen-service=$(ProtocPlugin)&quot;</GenProtosCmd>
      <GenProtosCmd>$(GenProtosCmd) --service_out=lang=c#,strip_out_path=1:%(ProtoService.OutDir)</GenProtosCmd>
      <GenProtosCmd>$(GenProtosCmd) --csharp_out=%(ProtoService.OutDir)</GenProtosCmd>
@@ -1,42 +1,56 @@
 cmake_minimum_required(VERSION 3.5)
 project(Spark.Admin.WebApp)

-include(GNUInstallDirs)
 include(${CMAKE_SOURCE_DIR}/cmake/safe_dep_glob.cmake)
-include(${CMAKE_SOURCE_DIR}/cmake/SymLink.cmake)
-include(${CMAKE_SOURCE_DIR}/cmake/ProtobufGenerateJs.cmake)
-include(${CMAKE_SOURCE_DIR}/cmake/ProtobufGenerateFs.cmake)
+include(${CMAKE_SOURCE_DIR}/cmake/Protobuf.cmake)
+
+# dependencies
+set(PROTO_GLOBS
+  ../lib/api/*.proto
+  ../lib/api/services/*.proto
+  ../lib/domain_model/*.proto
+  ../lib/sparkproto/*.proto
+  ../lib/md/protos/*.proto
+  ../../externals/linux/include/google/protobuf/descriptor.proto
+  ../../externals/linux/include/google/protobuf/timestamp.proto)

 file(
-  GLOB_RECURSE ${PROJECT_NAME}_PROTO_SOURCES
+  GLOB_RECURSE _proto_rel_SOURCES
  RELATIVE ${CMAKE_CURRENT_SOURCE_DIR}
-  ../lib_spark/spark/api/*.proto
-  ../lib_spark/spark/api/services/*.proto
-  ../lib_spark/spark/domain_model/*.proto
-  ../lib_spark/spark/sparkproto/*.proto
-  ../lib_spark/spark/md/protos/*.proto
-  ../../externals/linux/include/google/protobuf/descriptor.proto
-  ../../externals/linux/include/google/protobuf/timestamp.proto
-)
+  ${PROTO_GLOBS})

-add_custom_command(
-  OUTPUT ${CMAKE_CURRENT_SOURCE_DIR}/package-lock.json
-  COMMAND npm install --include=dev
-  DEPENDS ${CMAKE_CURRENT_SOURCE_DIR}/package.json
-  WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}
-)
+file(
+  GLOB_RECURSE _proto_abs_SOURCES
+  ${PROTO_GLOBS})
+
+file(
+  GLOB_RECURSE _ts_SOURCES
+  RELATIVE ${CMAKE_CURRENT_SOURCE_DIR}
+  src/*.ts)
+
+file(
+  GLOB_RECURSE _fs_SOURCES
+  RELATIVE ${CMAKE_CURRENT_SOURCE_DIR}
+  src/*.fs)

 set(_tools ${CMAKE_SOURCE_DIR}/tools)
 set(_quiet_exec ${_tools}/quiet_exec.sh)
 set(_venv_exec ${_tools}/venv_exec.sh)
 set(_tools_py ${CMAKE_SOURCE_DIR}/tools/py)

+# The tsconfig file is generated as a function of the cmake build type, and
+# binary dir. Depending on the build type, it extends either
+# 'tsconfig.debug.json' or 'tsconfig.release.json'.
 set(_tsconfig_base "${CMAKE_CURRENT_SOURCE_DIR}/tsconfig.$<IF:$<CONFIG:Debug>,debug,release>.json")
+set(WEBPACK_MODE $<IF:$<CONFIG:Debug>,"development","production">)
 set(_tsconfig "${CMAKE_CURRENT_BINARY_DIR}/tsconfig.json")

 set(_venv_python ${CMAKE_SOURCE_DIR}/.venv/bin/python)
 set(_gen_tsconfig_py ${_tools_py}/gen_tsconfig.py)
 set(_build_admin_web_app_py ${_tools_py}/build_admin_web_app.py)
+
+set(_gen_tsconfig_deps ${_venv_exec} ${_venv_python} ${_gen_tsconfig_py} ${_tsconfig_base})
+
 set(_build_admin_web_app_deps
  ${_quiet_exec}
  ${_venv_exec}
@@ -44,143 +58,51 @@ set(_build_admin_web_app_deps
  ${_build_admin_web_app_py}
  ${CMAKE_CURRENT_SOURCE_DIR}/src/Spark.Admin.WebApp.fsproj
  ${CMAKE_CURRENT_SOURCE_DIR}/webpack.config.js
-  ${CMAKE_SOURCE_DIR}/src/lib_spark/spark/api/service_compiler/templates/fs/fable.fs
+  ${CMAKE_SOURCE_DIR}/src/lib/api/service_compiler/templates/fs/fable.fs
  ${CMAKE_CURRENT_SOURCE_DIR}/package-lock.json
-  ${${PROJECT_NAME}_PROTO_SOURCES}
+  ${_proto_rel_SOURCES}
  ${_api_ts_SOURCES}
  ${_ui_fs_SOURCES}
  ${_tsconfig})

 update_deps_file("${_build_admin_web_app_deps}")

+function(npm_install path)
+  add_custom_command(
+    OUTPUT ${path}/package-lock.json
+    COMMAND "npm" ARGS "install" "--include=dev"
+    DEPENDS ${path}/package.json
+    WORKING_DIRECTORY ${path}
+    COMMENT "Run 'npm install' in ${path}")
+endfunction()
+
+npm_install(${CMAKE_CURRENT_SOURCE_DIR})
+
 add_custom_command(
  OUTPUT ${_tsconfig}
  COMMAND ${_venv_exec}
  ARGS
-    ${_venv_python}
-    ${_gen_tsconfig_py}
-    --config-base ${_tsconfig_base}
-    --config-out ${_tsconfig}
-    --build-out ${CMAKE_CURRENT_BINARY_DIR}
-  DEPENDS
-    ${_venv_exec}
-    ${_venv_python}
-    ${_gen_tsconfig_py}
-    ${_tsconfig_base}
-    ${CMAKE_CURRENT_SOURCE_DIR}/package-lock.json
-)
-
-protobuf_generate_js(
-  TARGET_NAME generate_js_protos
-  PROTO_INCLUDE_DIRS
-    ../lib_spark
-    ../../externals/linux/include
-  PROTO_SOURCES ${${PROJECT_NAME}_PROTO_SOURCES}
-  OUTPUT ${CMAKE_CURRENT_BINARY_DIR}/proto.js
-)
-
-protobuf_generate_fs(
-  TARGET_NAME generate_fs_protos
-  PROTO_INCLUDE_DIRS
-    ../lib_spark
-    ../../externals/linux/include
-  PROTO_SOURCES ${${PROJECT_NAME}_PROTO_SOURCES}
-  OUT_DIR ${CMAKE_CURRENT_BINARY_DIR}/fable
-)
-
-# Each input file maps to multiple output files with different extensions
-# (js, js.map, d.ts, fs). Transpile ts -> d.ts (and js), then d.ts -> fs.
-# Who knew front-end development could be so complicated? `\_(=_=)_/`
-file(
-  GLOB_RECURSE ${PROJECT_NAME}_TS_SOURCES
-  RELATIVE ${CMAKE_CURRENT_SOURCE_DIR}
-  src/*.ts
-)
-set(${PROJECT_NAME}_JS_PROTOS)
-set(${PROJECT_NAME}_TS_PROTOS)
-set(${PROJECT_NAME}_FS_PROTOS)
-foreach(_ts_source ${${PROJECT_NAME}_TS_SOURCES})
-  get_filename_component(_ts_source_we ${_ts_source} NAME_WE)
-  set(_prefix ${CMAKE_CURRENT_BINARY_DIR}/${_ts_source_we})
-  list(APPEND ${PROJECT_NAME}_JS_PROTOS ${_prefix}.js ${_prefix}.js.map)
-  list(APPEND ${PROJECT_NAME}_TS_PROTOS ${_prefix}.d.ts ${_prefix}.d.ts.map)
-  list(APPEND ${PROJECT_NAME}_FS_PROTOS ${_prefix}.fs)
-  add_custom_command(
-    OUTPUT ${_prefix}.fs
-    WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}
-    COMMAND npx --no-install ts2fable ${_prefix}.d.ts ${_prefix}.fs
-    DEPENDS ${_prefix}.d.ts
-  )
-endforeach()
+  ${_venv_python}
+  ${_gen_tsconfig_py}
+  --config-base ${_tsconfig_base}
+  --config-out ${_tsconfig}
+  --build-out ${CMAKE_CURRENT_BINARY_DIR}
+  DEPENDS ${_gen_tsconfig_deps})

 add_custom_command(
-  OUTPUT
-    ${${PROJECT_NAME}_JS_PROTOS}
-    ${${PROJECT_NAME}_TS_PROTOS}
-  WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}
-  COMMAND npx
-    --no-install tsc
-    -p ${_tsconfig}
-  DEPENDS
-    ${_tsconfig}
-    ${CMAKE_CURRENT_BINARY_DIR}/proto.js
-)
-add_custom_target(
-  transpile_type_script
-  ALL
-  DEPENDS
-    ${${PROJECT_NAME}_JS_PROTOS}
-    ${${PROJECT_NAME}_TS_PROTOS}
-)
-add_custom_target(generate_fsts_stubs ALL DEPENDS ${${PROJECT_NAME}_FS_PROTOS})
+  OUTPUT ${CMAKE_CURRENT_BINARY_DIR}/bundle.js
+  COMMAND ${_quiet_exec}
+  ARGS
+  ${_venv_exec}
+  ${_venv_python}
+  ${_build_admin_web_app_py}
+  --protos ${_proto_abs_SOURCES}
+  --tsconfig ${_tsconfig}
+  --mode ${WEBPACK_MODE}
+  --output-dir ${CMAKE_CURRENT_BINARY_DIR}
+  COMMENT "Building Spark.Admin.WebApp"
+  DEPENDS ${_build_admin_web_app_deps})

-sym_link_dir(
-  TARGET_NAME link_webapp_public
-  SOURCE ${CMAKE_CURRENT_SOURCE_DIR}/public
-  DEST ${CMAKE_CURRENT_BINARY_DIR}/public
-)
-sym_link_dir(
-  TARGET_NAME link_webapp_root
-  SOURCE ${CMAKE_CURRENT_BINARY_DIR}
-  DEST ${CMAKE_SOURCE_DIR}/build/Spark.Admin.WebApp
-)
+add_custom_target(admin_web_app_bundle ALL DEPENDS ${CMAKE_CURRENT_BINARY_DIR}/bundle.js)

-file(
-  GLOB_RECURSE ${PROJECT_NAME}_FS_SOURCES
-  RELATIVE ${CMAKE_CURRENT_SOURCE_DIR}
-  src/*.fs
-)
-add_custom_command(
-  OUTPUT
-    ${CMAKE_CURRENT_BINARY_DIR}/bundle.js
-    $<$<CONFIG:DEBUG>:${CMAKE_CURRENT_BINARY_DIR}/bundle.js.map>
-  WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/src
-  COMMAND dotnet restore
-  COMMAND dotnet tool restore
-  COMMAND export SPARK_ADMIN_WEB_APP_OUTPUT_DIR=${CMAKE_CURRENT_BINARY_DIR}
-  COMMAND dotnet fable -o ${CMAKE_CURRENT_BINARY_DIR} --verbose
-  COMMAND npm run build -- --mode $<IF:$<CONFIG:Debug>,"development","production">
-  DEPENDS
-    ${${PROJECT_NAME}_FS_SOURCES}
-    ${${PROJECT_NAME}_FS_PROTOS}
-    ${CMAKE_CURRENT_SOURCE_DIR}/src/Spark.Admin.WebApp.fsproj
-)
-add_custom_target(
-  bundle_spark_admin_webapp ALL
-  DEPENDS
-    ${CMAKE_CURRENT_BINARY_DIR}/bundle.js
-    $<$<CONFIG:DEBUG>:${CMAKE_CURRENT_BINARY_DIR}/bundle.js.map>
-  COMMENT "Bundling Spark.Admin.WebApp"
-)
-
-# Exclude 'public' dir because it is a symlink
-install(
-  DIRECTORY "${CMAKE_CURRENT_BINARY_DIR}"
-  DESTINATION "${CMAKE_INSTALL_BINDIR}"
-  PATTERN "public" EXCLUDE
-)
-# Install actual (non-symlinked) 'public' dir
-install(
-  DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}/public"
-  DESTINATION "${CMAKE_INSTALL_BINDIR}/${PROJECT_NAME}"
-)
+install(DIRECTORY "${CMAKE_CURRENT_BINARY_DIR}" DESTINATION "${CMAKE_INSTALL_BINDIR}")
@@ -42,7 +42,7 @@
    "ts-loader": "^9.4.2",
    "tsutils": "^3.21.0",
    "typescript": "^4.9.5",
-    "webpack-cli": "^5.1.4",
+    "webpack-cli": "^5.0.1",
    "webpack-dev-server": "^4.11.1",
    "ws": "^8.12.1"
  }
@@ -17,7 +17,7 @@ type Tab =
  | PositionManagementTab
  | MarginTab

-let venues = [| Venue.BINANCE; Venue.BinanceUs; Venue.BinanceUsdsFutures; Venue.CBOE; Venue.COINBASE; Venue.DERIBIT; Venue.OKX; Venue.GLBX; Venue.ICE; Venue.TAIFEX; Venue.TWSE; Venue.SIM; |]
+let venues = [| Venue.BINANCE; Venue.BinanceUs; Venue.BinanceUsdsFutures; Venue.CBOE; Venue.COINBASE; Venue.DERIBIT; Venue.GLBX; Venue.ICE; Venue.TAIFEX; Venue.TWSE; Venue.SIM; |]

 let securityTypes = [| SecurityType.SPOT; SecurityType.FUTURE; SecurityType.OPTION; SecurityType.SPREAD; |]

@@ -79,11 +79,6 @@ type TwseExchangeField =
  | Origin of CustomerOrFirm
  | ClearingAccount of string

-type OkxExchangeField =
-  | CTI of CTICode
-  | Origin of CustomerOrFirm
-  | ClearingAccount of string
-
 type SIMExchangeField =
  | CTI of CTICode
  | Origin of CustomerOrFirm
@@ -1,7 +1,7 @@
 <Project Sdk="Microsoft.NET.Sdk">

  <PropertyGroup>
-    <TargetFramework>net8.0</TargetFramework>
+    <TargetFramework>net7.0</TargetFramework>
    <Version>1.0.0</Version>
    <AssemblyVersion>1.0.0</AssemblyVersion>
  </PropertyGroup>
@@ -1,186 +0,0 @@
-cmake_minimum_required(VERSION 3.14)
-project(sparkd)
-
-include(FetchContent)
-include(${CMAKE_SOURCE_DIR}/cmake/safe_dep_glob.cmake)
-include(${CMAKE_SOURCE_DIR}/cmake/ProtobufGenerateCpp.cmake)
-include(${CMAKE_SOURCE_DIR}/cmake/InstallBasic.cmake)
-include(GNUInstallDirs)
-
-if (NOT DEFINED PROJECT_VERSION)
-  set(PROJECT_VERSION 1.0.0)
-endif()
-
-# dependencies
-file(
-  GLOB_RECURSE ${PROJECT_NAME}_PROTO_SOURCES
-  RELATIVE ${CMAKE_CURRENT_SOURCE_DIR}
-  *.proto
-)
-file(
-  GLOB_RECURSE ${PROJECT_NAME}_SOURCES
-  RELATIVE ${CMAKE_CURRENT_SOURCE_DIR}
-  *.cpp
-)
-update_deps_file("${${PROJECT_NAME}_PROTO_SOURCES};${${PROJECT_NAME}_SOURCES}")
-
-protobuf_generate_cpp(
-  CXX_SOURCES ${PROJECT_NAME}_PROTO_CXX_SOURCES
-  CXX_HEADERS ${PROJECT_NAME}_PROTO_CXX_HEADERS
-  CXX_OUT_DIR ${CMAKE_CURRENT_BINARY_DIR}/include
-  PROTO_INCLUDE_DIRS
-    ${EXTERNALS_INCLUDE}
-    ${CMAKE_SOURCE_DIR}/src/lib_spark
-    ${CMAKE_CURRENT_SOURCE_DIR}
-  PROTO_SOURCES
-    ${${PROJECT_NAME}_PROTO_SOURCES}
-)
-add_executable(sparkd ${${PROJECT_NAME}_SOURCES} ${${PROJECT_NAME}_PROTO_CXX_SOURCES})
-
-# include paths
-target_include_directories(
-  sparkd
-  PUBLIC
-    $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}>
-    $<BUILD_INTERFACE:${CMAKE_CURRENT_BINARY_DIR}>/include
-    $<BUILD_INTERFACE:${emumba-deribit-connector-library_INCLUDE_DIRS}>
-)
-
-find_package(PkgConfig REQUIRED)
-pkg_check_modules(emumba-deribit-connector-library REQUIRED emumba-deribit-connector)
-
-# This dir contains rapidjsonConfig.cmake
-list(APPEND CMAKE_PREFIX_PATH ${CMAKE_CURRENT_SOURCE_DIR}/cmake_modules)
-find_package(rapidjson REQUIRED)
-target_include_directories(
-  ${PROJECT_NAME} SYSTEM
-  PUBLIC
-    $<BUILD_INTERFACE:${rapidjson_INCLUDE_DIR}> 
-)
-add_dependencies(${PROJECT_NAME} rapidjson_project)
-
-# databento
-FetchContent_Declare(
-  databento
-  GIT_REPOSITORY https://github.com/databento/databento-cpp
-  GIT_TAG v0.31.0
-)
-FetchContent_MakeAvailable(databento)
-
-# deribit-demo log level
-if(NOT DEFINED DERIBIT_DEMO_LOG_LEVEL)
-  add_definitions(-DSPDLOG_ACTIVE_LEVEL=0)
-  message("-- Building of DERIBIT_DEMO_LOG_LEVEL=TRACE")
-elseif(DERIBIT_DEMO_LOG_LEVEL STREQUAL "TRACE")
-  add_definitions(-DSPDLOG_ACTIVE_LEVEL=0)
-  message("-- Building of DERIBIT_DEMO_LOG_LEVEL=TRACE")
-elseif(DERIBIT_DEMO_LOG_LEVEL STREQUAL "DEBUG")
-  add_definitions(-DSPDLOG_ACTIVE_LEVEL=1)
-  message("-- Building of DERIBIT_DEMO_LOG_LEVEL=DEBUG")
-elseif(DERIBIT_DEMO_LOG_LEVEL STREQUAL "INFO")
-  add_definitions(-DSPDLOG_ACTIVE_LEVEL=2)
-  message("-- Building of DERIBIT_DEMO_LOG_LEVEL=INFO")
-elseif(DERIBIT_DEMO_LOG_LEVEL STREQUAL "WARN")
-  add_definitions(-DSPDLOG_ACTIVE_LEVEL=3)
-  message("-- Building of DERIBIT_DEMO_LOG_LEVEL=WARN")
-elseif(DERIBIT_DEMO_LOG_LEVEL STREQUAL "ERROR")
-  add_definitions(-DSPDLOG_ACTIVE_LEVEL=4)
-  message("-- Building of DERIBIT_DEMO_LOG_LEVEL=ERROR")
-elseif(DERIBIT_DEMO_LOG_LEVEL STREQUAL "CRITICAL")
-  add_definitions(-DSPDLOG_ACTIVE_LEVEL=5)
-  message("-- Building of DERIBIT_DEMO_LOG_LEVEL=CRITICAL")
-elseif(DERIBIT_DEMO_LOG_LEVEL STREQUAL "OFF")
-  add_definitions(-DSPDLOG_ACTIVE_LEVEL=6)
-  message("-- Building of DERIBIT_DEMO_LOG_LEVEL=OFF")
-else()
-  add_definitions(-DSPDLOG_ACTIVE_LEVEL=0)
-  message("-- Building of DERIBIT_DEMO_LOG_LEVEL=TRACE")
-endif()
-
-target_link_directories(
-  ${PROJECT_NAME} PUBLIC
-  $<BUILD_INTERFACE:${CMAKE_CURRENT_BINARY_DIR}>
-  $<INSTALL_INTERFACE:${CMAKE_INSTALL_LIBDIR}>
-)
-
-# Set RPATH so sparkd knows where to find our shared libs (e.g., libspark).
-# The dirs are relative to sparkd's path, represented by "$ORIGIN".
-set_target_properties(
-  ${PROJECT_NAME} PROPERTIES
-  BUILD_RPATH "$ORIGIN/../lib_spark;$ORIGIN/../lib_python_md"
-  INSTALL_RPATH "$ORIGIN/../${CMAKE_INSTALL_LIBDIR}"
-)
-
-# Python interop (for python md providers)
-add_dependencies(${PROJECT_NAME}
-  python_md
-  python_md_module
-  python_md_providers
-  copy_python_md_providers
-)
-include(${CMAKE_SOURCE_DIR}/cmake/SymLink.cmake)
-
-sym_link_local_lib(python_md)
-sym_link_file(
-  SOURCE ${CMAKE_BINARY_DIR}/src/lib_python_md/python_md_module.cpython-310-x86_64-linux-gnu.so
-  DEST ${CMAKE_CURRENT_BINARY_DIR}/python_md_module.cpython-310-x86_64-linux-gnu.so
-  INSTALL_DEST ${CMAKE_INSTALL_BINDIR}/
-)
-get_target_property(providers python_md_providers INTERFACE_PYTHON_SCRIPTS_DIR)
-sym_link_dir(
-  TARGET_NAME link_python_md_providers
-  SOURCE ${providers}
-  DEST ${CMAKE_CURRENT_BINARY_DIR}/python_md_providers
-)
-
-target_link_libraries(
-  sparkd
-  spark
-  binance_execution
-  okx_execution
-  python_md
-  python_md_module
-  databento::databento
-  ${emumba-deribit-connector-library_LIBRARIES}
-  tbb
-  quickfix
-  benchmark
-  Common++
-  Packet++
-  Pcap++
-  lttng-ust
-  -lpcap
-  -static-libgcc
-  -static-libstdc++
-  -pthread)
-
-# Onixs
-target_link_libraries(
-  ${PROJECT_NAME}
-  onixs-cmemdh
-  OnixS.IceImpactMulticastPriceFeedHandlerCpp
-  # gcc doesn't resolve onixs-fix-engine and ACE in runtime path unless we specify
-  # --no-as-needed. These are onixs deps that aren't called directly by our code.
-  # Note: this isn't an issue for clang
-  -Wl,--no-as-needed
-  onixs-fix-engine
-  ACE
-)
-
-# Catch2
-find_package(Catch2 CONFIG REQUIRED)
-target_link_libraries(${PROJECT_NAME} Catch2::Catch2)
-
-install_basic()
-install(
-  FILES
-    "${EXTERNALS_LIB}/libonixs-cmemdh.so"
-    "${EXTERNALS_LIB}/libonixs-cmemdh.so.4"
-    "${EXTERNALS_LIB}/libonixs-cmemdh.so.4.8.0"
-    "${EXTERNALS_LIB}/libOnixS.IceImpactMulticastPriceFeedHandlerCpp.so.3.16.0.0"
-    "${EXTERNALS_LIB}/libonixs-fix-engine.so"
-    "${EXTERNALS_LIB}/libonixs-fix-engine.so.4"
-    "${EXTERNALS_LIB}/libonixs-fix-engine.so.4.10.1"
-    "${EXTERNALS_LIB}/libACE-5.7.5.so"
-  DESTINATION ${CMAKE_INSTALL_LIBDIR}
-)
@@ -1,5 +0,0 @@
-@PACKAGE_INIT@
-
-include("${CMAKE_CURRENT_LIST_DIR}/@PROJECT_NAME@Targets.cmake")
-
-check_required_components(@PROJECT_NAME@)
@@ -1,19 +0,0 @@
-FROM bluetradingsystems/linux_dotnet7
-
-ARG SOURCE_DIR="install/sparkd-release-gcc"
-ARG CONF_FILE="src/app_sparkd/spark.conf"
-
-USER "root"
-RUN apt-get update \
-  && apt-get install -y liblttng-ust-dev
-
-RUN apt-get update \
-  && apt-get install -y libpcap0.8-dev
-
-RUN apt-get update \
-  && apt-get install -y python3.10-dev
-
-USER "bts"
-COPY --chown=bts:bts "${SOURCE_DIR}" "."
-COPY --chown=bts:bts "${CONF_FILE}" ".local/share/bts/sparkd/conf/spark.conf"
-WORKDIR "/home/bts/bin"
@@ -1,4 +0,0 @@
-SPARKD__CONFIG_DIR=src/app_sparkd/configurations/alphaworx_prod
-SPARKD__SHM_SIZE=8gb
-SPARKD__MEMORY=8gb
-SPARKD__RUNTIME_CONF_DIR=/home/ubuntu/8000/sparkdata/conf/
@@ -1,6 +0,0 @@
-* Connects to Binance and Okx PRODUCTION sites for market data and execution
-* Api keys NOT included
-* Configured for Alphaworx
-
-To retrieve api keys, run "python init_env.py". It should dump them to a file
-called  ".env.local".
@@ -1,9 +0,0 @@
-{
-  "api_key": "${BINANCE_API_KEY}",
-  "secret_key": "${BINANCE_SECRET_KEY}",
-  "account": "PROD",
-  "rest_endpoint": "https://api.binance.com",
-  "ws_endpoint": "wss://ws-api.binance.com:443/ws-api/v3",
-  "ws_stream_endpoint": "wss://stream.binance.com:9443",
-  "use_cross_margin_api": true
-}
@@ -1,9 +0,0 @@
-{
-  "api_key": "${BINANCE_API_KEY}",
-  "secret_key": "${BINANCE_SECRET_KEY}",
-  "account": "PROD",
-  "rest_endpoint": "https://fapi.binance.com",
-  "ws_endpoint": "wss://ws-fapi.binance.com/ws-fapi/v1",
-  "ws_stream_endpoint": "wss://fstream.binance.com",
-  "use_cross_margin_api": false
-}
@@ -1,88 +0,0 @@
-import botocore
-import json
-import os
-
-from boto3.session import Session
-from botocore.exceptions import ClientError
-from fileinput import FileInput
-
-
-def update_env_file(path, env_vars: dict):
-    """Update or add vars in a .env file in place."""
-    if not os.path.exists(path):
-        with open(path, "w") as file:
-            for k, v in env_vars.items():
-                file.write(f"{k}={v}\n")
-        return
-
-    with FileInput(path, inplace=True) as file:
-        for line in file:
-            if not line.startswith("#"):
-                stripped: str = line.lstrip()
-                key_len = stripped.find("=")
-                if key_len >= 0:
-                    k = stripped[0:key_len]
-                    if k in env_vars:
-                        print(f"{k}={env_vars[k]}")
-                        del env_vars[k]
-                        continue
-            print(line, end="")
-
-    if len(env_vars) > 0:
-        with open(path, "a") as file:
-            for k, v in env_vars.items():
-                file.write(f"{k}={v}\n")
-
-
-def get_secret():
-    secret_name = "alphaworx-binance-api-keys"
-    region_name = "ap-northeast-1"
-    profile_name = "bts-ap-northeast-1"
-    session = Session(profile_name=profile_name)
-    client = session.client(
-        service_name="secretsmanager",
-        region_name=region_name,
-    )
-
-    def on_token_error(e: Exception):
-        print(f"""
-Failed to get secret value. {e}
-Verify you are logged in to aws, then try again.
-
-  aws sso login --profile {profile_name}
-
-If this error persists, verify you have access to these api keys.
-For more help or information, talk to John.
-""")
-
-    try:
-        get_secret_value_response = client.get_secret_value(
-            SecretId=secret_name)
-    except ClientError as e:
-        print(f"Failed to get secret value.\n{e}")
-        raise e
-    except botocore.exceptions.SSOTokenLoadError as tl_err:
-        on_token_error(tl_err)
-        return None
-    except botocore.exceptions.TokenRetrievalError as tr_err:
-        on_token_error(tr_err)
-        return None
-    return get_secret_value_response["SecretString"]
-
-
-def _run():
-    this_dir = os.path.dirname(__file__)
-    secret_str = get_secret()
-    if secret_str is None:
-        exit(1)
-    secret = json.loads(secret_str)
-    env_vars = {}
-    if "api-key" in secret:
-        env_vars["BINANCE_API_KEY"] = secret["api-key"]
-    if "secret-key" in secret:
-        env_vars["BINANCE_SECRET_KEY"] = secret["secret-key"]
-    update_env_file(f"{this_dir}/.env.local", env_vars)
-
-
-if __name__ == "__main__":
-    _run()
@@ -1,124 +0,0 @@
-QuickbusCfg {
-  value {
-    node {
-      reconnect_period_secs: 1
-      hb_period_msecs: 15000
-      max_missed_hbs: 5
-      parent_host: ""
-      parent_port: 0
-      my_port: 0
-    }
-    client {
-      reconnect_period_secs: 1
-      hb_period_msecs: 15000
-      max_missed_hbs: 5
-      parent_host: @SystemConfig.config_ip
-      parent_port: @Ports.qbus_port
-      my_port: 0
-    }
-  }
-}
-
-JetFIX {
-  value {
-    connection {
-      name: "binance_1"
-      venue: VENUE_BINANCE
-      max_order_msgs_per_window: 300
-      window_duration_msecs: 1000
-    }
-    connection {
-      name: "binance_usds_futures_1"
-      venue: VENUE_BINANCE_USDS_FUTURES
-      max_order_msgs_per_window: 300
-      window_duration_msecs: 1000
-    }
-    connection {
-      name: "okx_1"
-      venue: VENUE_OKX
-      max_order_msgs_per_window: 300
-      window_duration_msecs: 1000
-    }
-  }
-}
-
-MarketDataConfig {
-  value {
-    binance {
-      num_depth_levels: 5
-      url_rest_base: "https://api.binance.com"
-      url_ws_base: "wss://stream.binance.com:9443"
-      security_criteria {
-        clauses {
-          security_types: SECURITY_TYPE_SPOT
-          symbols: @SymbolLists.list[Binance].symbol
-          name_regexps: "USDC|USDT$"
-        }
-      }
-    }
-    binance_usds_futures {
-      num_depth_levels: 5
-      url_rest_base: "https://fapi.binance.com"
-      url_ws_base: "wss://fstream.binance.com"
-      security_criteria {
-        clauses {
-          security_types: SECURITY_TYPE_FUTURE
-          symbols: @SymbolLists.list[BinanceUsdsFutures].symbol
-        }
-      }
-    }
-    okx {
-      num_depth_levels: 5
-      url_rest_base: "https://www.okx.com"
-      url_ws_base: "wss://ws.okx.com:8443"
-      security_criteria {
-        clauses {
-          security_types: SECURITY_TYPE_SPOT
-          symbols: @SymbolLists.list[OkxSpots].symbol
-        }
-        clauses {
-          security_types: SECURITY_TYPE_FUTURE
-          symbols: @SymbolLists.list[OkxFutures].symbol
-        }
-      }
-      is_application_ping_pong_enabled: true
-      application_ping_interval_ms: 20000
-      application_pong_timeout_ms: 10000
-      is_protocol_ping_pong_enabled: true
-      protocol_ping_interval_ms: 60000
-      protocol_pong_timeout_ms: 30000
-    }
-  }
-}
-
-NotSimExecution {
-  value {
-    entries {
-      venue: VENUE_BINANCE
-    }
-    entries {
-      venue: VENUE_BINANCE_USDS_FUTURES
-    }
-    entries {
-      venue: VENUE_OKX
-    }
-  }
-}
-
-OkxExeCfg {
-  value {
-    account: "PROD"
-    ops_id: "op_id_bts"
-    secret_key: "240038D7A28D7F9461247CFCB8E43710"
-    api_key: "dc188532-c74c-4d40-affe-3c6bf4d0ca9a"
-    passphrase: ":,7i%tA%ivfj%SCbX?Ut"
-    ws_url: "wss://wsaws.okx.com:8443"
-    rest_url: "https://www.okx.com"
-    ws_ping_timeout_seconds: 10 
-    ws_ping_interval_seconds: 25
-    ws_reconnect_interval_seconds: 5
-    ws_reconnect_max_count: 5
-	  simulated_trading: false
-    tdmode: "cross"
-  }
-}
@@ -1,124 +0,0 @@
-SymbolLists {
-  value {
-    list {
-      key: "Binance"
-      symbol: "ADA"
-      symbol: "APT"
-      symbol: "AVAX"
-      symbol: "BCH"
-      symbol: "BNB"
-      symbol: "BTC"
-      symbol: "DOGE"
-      symbol: "DOT"
-      symbol: "ETH"
-      symbol: "HBAR"
-      symbol: "LINK"
-      symbol: "LTC"
-      symbol: "OM"
-      symbol: "PAXG"
-      symbol: "PEPE"
-      symbol: "S"
-      symbol: "SHIB"
-      symbol: "SOL"
-      symbol: "SUI"
-      symbol: "TAO"
-      symbol: "TRUMP"
-      symbol: "USDC"
-      symbol: "XRP"
-    }
-    list {
-      key: "BinanceUsdsFutures"
-      symbol: "1000PEPE"
-      symbol: "1000SHIB"
-      symbol: "ADA"
-      symbol: "APT"
-      symbol: "AVAX"
-      symbol: "BCH"
-      symbol: "BNB"
-      symbol: "BTC"
-      symbol: "DOGE"
-      symbol: "DOT"
-      symbol: "ETH"
-      symbol: "HBAR"
-      symbol: "LINK"
-      symbol: "LTC"
-      symbol: "OM"
-      symbol: "S"
-      symbol: "SOL"
-      symbol: "SUI"
-      symbol: "TAO"
-      symbol: "TRUMP"
-      symbol: "USDC"
-      symbol: "XRP"
-    }
-    list {
-      key: "OkxSpots"
-      symbol: "ADA"
-      symbol: "APT"
-      symbol: "AVAX"
-      symbol: "BCH"
-      symbol: "BETH"
-      symbol: "BNB"
-      symbol: "BTC"
-      symbol: "DOGE"
-      symbol: "DOT"
-      symbol: "ETH"
-      symbol: "HBAR"
-      symbol: "LINK"
-      symbol: "LTC"
-      symbol: "OM"
-      symbol: "PEPE"
-      symbol: "S"
-      symbol: "SHIB"
-      symbol: "SOL"
-      symbol: "SUI"
-      symbol: "TRUMP"
-      symbol: "USDC"
-      symbol: "USDT"
-      symbol: "XRP"
-    }
-    list {
-      key: "OkxFutures"
-      symbol: "ADA"
-      symbol: "APT"
-      symbol: "AVAX"
-      symbol: "BCH"
-      symbol: "BNB"
-      symbol: "BTC"
-      symbol: "DOGE"
-      symbol: "DOT"
-      symbol: "ETH"
-      symbol: "HBAR"
-      symbol: "LINK"
-      symbol: "LTC"
-      symbol: "OM"
-      symbol: "PEPE"
-      symbol: "S"
-      symbol: "SHIB"
-      symbol: "SOL"
-      symbol: "SUI"
-      symbol: "TAO"
-      symbol: "TRUMP"
-      symbol: "USDC"
-      symbol: "XRP"
-    }
-  }
-}
-
-Symbols {
-  value {
-    venues {
-      name: "VENUE_BINANCE"
-      spot: @SymbolLists.list[Binance].symbol
-    }
-    venues {
-      name: "VENUE_BINANCE_USDS_FUTURES"
-      future: @SymbolLists.list[BinanceUsdsFutures].symbol
-    }
-    venues {
-      name: "VENUE_OKX"
-      spot: @SymbolLists.list[OkxSpots].symbol
-      future: @SymbolLists.list[OkxFutures].symbol
-    }
-  }
-}
@@ -1,5 +0,0 @@
-SPARKD__CONFIG_DIR=src/app_sparkd/configurations/alphaworx_zen_hk_live
-SPARKD__SHM_SIZE=8gb
-SPARKD__MEMORY=8gb
-SPARKD__IMAGE=sparkd_symbols
-SPARKD__RUNTIME_CONF_DIR=/home/bts/8000/sparkdata/conf/
@@ -1,6 +0,0 @@
-* Connects to Binance and Okx PRODUCTION sites for market data and execution
-* Api keys NOT included
-* Configured for Alphaworx
-
-To retrieve api keys, run "python init_env.py". It should dump them to a file
-called  ".env.local".
Author	SHA1	Message	Date
Adeel Khan	6c1fe0f561	Fix: Taifex Time Conversion	2025-03-26 18:49:17 +00:00
Adeel Khan	a0eaf60eec	FIX: Taifex Crash Fixed, Added Diag Logs	2025-03-26 18:48:27 +00:00
Adeel Khan	f2080d558a	ADD: TBBO & Symbol wise Naming in Recorder	2025-03-26 18:25:28 +00:00
Adeel Khan	06bfe9c29f	ADD: TWSE Code Complete	2025-03-26 18:20:39 +00:00
Adeel Khan	022546a6c1	Revert "Diag: Pushing changes prior to 6th March Crash" This reverts commit `344d180557`.	2025-03-26 17:59:59 +00:00
Adeel Khan	344d180557	Diag: Pushing changes prior to 6th March Crash	2025-03-06 05:56:01 +00:00
Adeel Khan	a485f8139c	ADD: TWSE to WebApp	2024-12-26 21:53:35 -05:00
Adeel Khan	f50066f4d2	ADD: TWSE throughout Sparkd	2024-12-26 21:53:35 -05:00
Taha	277c3056c4	Merge branch 'development' of bitbucket.org:bluetradesys/spark into development	2024-12-09 15:37:50 -05:00
Taha	5eb95e69cb	Cleaning up some streams,new prompt etc	2024-12-09 15:37:45 -05:00