CLEANUP: BYBIT Symbols.conf.

ADD: BYBIT MD Working, NaN's also Added.
ADD: BYBIT Securities showing up on frontend. Default Contract Spec
2025-05-26 11:38:19 +00:00 · 2025-05-26 11:01:20 +00:00 · 2025-05-22 16:17:19 +00:00 · 2025-05-22 16:17:19 +00:00 · 2025-05-22 16:17:19 +00:00 · 2025-05-22 16:17:19 +00:00
2137 changed files with 18430 additions and 10054 deletions
@@ -199,24 +199,27 @@
      "request": "launch",
      "program": "",
      "linux": {
-        "preLaunchTask": "Build Sparkd (Debug)",
-        "program": "${workspaceFolder}/build/sparkd-debug-clang/server/sparkd",
-        "args": [
-          "run"
-        ],
-        "stopAtEntry": false,
-        "cwd": "${workspaceFolder}/build/sparkd-debug-clang/server",
-        "environment": [],
-        "externalConsole": false,
-        "MIMode": "gdb",
-        "setupCommands": [
-          {
-            "description": "Enable pretty-printing for gdb",
-            "text": "-enable-pretty-printing",
-            "ignoreFailures": true
-          }
-        ]
-      }
+        "program": "${workspaceFolder}/build/sparkd-debug-clang/src/app_sparkd/sparkd",
+      },
+      "args": [
+        "run"
+      ],
+      "cwd": "${workspaceFolder}/build/sparkd-debug-clang/src/app_sparkd",
+      "stopAtEntry": false,
+      "externalConsole": false,
+      "MIMode": "gdb",
+      "setupCommands": [
+        {
+          "description": "Enable pretty-printing for gdb",
+          "text": "-enable-pretty-printing",
+          "ignoreFailures": true
+        },
+        {
+          "description": "Set Disassembly Flavor to Intel",
+          "text": "-gdb-set disassembly-flavor intel",
+          "ignoreFailures": true
+        }
+      ]
    },
    {
      "name": "Sparkd (Skip Build)",
@@ -224,74 +227,27 @@
      "request": "launch",
      "program": "",
      "linux": {
-        "program": "${workspaceFolder}/build/sparkd-debug-clang/server/sparkd",
-        "args": [
-          "run"
-        ],
-        "stopAtEntry": false,
-        "cwd": "${workspaceFolder}/build/sparkd-debug-clang/server",
-        "environment": [],
-        "externalConsole": false,
-        "MIMode": "gdb",
-        "setupCommands": [
-          {
-            "description": "Enable pretty-printing for gdb",
-            "text": "-enable-pretty-printing",
-            "ignoreFailures": true
-          }
-        ]
-      }
-    },
-    {
-      "name": "Sparkd Test",
-      "type": "cppdbg",
-      "request": "launch",
-      "program": "",
-      "linux": {
-        "preLaunchTask": "Build Sparkd (Debug)",
-        "program": "${workspaceFolder}/build/sparkd-debug-clang/server/sparkd",
-        "args": [
-          "test",
-          "${input:sparkdTest}",
-        ],
-        "stopAtEntry": false,
-        "cwd": "${workspaceFolder}/build/sparkd-debug-clang/server",
-        "environment": [],
-        "externalConsole": false,
-        "MIMode": "gdb",
-        "setupCommands": [
-          {
-            "description": "Enable pretty-printing for gdb",
-            "text": "-enable-pretty-printing",
-            "ignoreFailures": true
-          }
-        ]
-      }
-    },
-    {
-      "name": "Sparkd Test (Skip Build)",
-      "type": "cppdbg",
-      "request": "launch",
-      "program": "",
-      "linux": {
-        "program": "${workspaceFolder}/build/sparkd-debug-clang/server/sparkd",
-        "args": [
-          "test",
-          "${input:sparkdTest}",
-        ],
-        "stopAtEntry": false,
-        "cwd": "${workspaceFolder}/build/sparkd-debug-clang/server",
-        "environment": [],
-        "externalConsole": false,
-        "MIMode": "gdb",
-        "setupCommands": [
-          {
-            "description": "Enable pretty-printing for gdb",
-            "text": "-enable-pretty-printing",
-            "ignoreFailures": true
-          }
-        ]
-      }
+        "program": "${workspaceFolder}/build/sparkd-debug-clang/src/app_sparkd/sparkd",
+      },
+      "args": [
+        "run"
+      ],
+      "cwd": "${workspaceFolder}/build/sparkd-debug-clang/src/app_sparkd",
+      "stopAtEntry": false,
+      "externalConsole": false,
+      "MIMode": "gdb",
+      "setupCommands": [
+        {
+          "description": "Enable pretty-printing for gdb",
+          "text": "-enable-pretty-printing",
+          "ignoreFailures": true
+        },
+        {
+          "description": "Set Disassembly Flavor to Intel",
+          "text": "-gdb-set disassembly-flavor intel",
+          "ignoreFailures": true
+        }
+      ]
    },
  ],
  "inputs": [
@@ -22,9 +22,9 @@ set(
 # can run simultaneously without fighting over the same shm files (e.g., sparkd,
 # api server, etc.).
 set(SHARED_MEMORY_PREFIX "bts" CACHE STRING "Prefix to use when creating shared memory files")
-set(SPARK_BIN "${EXTERNALS_ROOT}/bin")
-set(SPARK_INCLUDE "${EXTERNALS_ROOT}/include")
-set(SPARK_LIB "${EXTERNALS_ROOT}/lib/$<CONFIG>")
+set(EXTERNALS_BIN "${EXTERNALS_ROOT}/bin")
+set(EXTERNALS_INCLUDE "${EXTERNALS_ROOT}/include")
+set(EXTERNALS_LIB "${EXTERNALS_ROOT}/lib/$<CONFIG>")
 set(BUILD_DOCS OFF CACHE BOOL "Build documentation")
 set(BUILD_SDK OFF CACHE BOOL "Build the sdk")
 set(EXPORT_MD_CONSUMER OFF CACHE BOOL "Export (extern) C functions for MD consumer interop")
@@ -36,8 +36,8 @@ else()
  set(BUILD_ADMIN_WEB_APP ON CACHE BOOL "Build the Admin WebApp")
 endif()

-include_directories(BEFORE SYSTEM ${SPARK_INCLUDE} ${EXTERNALS_ROOT}/include/zlib)
-link_directories(${SPARK_LIB})
+include_directories(BEFORE SYSTEM ${EXTERNALS_INCLUDE} ${EXTERNALS_ROOT}/include/zlib)
+link_directories(${EXTERNALS_LIB})

 add_compile_definitions(SHARED_MEMORY_PREFIX="${SHARED_MEMORY_PREFIX}")
 add_compile_definitions(APP_DATA_DIR="${APP_DATA_DIR}")
@@ -47,12 +47,8 @@ endif()
 add_compile_definitions(EXPORT_MD_CONSUMER="${EXPORT_MD_CONSUMER}")
 if(WIN32)
  # subdirs
-  add_subdirectory(src/lib)
+  add_subdirectory(src/lib_spark)
 else()
-  # make sparkd look for libspark.so in its directory
-  set(CMAKE_BUILD_WITH_INSTALL_RPATH TRUE)
-  set(CMAKE_INSTALL_RPATH ".")
-
  if(BUILD_SPARKD)
    # Note: All macros that begin w/ 'CCAPI' are ccapi's macros (defined and/or
    # read by their code). We override some below.
@@ -118,11 +114,14 @@ else()

  # subdirs
  if (BUILD_SPARKD)
+    add_subdirectory(src/lib_onixs_fix)
    add_subdirectory(src/lib_binance_execution)
-    add_subdirectory(src/server/sparkd)
+    add_subdirectory(src/lib_okx_execution)
+    add_subdirectory(src/lib_python_md)
+    add_subdirectory(src/app_sparkd)
    add_subdirectory(src/data_files)
  endif()
-  add_subdirectory(src/lib)
+  add_subdirectory(src/lib_spark)
  if (BUILD_ADMIN_WEB_APP)
    add_subdirectory(src/Spark.Admin.WebApp)
  endif()
@@ -132,37 +131,4 @@ else()
  if(BUILD_SDK)
    add_subdirectory(src/sdk)
  endif()
-
-  if(BUILD_SPARKD)
-    add_custom_command(
-      OUTPUT ${CMAKE_BINARY_DIR}/server
-      COMMAND mkdir -p ${CMAKE_BINARY_DIR}/server
-      COMMAND ln ARGS -sf ${CMAKE_BINARY_DIR}/src/server/sparkd/sparkd ${CMAKE_BINARY_DIR}/server
-      COMMAND ln ARGS -sf ${CMAKE_BINARY_DIR}/data_files ${CMAKE_BINARY_DIR}/server
-      COMMAND ln ARGS -sf ${CMAKE_BINARY_DIR}/src/lib/libspark.so ${CMAKE_BINARY_DIR}/server
-      COMMAND ln ARGS -sf ${EXTERNALS_ROOT}/lib/${CMAKE_BUILD_TYPE}/libonixs-fix-engine.so.4 ${CMAKE_BINARY_DIR}/server
-      
-      COMMAND
-        ln ARGS -sf
-        ${EXTERNALS_ROOT}/lib/Release/libOnixS.IceImpactMulticastPriceFeedHandlerCpp.so.3.16.0.0
-        ${CMAKE_BINARY_DIR}/server
-      COMMAND ln ARGS -sf ${EXTERNALS_ROOT}/lib/Release/libACE-5.7.5.so ${CMAKE_BINARY_DIR}/server
-      COMMAND mkdir ARGS -p ${CMAKE_BINARY_DIR}/server/plugins
-      COMMENT "Creating server runtime layout"
-      DEPENDS sparkd data_files_dir)
-
-    add_custom_target(server_layout ALL DEPENDS ${CMAKE_BINARY_DIR}/server)
-
-    install(
-      DIRECTORY "${CMAKE_BINARY_DIR}/server/plugins"
-      DESTINATION "${CMAKE_INSTALL_BINDIR}")
-    install(
-      FILES
-      "${SPARK_LIB}/libACE-5.7.5.so"
-      "${SPARK_LIB}/libOnixS.IceImpactMulticastPriceFeedHandlerCpp.so.3.16.0.0"
-      "${SPARK_LIB}/libonixs-fix-engine.so.4.10.1"
-      "${SPARK_LIB}/libonixs-fix-engine.so.4"
-      "${SPARK_LIB}/libonixs-fix-engine.so"
-      DESTINATION "${CMAKE_INSTALL_BINDIR}")
-  endif()
 endif()
@@ -31,9 +31,8 @@
        "CMAKE_SHARED_LINKER_FLAGS_RELEASE": "/OPT:REF /OPT:ICF /LTCG",
        "CMAKE_TOOLCHAIN_FILE": "${sourceDir}/vcpkg/scripts/buildsystems/vcpkg.cmake",
        "EXTERNALS_ROOT": "${sourceDir}/externals/windows",
-        "CONFIG_DEFAULTS_PATH": "config/profiles/default/*.conf",
-        "DOTNET_BINARY_DIR": "${sourceDir}/build/$<CONFIG>/bin/net8.0",
-        "DOTNET_WINDOWS_BINARY_DIR": "${sourceDir}/build/$<CONFIG>/bin/net8.0-windows",
+        "CONFIG_DEFAULTS_PATH": "spark/config/profiles/default/*.conf",
+        "DOTNET_BINARY_DIR": "${sourceDir}/build/$<CONFIG>/bin/net8.0;${sourceDir}/build/$<CONFIG>/bin/net8.0-windows",
        "EXPORT_MD_CONSUMER": {
          "type": "BOOL",
          "value": "ON"
@@ -58,11 +57,12 @@
        "APP_DATA_DIR_OVERRIDE": "${dollar}{SPARKD_DATA_DIR}",
        "CMAKE_BUILD_WITH_INSTALL_RPATH": {
          "type": "BOOL",
-          "value": "ON"
+          "value": "OFF"
        },
        "CMAKE_CXX_STANDARD": "20",
        "CMAKE_INSTALL_RPATH": ".",
        "CMAKE_INSTALL_PREFIX": "${sourceDir}/install/${presetName}",
+        "CMAKE_PREFIX_PATH": "${sourceDir}/cmake/;${sourceDir}/.venv/lib/python3.10/site-packages/pybind11/share/cmake/pybind11",
        "CMAKE_EXE_LINKER_FLAGS": "-Wno-undef",
        "CMAKE_TOOLCHAIN_FILE": "${sourceDir}/vcpkg/scripts/buildsystems/vcpkg.cmake",
        "DATABENTO_USE_EXTERNAL_JSON": {
@@ -93,7 +93,7 @@
          "value": "ON"
        },
        "CCAPI_LOG_LEVEL": "Info",
-        "CONFIG_DEFAULTS_PATH": "config/profiles/md_dev/*.conf;config/profiles/default/*.conf"
+        "CONFIG_DEFAULTS_PATH": "spark/config/profiles/md_dev/*.conf;spark/config/profiles/default/*.conf"
      }
    },

@@ -104,7 +104,7 @@
      "cacheVariables": {
        "CMAKE_BUILD_TYPE": "Release",
        "CCAPI_LOG_LEVEL": "Warn",
-        "CONFIG_DEFAULTS_PATH": "config/profiles/md_prod/*.conf;config/profiles/default/*.conf"
+        "CONFIG_DEFAULTS_PATH": "spark/config/profiles/md_prod/*.conf;spark/config/profiles/default/*.conf"
      }
    },

@@ -186,7 +186,11 @@
        "CMAKE_CXX_FLAGS_DEBUG": "-g -fPIC -Wall -pedantic -Wextra -Werror -Wno-missing-braces -Wno-ignored-qualifiers --std=c++20 -fdiagnostics-color -DDEBUG",
        "CMAKE_CXX_FLAGS_RELEASE": "-fPIC -Wall -pedantic -Wextra -Werror -Wno-missing-braces -Wno-ignored-qualifiers --std=c++20 -fdiagnostics-color -fno-omit-frame-pointer -DNDEBUG -O3",
        "CMAKE_INSTALL_PREFIX": "${sourceDir}/install/${presetName}",
-        "CONFIG_DEFAULTS_PATH": "config/profiles/default/*.conf",
+        "CMAKE_EXPORT_COMPILE_COMMANDS": {
+          "type": "BOOL",
+          "value": "ON"
+        },
+        "CONFIG_DEFAULTS_PATH": "spark/config/profiles/default/*.conf",
        "DOTNET_BINARY_DIR": "${sourceDir}/build/$<CONFIG>/bin/net8.0",
        "EXPORT_MD_CONSUMER": {
          "type": "BOOL",
@@ -227,7 +231,7 @@
      "cacheVariables": {
        "APP_DATA_DIR": "${dollar}{APPDATA}/bts/DataStream",
        "APP_DATA_DIR_OVERRIDE": "${dollar}{DATA_STREAM_DATA_DIR}",
-        "CONFIG_DEFAULTS_PATH": "config/profiles/default/*.conf;config/profiles/data_stream/*.conf",
+        "CONFIG_DEFAULTS_PATH": "spark/config/profiles/default/*.conf;spark/config/profiles/data_stream/*.conf",
        "SHARED_MEMORY_PREFIX": "data_stream"
      }
    },
@@ -262,8 +266,7 @@
        "CMAKE_CXX_COMPILER": "clang++",
        "CMAKE_CXX_FLAGS_DEBUG": "-g -fPIC -Wall -pedantic -Wextra -Werror -Wno-missing-braces -Wno-ignored-qualifiers --std=c++20 -fdiagnostics-color -DDEBUG",
        "CMAKE_CXX_FLAGS_RELEASE": "-fPIC -Wall -pedantic -Wextra -Werror -Wno-missing-braces -Wno-ignored-qualifiers --std=c++20 -fdiagnostics-color -fno-omit-frame-pointer -DNDEBUG -O3",
-        "CMAKE_INSTALL_PREFIX": "${sourceDir}/install/${presetName}",
-        "CONFIG_DEFAULTS_PATH": "config/profiles/default/*.conf;config/profiles/data_stream/*.conf",
+        "CONFIG_DEFAULTS_PATH": "spark/config/profiles/default/*.conf;spark/config/profiles/data_stream/*.conf",
        "DOTNET_BINARY_DIR": "${sourceDir}/build/$<CONFIG>/bin/net7.0",
        "EXPORT_MD_CONSUMER": {
          "type": "BOOL",
@@ -0,0 +1,7 @@
+include_guard(GLOBAL)
+
+macro(export_to_parent)
+  foreach(var IN ITEMS ${ARGN})
+    set(${var} ${${var}} PARENT_SCOPE)
+  endforeach()
+endmacro()
@@ -0,0 +1,142 @@
+include_guard(DIRECTORY)
+include(${CMAKE_SOURCE_DIR}/cmake/ExportToParent.cmake)
+include(${CMAKE_SOURCE_DIR}/cmake/InstallBasic.cmake)
+include(${CMAKE_SOURCE_DIR}/cmake/ProtobufGenerateCpp.cmake)
+include(${CMAKE_SOURCE_DIR}/cmake/SetFirstDefined.cmake)
+include(${CMAKE_SOURCE_DIR}/cmake/safe_dep_glob.cmake)
+
+function(init_library)
+  set(_options SHARED STATIC)
+  set(_one_value_args
+    NAME
+    SOURCE_DIR
+    INCLUDE_DIR
+  )
+  set(_multi_value_args)
+  cmake_parse_arguments(_ARGS "${_options}" "${_one_value_args}" "${_multi_value_args}" ${ARGN})
+  if (_ARGS_SHARED AND _ARGS_STATIC)
+    message(FATAL_ERROR "Invalid args passed to init_library: cannot specify STATIC and SHARED")
+  endif()
+  set_first_defined(_name _ARGS_NAME PROJECT_NAME)
+  set_first_defined(_source_dir _ARGS_SOURCE_DIR _name)
+  set_first_defined(_include_dir _ARGS_INCLUDE_DIR _source_dir)
+
+  # Cpp sources
+  file(
+    GLOB_RECURSE ${_name}_SOURCES
+    RELATIVE ${CMAKE_CURRENT_SOURCE_DIR}
+    ${_source_dir}/*.cpp
+  )
+  # Note: cpp files inside 'tests' dir(s) are considered tests.
+  # Remove tests from main sources, and add them to test sources
+  foreach(item ${${_name}_SOURCES})
+    if(${item} MATCHES "tests/.*\.cpp$")
+      list(APPEND ${_name}_TEST_SOURCES ${item})
+    endif()
+  endforeach()
+  foreach(item ${${_name}_TEST_SOURCES})
+    list(REMOVE_ITEM ${_name}_SOURCES ${item})
+  endforeach()
+
+  # Proto sources 
+  file(
+    GLOB_RECURSE ${_name}_PROTO_SOURCES
+    RELATIVE ${CMAKE_CURRENT_SOURCE_DIR}
+    ${_source_dir}/protobufs/*.proto
+  )
+  update_deps_file("${${_name}_PROTO_SOURCES};")
+  protobuf_generate_cpp(
+    CXX_SOURCES ${_name}_CXX_PROTO_SOURCES
+    CXX_HEADERS ${_name}_CXX_PROTO_HEADERS
+    CXX_OUT_DIR ${CMAKE_CURRENT_BINARY_DIR}
+    PROTO_INCLUDE_DIRS
+      ${EXTERNALS_INCLUDE}
+      ${CMAKE_CURRENT_SOURCE_DIR}
+    PROTO_SOURCES
+      ${${_name}_PROTO_SOURCES}
+  )
+
+  export_to_parent(
+    ${_name}_SOURCES
+    ${_name}_TEST_SOURCES
+    ${_name}_PROTO_SOURCES
+    ${_name}_CXX_PROTO_SOURCES
+    ${_name}_CXX_PROTO_HEADERS
+  )
+
+  # Add main lib
+  if (_ARGS_STATIC)
+    add_library(${_name} STATIC ${${_name}_SOURCES} ${${_name}_CXX_PROTO_SOURCES})
+  elseif(_ARGS_SHARED)
+    add_library(${_name} SHARED ${${_name}_SOURCES} ${${_name}_CXX_PROTO_SOURCES})
+  else()
+    add_library(${_name} ${${_name}_SOURCES} ${${_name}_CXX_PROTO_SOURCES})
+  endif()
+  target_include_directories(
+    ${_name}
+    SYSTEM
+    BEFORE
+    PUBLIC
+      $<BUILD_INTERFACE:${EXTERNALS_ROOT}/include>
+      $<BUILD_INTERFACE:${EXTERNALS_ROOT}/include/zlib>
+  )
+  target_include_directories(
+    ${_name}
+    PUBLIC
+      $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}>
+      $<INSTALL_INTERFACE:${CMAKE_INSTALL_INCLUDEDIR}>
+  )
+  # Include protos (if any)
+  if (${_name}_PROTO_SOURCES)
+    target_include_directories(
+      ${_name}
+      PUBLIC
+        $<BUILD_INTERFACE:${CMAKE_CURRENT_BINARY_DIR}>
+    )
+  endif()
+  target_link_directories(
+    ${_name}
+    PUBLIC
+      $<BUILD_INTERFACE:${CMAKE_CURRENT_BINARY_DIR}>
+      $<BUILD_INTERFACE:${EXTERNALS_ROOT}/lib/$<CONFIG>>
+      $<INSTALL_INTERFACE:${CMAKE_INSTALL_LIBDIR}>
+  )
+  # Install main lib
+  foreach(_hdr ${${_name}_CXX_PROTO_HEADERS})
+    file(RELATIVE_PATH _relative_path ${CMAKE_CURRENT_BINARY_DIR} ${_hdr})
+    install(FILES ${_hdr} DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}/${_relative_path})
+  endforeach()
+  install_basic(
+    NAME ${_name}
+    INCLUDE_DIR ${_include_dir}
+  )
+
+  # Add tests executable (if there are any)
+  if (${_name}_TEST_SOURCES AND NOT WIN32)
+    add_executable(${_name}-tests ${${_name}_TEST_SOURCES})
+
+    get_target_property(_lib_type ${_name} TYPE)
+    if (_lib_type STREQUAL "SHARED_LIBRARY")
+      # Set RPATH so the tests exe knows where to find the shared lib.
+      set_target_properties(
+        ${_name}-tests
+        PROPERTIES
+          BUILD_RPATH "$ORIGIN"
+          INSTALL_RPATH "$ORIGIN/../${CMAKE_INSTALL_LIBDIR}"
+      )
+    endif()
+
+    # Link main lib
+    target_link_directories(
+      ${_name}-tests
+      PRIVATE
+        $<BUILD_INTERFACE:${CMAKE_CURRENT_BINARY_DIR}>
+        $<INSTALL_INTERFACE:${CMAKE_INSTALL_LIBDIR}>
+    )
+    target_link_libraries(${_name}-tests PRIVATE ${_name})
+
+    # Catch2
+    find_package(Catch2 CONFIG REQUIRED COMPONENTS Catch2WithMain)
+    target_link_libraries(${_name}-tests PRIVATE Catch2::Catch2WithMain)
+  endif()
+endfunction()
@@ -0,0 +1,39 @@
+include_guard(DIRECTORY)
+include(GNUInstallDirs)
+include(${CMAKE_SOURCE_DIR}/cmake/PackBasic.cmake)
+include(${CMAKE_SOURCE_DIR}/cmake/SetFirstDefined.cmake)
+
+function(install_basic)
+  set(_options)
+  set(_one_value_args NAME INCLUDE_DIR)
+  set(_multi_value_args)
+  cmake_parse_arguments(_ARGS "${_options}" "${_one_value_args}" "${_multi_value_args}" ${ARGN})
+  set_first_defined(_name _ARGS_NAME PROJECT_NAME)
+  set_first_defined(_include_dir _ARGS_INCLUDE_DIR _name)
+
+  file(
+    GLOB_RECURSE ${_name}_HEADERS
+    RELATIVE ${CMAKE_CURRENT_SOURCE_DIR}
+    ${_include_dir}/*.h)
+
+  target_sources(${_name}
+    PUBLIC
+      FILE_SET HEADERS FILES ${${_name}_HEADERS})
+
+  install(
+    TARGETS ${_name}
+    EXPORT ${_name}Targets
+    RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR}
+    LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR}
+    ARCHIVE DESTINATION ${CMAKE_INSTALL_LIBDIR}
+    FILE_SET HEADERS DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}
+  )
+
+  install(
+    EXPORT ${_name}Targets
+    FILE ${_name}Targets.cmake
+    NAMESPACE ${_name}::
+    DESTINATION ${CMAKE_INSTALL_LIBDIR}/cmake/${_name})
+
+  pack_basic()
+endfunction()
@@ -0,0 +1,44 @@
+include_guard(DIRECTORY)
+include(CMakePackageConfigHelpers)
+include(${CMAKE_SOURCE_DIR}/cmake/SetFirstDefined.cmake)
+
+function(pack_basic)
+  set(_options)
+  set(_one_value_args NAME)
+  set(_multi_value_args)
+  cmake_parse_arguments(_ARGS "${_options}" "${_one_value_args}" "${_multi_value_args}" ${ARGN})
+  set_first_defined(_name _ARGS_NAME PROJECT_NAME)
+
+  find_package(Git QUIET)
+
+  if(GIT_FOUND)
+    execute_process(
+      COMMAND ${GIT_EXECUTABLE} describe --tags --always --dirty
+      WORKING_DIRECTORY ${CMAKE_SOURCE_DIR}
+      OUTPUT_VARIABLE PROJECT_VERSION
+      OUTPUT_STRIP_TRAILING_WHITESPACE
+    )
+  endif()
+
+  if(NOT GIT_FOUND OR PROJECT_VERSION STREQUAL "")
+    set(PROJECT_VERSION "0.0.0")
+  endif()
+
+  write_basic_package_version_file(
+    "${CMAKE_CURRENT_BINARY_DIR}/${_name}ConfigVersion.cmake"
+    VERSION ${PROJECT_VERSION}
+    COMPATIBILITY AnyNewerVersion
+  )
+
+  configure_package_config_file(
+    "${CMAKE_CURRENT_SOURCE_DIR}/Config.cmake.in"
+    "${CMAKE_CURRENT_BINARY_DIR}/${_name}Config.cmake"
+    INSTALL_DESTINATION ${CMAKE_INSTALL_LIBDIR}/cmake/${_name}
+  )
+
+  install(
+    FILES
+      "${CMAKE_CURRENT_BINARY_DIR}/${_name}Config.cmake"
+      "${CMAKE_CURRENT_BINARY_DIR}/${_name}ConfigVersion.cmake"
+    DESTINATION ${CMAKE_INSTALL_LIBDIR}/cmake/${_name})
+endfunction()
@@ -1,12 +1,23 @@
-set(Protobuf_PROTOC_EXECUTABLE "${SPARK_BIN}/protoc")
-set(PROTOBUF_INCLUDE_DIRS "${SPARK_INCLUDE}")
+set(Protobuf_PROTOC_EXECUTABLE "${EXTERNALS_BIN}/protoc")
+set(PROTOBUF_INCLUDE_DIRS "${EXTERNALS_INCLUDE}")
 set(PROTOBUF_STATIC_LIB "libprotobuf")
+set(_protobuf_src_dir ${CMAKE_SOURCE_DIR}/src/lib_spark)
+set(_protobuf_bin_dir ${CMAKE_BINARY_DIR}/src/lib_spark)

-function(Protobuf_Generate_Cpp SRCS HDRS)
-  set(${SRCS})
-  set(${HDRS})
+function(Protobuf_Generate_Cpp)
+  set(_options)
+  set(_one_value_args CXX_SOURCES CXX_HEADERS CXX_OUT_DIR)
+  set(_multi_value_args PROTO_INCLUDE_DIRS PROTO_SOURCES)
+  cmake_parse_arguments(_ARGS "${_options}" "${_one_value_args}" "${_multi_value_args}" ${ARGN})
+  set(${_ARGS_CXX_SOURCES})
+  set(${_ARGS_CXX_HEADERS})

-  foreach(FIL ${ARGN})
+  set(_proto_include_cli_args)
+  foreach(_dir ${_ARGS_PROTO_INCLUDE_DIRS})
+    list(APPEND _proto_include_cli_args -I ${_dir})
+  endforeach()
+
+  foreach(FIL ${_ARGS_PROTO_SOURCES})
    get_filename_component(ABS_FIL ${FIL} ABSOLUTE)
    get_filename_component(FIL_WE ${FIL} NAME_WE)
    get_filename_component(FIL_DIR ${FIL} DIRECTORY)
@@ -14,48 +25,56 @@ function(Protobuf_Generate_Cpp SRCS HDRS)
      set(FIL_WE "${FIL_DIR}/${FIL_WE}")
    endif()

-    list(APPEND ${SRCS} "${CMAKE_CURRENT_BINARY_DIR}/${FIL_WE}.pb.cc")
-    list(APPEND ${HDRS} "${CMAKE_CURRENT_BINARY_DIR}/${FIL_WE}.pb.h")
+    list(APPEND ${_ARGS_CXX_SOURCES} "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.pb.cc")
+    list(APPEND ${_ARGS_CXX_HEADERS} "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.pb.h")

    add_custom_command(
-      OUTPUT "${CMAKE_CURRENT_BINARY_DIR}/${FIL_WE}.pb.cc"
-             "${CMAKE_CURRENT_BINARY_DIR}/${FIL_WE}.pb.h"
+      OUTPUT "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.pb.cc"
+             "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.pb.h"
      COMMAND
-        ${Protobuf_PROTOC_EXECUTABLE} ARGS --cpp_out=${CMAKE_BINARY_DIR}/src -I
-        ${PROTOBUF_INCLUDE_DIRS} -I ${CMAKE_SOURCE_DIR}/src ${ABS_FIL}
+        ${Protobuf_PROTOC_EXECUTABLE} ARGS --cpp_out=${_ARGS_CXX_OUT_DIR} ${_proto_include_cli_args} ${ABS_FIL}
      DEPENDS ${ABS_FIL}
      COMMENT "Running C++ protocol buffer compiler on ${FIL}"
      VERBATIM)
  endforeach()

-  set_source_files_properties(${${SRCS}} ${${HDRS}} PROPERTIES GENERATED TRUE)
+  set_source_files_properties(${${_ARGS_CXX_SOURCES}} ${${_ARGS_CXX_HEADERS}} PROPERTIES GENERATED TRUE)

  # remove some warnings from protobuf generated files
  if(WIN32)
-    set_source_files_properties(${${SRCS}} PROPERTIES COMPILE_FLAGS
+    set_source_files_properties(${${_ARGS_CXX_SOURCES}} PROPERTIES COMPILE_FLAGS
                                                      "/wd4309 /wd4838")
  else()
    set_source_files_properties(
-      ${${SRCS}}
+      ${${_ARGS_CXX_SOURCES}}
      PROPERTIES
        COMPILE_FLAGS
        "-Wno-unused-parameter -Wno-effc++ -Wno-pedantic -Wno-sign-compare -Wno-uninitialized"
    )
  endif()

-  set(${SRCS}
-      ${${SRCS}}
+  set(${_ARGS_CXX_SOURCES}
+      ${${_ARGS_CXX_SOURCES}}
      PARENT_SCOPE)
-  set(${HDRS}
-      ${${HDRS}}
+  set(${_ARGS_CXX_HEADERS}
+      ${${_ARGS_CXX_HEADERS}}
      PARENT_SCOPE)
 endfunction()

-function(Service_Compiler_Generate_Cpp SRCS HDRS)
-  set(${SRCS})
-  set(${HDRS})
+function(Service_Compiler_Generate_Cpp)
+  set(_options)
+  set(_one_value_args CXX_SOURCES CXX_HEADERS CXX_OUT_DIR)
+  set(_multi_value_args PROTO_INCLUDE_DIRS PROTO_SOURCES)
+  cmake_parse_arguments(_ARGS "${_options}" "${_one_value_args}" "${_multi_value_args}" ${ARGN})
+  set(${_ARGS_CXX_SOURCES})
+  set(${_ARGS_CXX_HEADERS})

-  foreach(FIL ${ARGN})
+  set(_proto_include_cli_args)
+  foreach(_dir ${_ARGS_PROTO_INCLUDE_DIRS})
+    list(APPEND _proto_include_cli_args -I ${_dir})
+  endforeach()
+
+  foreach(FIL ${_ARGS_PROTO_SOURCES})
    get_filename_component(ABS_FIL ${FIL} ABSOLUTE)
    get_filename_component(FIL_WE ${FIL} NAME_WE)
    get_filename_component(FIL_DIR ${FIL} DIRECTORY)
@@ -63,58 +82,58 @@ function(Service_Compiler_Generate_Cpp SRCS HDRS)
      set(FIL_WE "${FIL_DIR}/${FIL_WE}")
    endif()

-    list(APPEND ${SRCS} "${CMAKE_CURRENT_BINARY_DIR}/${FIL_WE}.pb.cc")
-    list(APPEND ${HDRS} "${CMAKE_CURRENT_BINARY_DIR}/${FIL_WE}.pb.h")
+    list(APPEND ${_ARGS_CXX_SOURCES} "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.pb.cc")
+    list(APPEND ${_ARGS_CXX_HEADERS} "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.pb.h")

-    list(APPEND ${SRCS} "${CMAKE_CURRENT_BINARY_DIR}/${FIL_WE}.client.pb.cc")
-    list(APPEND ${HDRS} "${CMAKE_CURRENT_BINARY_DIR}/${FIL_WE}.client.pb.h")
+    list(APPEND ${_ARGS_CXX_SOURCES} "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.client.pb.cc")
+    list(APPEND ${_ARGS_CXX_HEADERS} "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.client.pb.h")

-    list(APPEND ${SRCS} "${CMAKE_CURRENT_BINARY_DIR}/${FIL_WE}.server.pb.cc")
-    list(APPEND ${HDRS} "${CMAKE_CURRENT_BINARY_DIR}/${FIL_WE}.server.pb.h")
+    list(APPEND ${_ARGS_CXX_SOURCES} "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.server.pb.cc")
+    list(APPEND ${_ARGS_CXX_HEADERS} "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.server.pb.h")

    set(TEMPLATES
-        ${CMAKE_CURRENT_SOURCE_DIR}/api/service_compiler/templates/cpp/client.pb.cc
-        ${CMAKE_CURRENT_SOURCE_DIR}/api/service_compiler/templates/cpp/client.pb.h
-        ${CMAKE_CURRENT_SOURCE_DIR}/api/service_compiler/templates/cpp/server.pb.cc
-        ${CMAKE_CURRENT_SOURCE_DIR}/api/service_compiler/templates/cpp/server.pb.h
+        ${CMAKE_SOURCE_DIR}/src/lib_spark/spark/api/service_compiler/templates/cpp/client.pb.cc
+        ${CMAKE_SOURCE_DIR}/src/lib_spark/spark/api/service_compiler/templates/cpp/client.pb.h
+        ${CMAKE_SOURCE_DIR}/src/lib_spark/spark/api/service_compiler/templates/cpp/server.pb.cc
+        ${CMAKE_SOURCE_DIR}/src/lib_spark/spark/api/service_compiler/templates/cpp/server.pb.h
    )

+    set(_plugin_py ${CMAKE_SOURCE_DIR}/src/lib_spark/spark/api/service_compiler/plugin.py)
    if(WIN32)
+      set(_plugin_cmd ${CMAKE_SOURCE_DIR}/src/lib_spark/spark/api/service_compiler/plugin.cmd)
      add_custom_command(
-        OUTPUT "${CMAKE_CURRENT_BINARY_DIR}/${FIL_WE}.pb.cc"
-               "${CMAKE_CURRENT_BINARY_DIR}/${FIL_WE}.pb.h"
-               "${CMAKE_CURRENT_BINARY_DIR}/${FIL_WE}.client.pb.cc"
-               "${CMAKE_CURRENT_BINARY_DIR}/${FIL_WE}.client.pb.h"
-               "${CMAKE_CURRENT_BINARY_DIR}/${FIL_WE}.server.pb.cc"
-               "${CMAKE_CURRENT_BINARY_DIR}/${FIL_WE}.server.pb.h"
+        OUTPUT "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.pb.cc"
+               "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.pb.h"
+               "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.client.pb.cc"
+               "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.client.pb.h"
+               "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.server.pb.cc"
+               "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.server.pb.h"
        COMMAND SET PYTHONPATH=${SPARK_PYTHON}
        COMMAND ${CMAKE_SOURCE_DIR}/.venv/Scripts/activate.bat
        COMMAND
-          ${Protobuf_PROTOC_EXECUTABLE} ARGS --cpp_out=${CMAKE_BINARY_DIR}/src
-          --plugin=protoc-gen-service=${CMAKE_SOURCE_DIR}/src/lib/api/service_compiler/plugin.cmd
-          --service_out=lang=cpp:${CMAKE_BINARY_DIR}/src -I
-          ${PROTOBUF_INCLUDE_DIRS} -I ${CMAKE_SOURCE_DIR}/src ${ABS_FIL}
-        DEPENDS ${ABS_FIL} ${TEMPLATES}
-                ${CMAKE_CURRENT_SOURCE_DIR}/api/service_compiler/plugin.py
+          ${Protobuf_PROTOC_EXECUTABLE} ARGS --cpp_out=${_ARGS_CXX_OUT_DIR}
+          --plugin=protoc-gen-service=${_plugin_cmd}
+          --service_out=lang=cpp:${_ARGS_CXX_OUT_DIR} -I
+          ${PROTOBUF_INCLUDE_DIRS} -I ${_protobuf_src_dir} ${ABS_FIL}
+        DEPENDS ${ABS_FIL} ${TEMPLATES} ${_plugin_cmd} ${_plugin_py}
        COMMENT "Running C++ service compiler on ${FIL}"
        VERBATIM)
    else()
      add_custom_command(
-        OUTPUT "${CMAKE_CURRENT_BINARY_DIR}/${FIL_WE}.pb.cc"
-               "${CMAKE_CURRENT_BINARY_DIR}/${FIL_WE}.pb.h"
-               "${CMAKE_CURRENT_BINARY_DIR}/${FIL_WE}.client.pb.cc"
-               "${CMAKE_CURRENT_BINARY_DIR}/${FIL_WE}.client.pb.h"
-               "${CMAKE_CURRENT_BINARY_DIR}/${FIL_WE}.server.pb.cc"
-               "${CMAKE_CURRENT_BINARY_DIR}/${FIL_WE}.server.pb.h"
+        OUTPUT "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.pb.cc"
+               "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.pb.h"
+               "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.client.pb.cc"
+               "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.client.pb.h"
+               "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.server.pb.cc"
+               "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.server.pb.h"
        COMMAND ${CMAKE_SOURCE_DIR}/tools/venv_exec.sh
        ARGS
          ${Protobuf_PROTOC_EXECUTABLE}
-          --cpp_out=${CMAKE_BINARY_DIR}/src
-          --plugin=protoc-gen-service=${CMAKE_SOURCE_DIR}/src/lib/api/service_compiler/plugin.py
-          --service_out=lang=cpp,make_file_name=${ABS_FIL}:${CMAKE_BINARY_DIR}/src
-          -I ${PROTOBUF_INCLUDE_DIRS} -I ${CMAKE_SOURCE_DIR}/src ${ABS_FIL}
-        DEPENDS ${ABS_FIL} ${TEMPLATES}
-                ${CMAKE_CURRENT_SOURCE_DIR}/api/service_compiler/plugin.py
+          --cpp_out=${_ARGS_CXX_OUT_DIR}
+          --plugin=protoc-gen-service=${_plugin_py}
+          --service_out=lang=cpp,make_file_name=${ABS_FIL}:${_ARGS_CXX_OUT_DIR}
+          ${_proto_include_cli_args} ${ABS_FIL}
+        DEPENDS ${ABS_FIL} ${TEMPLATES} ${_plugin_py}
        COMMENT "Running C++ service compiler on ${FIL}"
        VERBATIM)
    endif()
@@ -123,22 +142,22 @@ function(Service_Compiler_Generate_Cpp SRCS HDRS)

  # remove some warnings from protobuf generated files
  if(WIN32)
-    set_source_files_properties(${${SRCS}} PROPERTIES COMPILE_FLAGS
+    set_source_files_properties(${${_ARGS_CXX_SOURCES}} PROPERTIES COMPILE_FLAGS
                                                      "/wd4309 /wd4838")
  else()
    set_source_files_properties(
-      ${${SRCS}}
+      ${${_ARGS_CXX_SOURCES}}
      PROPERTIES
        COMPILE_FLAGS
        "-Wno-unused-parameter -Wno-effc++ -Wno-pedantic -Wno-sign-compare -Wno-uninitialized"
    )
  endif()

-  set(${SRCS}
-      ${${SRCS}}
+  set(${_ARGS_CXX_SOURCES}
+      ${${_ARGS_CXX_SOURCES}}
      PARENT_SCOPE)
-  set(${HDRS}
-      ${${HDRS}}
+  set(${_ARGS_CXX_HEADERS}
+      ${${_ARGS_CXX_HEADERS}}
      PARENT_SCOPE)

 endfunction()
@@ -0,0 +1,83 @@
+include_guard(GLOBAL)
+
+function(protobuf_generate_cpp)
+  set(_options)
+  set(_one_value_args
+    CXX_SOURCES
+    CXX_HEADERS
+    CXX_OUT_DIR
+    PROTOC
+  )
+  set(_multi_value_args
+    PROTO_INCLUDE_DIRS
+    PROTO_SOURCES
+  )
+  cmake_parse_arguments(_ARGS "${_options}" "${_one_value_args}" "${_multi_value_args}" ${ARGN})
+
+  if (NOT DEFINED _ARGS_PROTOC)
+    if (WIN32)
+      set(_ARGS_PROTOC "${EXTERNALS_BIN}/protoc.exe")
+    else()
+      set(_ARGS_PROTOC "${EXTERNALS_BIN}/protoc")
+    endif()
+  endif()
+  set(${_ARGS_CXX_SOURCES})
+  set(${_ARGS_CXX_HEADERS})
+
+  set(_proto_include_cli_args)
+  foreach(_dir ${_ARGS_PROTO_INCLUDE_DIRS})
+    list(APPEND _proto_include_cli_args -I ${_dir})
+  endforeach()
+
+  foreach(_fil ${_ARGS_PROTO_SOURCES})
+    get_filename_component(_abs_fil ${_fil} ABSOLUTE)
+    get_filename_component(_fil_we ${_fil} NAME_WE)
+    get_filename_component(_fil_dir ${_fil} DIRECTORY)
+    if(_fil_dir)
+      set(_fil_we "${_fil_dir}/${_fil_we}")
+    endif()
+    list(APPEND ${_ARGS_CXX_SOURCES} "${_ARGS_CXX_OUT_DIR}/${_fil_we}.pb.cc")
+    list(APPEND ${_ARGS_CXX_HEADERS} "${_ARGS_CXX_OUT_DIR}/${_fil_we}.pb.h")
+    message(STATUS "add_custom_command for ${_fil} -> ${_ARGS_CXX_OUT_DIR}/${_fil_we}.pb.h")
+    # TODO: Pass proto sources by relative path (so this fn can be called from
+    # arbitrary subdirs).
+    add_custom_command(
+      OUTPUT
+        "${_ARGS_CXX_OUT_DIR}/${_fil_we}.pb.cc"
+        "${_ARGS_CXX_OUT_DIR}/${_fil_we}.pb.h"
+      COMMAND
+        ${_ARGS_PROTOC}
+      ARGS
+        --cpp_out=${_ARGS_CXX_OUT_DIR} ${_proto_include_cli_args} ${_abs_fil}
+      DEPENDS ${_abs_fil}
+      COMMENT "Running C++ protocol buffer compiler on ${_fil}"
+      VERBATIM
+    )
+  endforeach()
+
+  set_source_files_properties(
+    ${${_ARGS_CXX_SOURCES}}
+    ${${_ARGS_CXX_HEADERS}}
+    PROPERTIES
+      GENERATED TRUE
+  )
+
+  # remove some warnings from protobuf generated files
+  if(WIN32)
+    set_source_files_properties(
+      ${${_ARGS_CXX_SOURCES}}
+      PROPERTIES
+        COMPILE_FLAGS "/wd4309 /wd4838"
+    )
+  else()
+    set_source_files_properties(
+      ${${_ARGS_CXX_SOURCES}}
+      PROPERTIES
+        COMPILE_FLAGS "-Wno-unused-parameter -Wno-effc++ -Wno-pedantic -Wno-sign-compare -Wno-uninitialized"
+    )
+  endif()
+
+  set(${_ARGS_CXX_SOURCES} ${${_ARGS_CXX_SOURCES}} PARENT_SCOPE)
+  set(${_ARGS_CXX_HEADERS} ${${_ARGS_CXX_HEADERS}} PARENT_SCOPE)
+endfunction()
+
@@ -0,0 +1,61 @@
+include_guard(GLOBAL)
+
+function(protobuf_generate_cpp_config)
+  set(_options)
+  set(_one_value_args
+    PROTO_SOURCES_HEADER
+    PROTO_DEFAULTS_HEADER
+  )
+  set(_multi_value_args
+    PROTO_SOURCES
+    PROTO_DEFAULTS
+  )
+  cmake_parse_arguments(_ARGS "${_options}" "${_one_value_args}" "${_multi_value_args}" ${ARGN})
+
+  # Configure the 'config_proto_files' header. It provides an array of strings populated with the
+  # files in PROTO_SOURCES. At runtime, we use this array to generate a special 'config' proto
+  # message with fields for each message defined in said files. A global instance of this 'config'
+  # proto is then constructed, and populated with [compile-time] defaults, and [runtime] overrides
+  # (i.e., in spark.conf). Calls to `bts::config::get<T>()` retrieve the value from the field
+  # associated with type `T` in this global instance.
+  message(STATUS "PROTO_SOURCES: ${_ARGS_PROTO_SOURCES}")
+  message(STATUS "PROTO_SOURCES_HEADER: ${_ARGS_PROTO_SOURCES_HEADER}")
+  message(STATUS "PROTO_DEFAULTS_HEADER: ${_ARGS_PROTO_DEFAULTS_HEADER}")
+  if (_ARGS_PROTO_SOURCES_HEADER)
+    set(PROTO_SOURCES)
+    set(PROTO_SOURCES_CXX_ARRAY_ITEMS "")
+    foreach(_proto_path ${_ARGS_PROTO_SOURCES})
+      list(APPEND PROTO_SOURCES ${_proto_path})
+      string(APPEND PROTO_SOURCES_CXX_ARRAY_ITEMS "\"${_proto_path}\",\n")
+    endforeach()
+    list(LENGTH PROTO_SOURCES PROTO_SOURCES_LENGTH)
+    message(STATUS "CONFIGURING PROTO_SOURCES_HEADER): ${_ARGS_PROTO_SOURCES_HEADER}")
+    configure_file(${CMAKE_SOURCE_DIR}/cmake/config_proto_files.h.in ${_ARGS_PROTO_SOURCES_HEADER})
+  endif()
+
+  # Generate static defaults. Ideally, this would be a single string, but we use an array because
+  # the compiler complains if the string is longer than 65k.
+  set(_cfg_defaults_str "inline const char * config_static_defaults[] = { ")
+
+  foreach(_cfg_file_path ${_ARGS_PROTO_DEFAULTS})
+    file(READ ${_cfg_file_path} _cfg_file_content)
+    string(REPLACE "\"" "\\\"" _cfg_file_content_escaped ${_cfg_file_content})
+    string(REPLACE "\n" ";" _cfg_lines ${_cfg_file_content_escaped})
+
+    foreach(_line ${_cfg_lines})
+      string(APPEND _cfg_defaults_str "\n  \"${_line}\",")
+    endforeach(_line)
+  endforeach(_cfg_file_path)
+
+  string(APPEND _cfg_defaults_str "};")
+  set(_old_cfg_defaults_str "")
+
+  if(EXISTS ${_ARGS_PROTO_DEFAULTS_HEADER})
+    file(READ ${_ARGS_PROTO_DEFAULTS_HEADER} _old_cfg_defaults_str)
+  endif()
+
+  if(NOT _cfg_defaults_str STREQUAL _old_cfg_defaults_str)
+    file(WRITE "${_ARGS_PROTO_DEFAULTS_HEADER}" "${_cfg_defaults_str}")
+  endif()
+endfunction()
+
@@ -10,7 +10,7 @@ function(protobuf_generate_fs)
    set(_ARGS_TARGET_NAME "${PROJECT_NAME}_generate_fs_protos")
  endif()
  if (NOT DEFINED _ARGS_PROTOC)
-    set(_ARGS_PROTOC "${SPARK_BIN}/protoc")
+    set(_ARGS_PROTOC "${EXTERNALS_BIN}/protoc")
  endif()

  set(_proto_include_cli_args)
@@ -27,8 +27,8 @@ function(protobuf_generate_fs)
  endforeach()

  set(_venv_exec ${CMAKE_SOURCE_DIR}/tools/venv_exec.sh)
-  set(_plugin_py ${CMAKE_SOURCE_DIR}/src/lib/api/service_compiler/plugin.py)
-  set(_template_fs ${CMAKE_SOURCE_DIR}/src/lib/api/service_compiler/templates/fs/fable.fs)
+  set(_plugin_py ${CMAKE_SOURCE_DIR}/src/lib_spark/spark/api/service_compiler/plugin.py)
+  set(_template_fs ${CMAKE_SOURCE_DIR}/src/lib_spark/spark/api/service_compiler/templates/fs/fable.fs)
  add_custom_command(
    OUTPUT ${_outputs}
    WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}
@@ -0,0 +1,10 @@
+include_guard(GLOBAL)
+
+macro(set_first_defined RESULT)
+  foreach(_arg ${ARGN})
+    if (DEFINED ${_arg})
+      set(${RESULT} ${${_arg}})
+      break()
+    endif()
+  endforeach()
+endmacro()
@@ -0,0 +1,11 @@
+#pragma once
+
+#include <array>
+
+namespace bts::@PROJECT_NAME@::protobufs {
+
+inline std::array<const char *, @PROTO_SOURCES_LENGTH@> config_proto_files = {
+@PROTO_SOURCES_CXX_ARRAY_ITEMS@
+};
+
+} // namespace bts::@PROJECT_NAME@::protobufs
@@ -1,24 +0,0 @@
-cmake_minimum_required(VERSION 3.5)
-
-# Explicitly set policy to convert relative paths to absolute (although this is a default policy,
-# setting it explicitly prevents a compile-warning)
-cmake_policy(SET CMP0076 NEW)
-
-# add protos to the library to facilitate source_group
-target_sources(libspark PUBLIC ${Config_Protos} ${NonConfig_Protos} ${Service_Protos})
-
-function(assign_source_group)
-  foreach(_source IN ITEMS ${ARGN})
-    if(IS_ABSOLUTE "${_source}")
-      file(RELATIVE_PATH _source_rel "${CMAKE_CURRENT_SOURCE_DIR}" "${_source}")
-    else()
-      set(_source_rel "${_source}")
-    endif()
-    get_filename_component(_source_path "${_source_rel}" PATH)
-    string(REPLACE "/" "\\" _source_path_msvc "${_source_path}")
-    source_group("${_source_path_msvc}" FILES "${_source}")
-    set(SOURCE_FILES "${_source}")
-  endforeach()
-endfunction(assign_source_group)
-
-assign_source_group(${SparkLib_Sources} ${Config_Protos} ${NonConfig_Protos} ${Service_Protos})
@@ -3,7 +3,7 @@ services:
    volumes:
      - ${SPARKD__DATA_DIR:-sparkd_data}:/home/bts/sparkdata/data
    build:
-      dockerfile: src/server/sparkd/Dockerfile
+      dockerfile: src/app_sparkd/Dockerfile
      args:
        - IMAGE=${SPARKD__IMAGE}
        - SOURCE_DIR=${SPARKD__SOURCE_DIR}
@@ -68,6 +68,9 @@ services:
      - "${SPARKD__QUICKBUS_PORT}:${SPARKD__QUICKBUS_PORT}"
      - "${SPARKD__SIM_FIX_PORT}:${SPARKD__SIM_FIX_PORT}"
    deploy:
+      restart_policy:
+        condition: any
+        delay: 3s
      resources:
        limits:
          memory: "${SPARKD__MEMORY:-4gb}"
@@ -124,8 +124,8 @@ service OrderService {
 ```
 ### C++ Example
 ```cpp
-#include "lib/api/public/ServiceContext.h"
-#include "lib/api/services/OrderService.client.pb.h"
+#include "spark/api/public/ServiceContext.h"
+#include "spark/api/services/OrderService.client.pb.h"
 using namespace bts::api;
 ServiceContext ctx;
 auto orders = ctx.locate<orderman::OrderService>();
@@ -29,9 +29,9 @@ namespace sample {
 ```

 ```cpp
-#include "lib/api/public/ServiceContext.h"
-#include "spark/lib/domain_model/Security.h"
-#include "lib/api/services/MarketDataService.client.pb.h"
+#include "spark/api/public/ServiceContext.h"
+#include "spark/domain_model/Security.h"
+#include "spark/api/services/MarketDataService.client.pb.h"
 #include <thread>

 using namespace bts::api;
@@ -74,7 +74,7 @@ namespace BCG {
      }

      public void _on_stream_cb(object dp_obj) {
-         Log.Gen.info($"NonIncJoinBookkeeper._on_stream_cb dp_obj: {dp_obj}");
+         //Log.Gen.info($"NonIncJoinBookkeeper._on_stream_cb dp_obj: {dp_obj}");
         var timer = Stopwatch.StartNew();
         DataPacket dp = (DataPacket)dp_obj;
         _fill_missing_data(dp);
@@ -50,7 +50,7 @@ namespace BCG {
         int data_col_no = 1;
         for (int jp = 0; jp < _join_specs.Length; jp++) {
            NonIncJoinBookkeeper s = new NonIncJoinBookkeeper(_join_specs[jp], jp, data_col_no, _row_hasher);
-            Log.Gen.info($"NonIncJoinStream.myOnFirstSubscriber jp: {jp} spec: {_join_specs[jp]}");
+            //Log.Gen.info($"NonIncJoinStream.myOnFirstSubscriber jp: {jp} spec: {_join_specs[jp]}");
            data_col_no += _join_specs[jp].data_col_names.Length;
            s.on_new_data += _stream_callback;
            _book_keeping_data.Add(s);
@@ -71,7 +71,7 @@ namespace BCG {
      }

      private void _stream_callback() {
-         Log.Gen.info("NonIncJoinStream._stream_callback received callback");
+         //Log.Gen.info("NonIncJoinStream._stream_callback received callback");
         _is_dirty = true;
         doWork();
      }
@@ -99,7 +99,7 @@ namespace BCG {

      private int _n_total_fires = 0, _n_expensive_fires = 0;
      private void _form_and_fire() {
-         Log.Gen.info($"NonIncJoinStream._form_and_fire _is_ready(): {_is_ready()}");
+         //Log.Gen.info($"NonIncJoinStream._form_and_fire _is_ready(): {_is_ready()}");
         if (_is_ready()) {
            _construct_schema();
            var timer = Stopwatch.StartNew();
@@ -126,11 +126,11 @@ namespace BCG {
            _n_total_fires++;
            //if (timer.ElapsedMilliseconds >= 250) {
            _n_expensive_fires++;
-            string log_msg = $"Join and map time for {_book_keeping_data.Count} "
-               + $"tables {timer.Elapsed.TotalMilliseconds} msecs, ratio {_n_expensive_fires}/{_n_total_fires}";
-            Log.Gen.info(log_msg);
+            //string log_msg = $"Join and map time for {_book_keeping_data.Count} "
+            //   + $"tables {timer.Elapsed.TotalMilliseconds} msecs, ratio {_n_expensive_fires}/{_n_total_fires}";
+            //Log.Gen.info(log_msg);
            //}
-            var dp_out = BlueDataUtility.build_dp_inc(dp_output_rows, myLastValue as DataPacket);
+            var dp_out = BlueDataUtility.build_dp_inc(dp_output_rows, myLastValue as DataPacket, true);
            myFireEvent(dp_out);
            //myFireEvent(new DataPacket(dp_output_rows, dp_output_rows, myLastValue == null ? new BlueTable(_output_schema) : (myLastValue as DataPacket).newRows));
         }
@@ -152,7 +152,7 @@ namespace BCG {
      }

      public void doWork() {
-         Log.Gen.info($"NonIncJoinStream.doWork _last_fire_time: {_last_fire_time} _is_dirty: {_is_dirty}");
+         //Log.Gen.info($"NonIncJoinStream.doWork _last_fire_time: {_last_fire_time} _is_dirty: {_is_dirty}");
         if (DateTime.Now - _last_fire_time >= _throttle_period && _is_dirty) {
            _is_dirty = false;
            _last_fire_time = DateTime.Now;
@@ -37,4 +37,42 @@ namespace BCG
      }
         
   }
+
+   [DataStreamUserRespresentation("throttle")]
+   public class ThrottleStream : ViewStream{
+      private readonly ViewStream _main_stream;
+      private readonly TimeSpan _ts;
+      private readonly ViewStreamDatabase _vsd;
+      private DateTime _last_fire_time = DateTime.MinValue;
+
+      public ThrottleStream (ViewStream main_stream, int throttle_seconds)
+      :
+      this(main_stream, TimeSpan.FromSeconds(throttle_seconds), ViewStreamDatabase.instance)
+      {
+      }
+
+      public ThrottleStream (ViewStream main_stream, TimeSpan ts, ViewStreamDatabase vsd){
+         _main_stream = main_stream;
+         _ts = ts;
+         _vsd = vsd;
+      }
+
+       protected override void myOnFirstSubscriber() {
+         _main_stream.onChange += _handle_base_stream_update;
+      }
+
+      protected override void myOnLastSubscriber() {
+         _main_stream.onChange -= _handle_base_stream_update;
+      }
+
+      private void _handle_base_stream_update(object dp_obj){
+         if (DateTime.Now - _last_fire_time >= _ts){
+            DataPacket dp_in = dp_obj as DataPacket;
+            _last_fire_time = DateTime.Now;
+            DataPacket dp = BlueDataUtility.build_dp_inc(dp_in.newRows, myLastValue as DataPacket);
+            myFireEvent(dp);
+         }
+      }
+ }
+
 }
@@ -0,0 +1,40 @@
+namespace bts.SilverPine{
+   using BGC;
+   using System;
+   [DataStreamUserRespresentation("throttle")]
+   public class ThrottleStream : ViewStream{
+      private readonly ViewStream _main_stream;
+      private readonly TimeSpan _ts;
+      private readonly ViewStreamDatabase _vsd;
+      private DateTime _last_fire_time = DateTime.MinValue;
+
+      public ThrottleStream (ViewStream main_stream, int throttle_seconds)
+      :
+      this(main_stream, TimeSpan.FromSeconds(throttle_seconds), ViewStreamDatabase.instance)
+      {
+      }
+
+      public ThrottleStream (ViewStream main_stream, TimeSpan ts, ViewStreamDatabase vsd){
+         _main_stream = main_stream;
+         _ts = ts;
+         _vsd;
+      }
+
+       protected override void myOnFirstSubscriber() {
+         _main_stream.onChange += _handle_base_stream_update;
+      }
+
+      protected override void myOnLastSubscriber() {
+         _main_stream.onChange -= _handle_base_stream_update;
+      }
+
+      private void _handle_base_stream_update(object dp_obj){
+         if (DateTime.Now - _last_fire_time >= _ts){
+            DataPacket dp_in = dp_obj;
+            _last_fire_time = DateTime.Now;
+            DataPacket dp = BlueDataUtility.build_dp_inc(dp_in.newRow, myLastValue as DataPacket);
+            myFireEvent(dp);
+         }
+      }
+ }
+}
@@ -19,8 +19,8 @@
  </ItemGroup>

  <ItemGroup>
-    <Proto Include="../../lib/config/protos/data_stream_config.proto" CSName="DataStreamConfig">
-      <OutDir>$(ProtoOutputPath)/lib/config/protos</OutDir>
+    <Proto Include="../../lib_spark/spark/config/protos/data_stream_config.proto" CSName="DataStreamConfig">
+      <OutDir>$(ProtoOutputPath)/lib_spark/spark/config/protos</OutDir>
    </Proto>
  </ItemGroup>

@@ -0,0 +1,17 @@
+namespace bts.SilverPine {
+   using BCG;
+   using System;
+
+   [DataStreamUserRespresentation("SpatialArbScoreStream")]
+   public class SpatialArbScoreStream : ViewStream {
+      public SpatialArbScoreStream(int window_sz_secs, int sample_period_sz)
+         : this(TimeSpan.FromSeconds(window_sz_secs), TimeSpan.FromSeconds(sample_period_sz), ViewStreamDatabase.instance) { }
+
+      public SpatialArbScoreStream(TimeSpan window_sz, TimeSpan sample_period, ViewStreamDatabase vsd) {
+         ViewStream rs = new SpatialArbStatsStream(window_sz, sample_period, vsd);
+         rs = new ThrottleStream(rs, TimeSpan.FromSeconds(1), vsd);
+         rs = new SortStream(rs, "-n_trade_ticks");
+         mySetRealSource(rs);
+      }
+   }
+}
@@ -0,0 +1,113 @@
+namespace bts.SilverPine {
+   using BCG;
+
+   using SharpCompress.Common.Rar;
+
+   using System;
+   using System.Collections.Generic;
+   using System.Linq;
+   using System.Text;
+   using System.Threading.Tasks;
+
+   // For all the pairs generated by SpatialFuturePairsStream, this stream 
+   // generates the statistics, by appending together a SpreadStatsStream
+   [DataStreamUserRespresentation("SpatialArbStatsStream")]
+   public class SpatialArbStatsStream : ViewStream{
+      private ViewStream _spatial_pairs_stream;
+      private ViewStreamDatabase _vsd;
+      private ViewStream _tot_str = null;
+      private HashSet<string> _key_map = new HashSet<string>();
+      private ViewStream _bbo_stream;
+      private readonly TimeSpan _window_sz, _sampling_period_sz;
+
+      public SpatialArbStatsStream(int window_sz_seconds, int sampling_period_sz_seconds)
+         : this(TimeSpan.FromSeconds(window_sz_seconds), TimeSpan.FromSeconds(sampling_period_sz_seconds), ViewStreamDatabase.instance) { }
+
+      public SpatialArbStatsStream(TimeSpan window_sz, TimeSpan sampling_period_sz, ViewStreamDatabase vsd) {
+         _vsd = vsd;
+         _window_sz = window_sz;
+         _sampling_period_sz = sampling_period_sz;
+         _spatial_pairs_stream = new SpatialFuturePairsStream(vsd);
+         _bbo_stream = new BBOStream(vsd);
+      }
+
+      protected override void myOnFirstSubscriber() {
+         _spatial_pairs_stream.onChange += _spatial_pairs_stream_onchange;
+      }
+
+      protected override void myOnLastSubscriber() {
+         _spatial_pairs_stream.onChange -= _spatial_pairs_stream_onchange;
+      }
+
+      private ViewStream _build_mapped_stream(ViewStream parent_str) {
+         Func<IBlueTableSchema, IBlueTableSchema> build_schema = (inp_schema) => {
+            IBlueTableSchema rv = new BlueTableSchema("SpreaddStats", new ColumnInfo("id", 0, TypeCode.Int32));
+            rv = rv.add_column(TypeCode.String, "spread");
+            rv = rv.add_column(TypeCode.String, "window");
+            // bids
+            rv = rv.add_column(TypeCode.Double, "bid_mean");
+            rv = rv.add_column(TypeCode.Double, "bid_stddev");
+            rv = rv.add_column(TypeCode.Double, "bid_skewness");
+            rv = rv.add_column(TypeCode.Double, "bid_kurtosis");
+            rv = rv.add_column(TypeCode.Double, "bid_max");
+            rv = rv.add_column(TypeCode.Double, "bid_min");
+            // asks
+            rv = rv.add_column(TypeCode.Double, "ask_mean");
+            rv = rv.add_column(TypeCode.Double, "ask_stddev");
+            rv = rv.add_column(TypeCode.Double, "ask_skewness");
+            rv = rv.add_column(TypeCode.Double, "ask_kurtosis");
+            rv = rv.add_column(TypeCode.Double, "ask_max");
+            rv = rv.add_column(TypeCode.Double, "ask_min");
+            // tail distribution
+            rv = rv.add_column(TypeCode.Double, "bid Z ≥ 2σ");
+            rv = rv.add_column(TypeCode.Double, "ask Z < -2σ");
+            // trading ticks
+            rv = rv.add_column(TypeCode.Int32, "n_buy_ticks");
+            rv = rv.add_column(TypeCode.Double, "cumm_buy_price");
+            rv = rv.add_column(TypeCode.Int32, "n_sell_ticks");
+            rv = rv.add_column(TypeCode.Double, "cumm_sell_price");
+            rv = rv.add_column(TypeCode.Int32, "n_trade_ticks");
+            return rv;
+         };
+         Func<IBlueRow, IBlueRow> build_row = (r) => {
+            IBlueTableSchema s = build_schema(r.schema);
+            return new BlueRow(s, r["id"], r["spread"], r["window"],
+                           r["bid_mean"], r["bid_stddev"], r["bid_skewness"], r["bid_kurtosis"], r["bid_max"], r["bid_min"],
+                           r["ask_mean"], r["ask_stddev"], r["ask_skewness"], r["ask_kurtosis"], r["ask_max"], r["ask_min"],
+                           r["bid Z ≥ 2σ"], r["ask Z < -2σ"],
+                           r["n_buy_ticks"], r["cumm_buy_price"], r["n_sell_ticks"], r["cumm_sell_price"], r["n_trade_ticks"]);
+         };
+         MapStream rs = new MapStream(parent_str, inp_schema => build_schema(inp_schema),
+            inp_r => build_row(inp_r));
+         return rs;
+      }
+
+      private void _spatial_pairs_stream_onchange(object dp_obj) {
+         if (_tot_str != null)
+            return;
+         MutableAppendStream append_str = new MutableAppendStream();
+         _tot_str = _build_mapped_stream(append_str);
+         //_tot_str = new ThrottleStream(_tot_str, 1000);
+         //_tot_str = new SortStream(_tot_str, "-n_trade_ticks");
+         _tot_str.onChange += _handle_tot_stream;
+
+         DataPacket dp = dp_obj as DataPacket;         
+         foreach (IBlueRow r in dp.newRows) {
+            string s1 = r.get<string>("s1");
+            string s2 = r.get<string>("s2");
+            string key = SpatialFuturePairsStream.get_key(s1, s2);
+            if (!_key_map.Contains(key)) {
+               _key_map.Add(key);
+               ViewStream spread_stats_stream = new SpreadStatsStream(s1, s2, 1, 1, Convert.ToInt32(_window_sz.TotalSeconds), 
+                                                                     Convert.ToInt32(_sampling_period_sz.TotalSeconds), _bbo_stream, _vsd);
+               append_str.addStream(spread_stats_stream);
+            }
+         }
+      }
+
+      private void _handle_tot_stream(object dp_obj) {
+         myFireEvent(dp_obj);
+      }
+   }
+
+}
@@ -0,0 +1,89 @@
+namespace bts.SilverPine {
+   using BCG;
+
+   using bts.api;
+
+   using System;
+   using System.Collections.Generic;
+   using System.Linq;
+   using System.Text;
+   using System.Threading.Tasks;
+
+   [DataStreamUserRespresentation("SpatialFuturePairsStream")]
+   public class SpatialFuturePairsStream : ViewStream {
+      private ViewStream _main_stream;
+      private IBlueTableSchema _schema;
+      private HashSet<string> _key_map = new HashSet<string>();
+      public SpatialFuturePairsStream()
+         : this(ViewStreamDatabase.instance) { }
+
+      public SpatialFuturePairsStream(ViewStreamDatabase vsd) {
+         _main_stream = new InstrumentsStream(vsd);
+         _main_stream = new FilterStream(_main_stream, "sec_type='Future'");
+         _schema = new BlueTableSchema("SpatialFutures", new ColumnInfo("id", 0, TypeCode.Int32));
+         _schema = _schema.add_column(TypeCode.String, "s1");
+         _schema = _schema.add_column(TypeCode.String, "venue_1");
+         _schema = _schema.add_column(TypeCode.String, "s2");
+         _schema = _schema.add_column(TypeCode.String, "venue_2");
+
+      }
+
+      protected override void myOnFirstSubscriber() {
+         _main_stream.onChange += _main_stream_onchange;
+      }
+
+      protected override void myOnLastSubscriber() {
+         _main_stream.onChange -= _main_stream_onchange;
+      }
+
+      public static string get_key(string s1, string s2) {
+         string key = string.Compare(s1, s2, StringComparison.Ordinal) <= 0 ? s1 + s2 : s2 + s1;
+         return key;
+      }
+
+      public static string get_key(IBlueRow r1, IBlueRow r2) {
+         string s1 = r1.get<string>("sec");
+         string s2 = r2.get<string>("sec");
+         return get_key(s1, s2);
+      }
+
+      private bool _combine(IBlueRow r1, IBlueRow r2) {
+         string key = get_key(r1, r2);
+         if (_key_map.Contains(key))
+            return false;
+         if (r1.get<string>("symbol") != r2.get<string>("symbol"))
+            return false;
+         if (r1.get_enum<Venue>("venue") == r2.get_enum<Venue>("venue"))
+            return false;
+         if (r1.get<int>("sec_key") == r2.get<int>("sec_key")) {
+            return false;
+         }
+         DateTime e1 = r1.get_datetime("expiration");
+         DateTime e2 = r2.get_datetime("expiration");
+         bool exps_eq = e1 == e2 || (e1.Year >= 2100 && e2.Year >= 2100);
+         if (!exps_eq)
+            return false;         
+         return true;
+      }
+
+      private void _main_stream_onchange(object dp_obj) {
+         _key_map.Clear();
+         DataPacket dp = dp_obj as DataPacket;
+         BlueTable table = new BlueTable(_schema);
+         var rows = dp.newRows.to_list();
+         foreach (var r1 in rows) {
+            foreach (var r2 in rows) {
+               if (_combine(r1, r2)) {
+                  IBlueRow nr = new BlueRow(_schema, table.Count, r1.get<string>("sec"), r1.get<string>("venue"),
+                     r2.get<string>("sec"), r2.get<string>("venue"));
+                  table.addRow(nr);
+                  string key = get_key(r1, r2);
+                  _key_map.Add(key);
+               }
+            }
+         }
+         var dp_out = BlueDataUtility.build_dp(table, myLastValue as DataPacket);
+         myFireEvent(dp_out);
+      }
+   }
+}
@@ -0,0 +1,656 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using MathNet.Numerics.Statistics;
+using MathNet.Numerics.LinearRegression;
+using System.Collections.Generic;
+using System.Linq;
+using MathNet.Numerics.Statistics;
+using MathNet.Numerics.LinearAlgebra;
+using Amazon.Runtime;
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using MathNet.Numerics.Statistics;
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using MathNet.Numerics.Statistics;
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using MathNet.Numerics.Statistics;
+// stats calculator
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using MathNet.Numerics.Statistics;
+
+/// <summary>
+/// Provides evaluation metrics to assess the quality of a synthetic spread for mean-reversion trading.
+/// Assumes the input spread data is a trailing, time-causal window of recent ticks (e.g., last 120 seconds).
+/// </summary>
+public static class SpreadPerformanceEvaluator
+{
+
+   public static class SpreadStatistics {
+      public class SpreadStatsResult {
+         public Dictionary<string, double> bid_zscore_distribution { get; set; }
+         public Dictionary<string, double> ask_zscore_distribution { get; set; }
+         public Dictionary<string, double> mid_zscore_distribution { get; set; }
+
+         public DescriptiveStatistics bid_stats { get; set; }
+         public DescriptiveStatistics ask_stats { get; set; }
+         public DescriptiveStatistics mid_stats { get; set; }
+      }
+
+      private static readonly string[] _zscore_labels = new[]
+      {
+        "Z < -2σ",
+        "-2σ ≤ Z < -1σ",
+        "-1σ ≤ Z < 0σ",
+        "0σ ≤ Z < 1σ",
+        "1σ ≤ Z < 2σ",
+        "Z ≥ 2σ"
+    };
+
+      public static SpreadStatsResult analyze_spread_quotes(List<(DateTime timestamp, double bid, double bid_qty, double ask, double ask_qty)> quotes) {
+         if (quotes == null || quotes.Count == 0)
+            throw new ArgumentException("Spread quote list cannot be null or empty.");
+
+         var bid_prices = quotes.Select(q => q.bid).ToList();
+         var ask_prices = quotes.Select(q => q.ask).ToList();
+         var mid_prices = quotes.Select(q => (q.bid + q.ask) / 2.0).ToList();
+
+         var bid_stats = new DescriptiveStatistics(bid_prices);
+         var ask_stats = new DescriptiveStatistics(ask_prices);
+         var mid_stats = new DescriptiveStatistics(mid_prices);
+
+         var bid_zscores = bid_prices.Select(x => zscore(x, bid_stats.Mean, bid_stats.StandardDeviation)).ToList();
+         var ask_zscores = ask_prices.Select(x => zscore(x, ask_stats.Mean, ask_stats.StandardDeviation)).ToList();
+         var mid_zscores = mid_prices.Select(x => zscore(x, mid_stats.Mean, mid_stats.StandardDeviation)).ToList();
+
+         var bid_distribution = bin_zscores_to_percentages(bid_zscores);
+         var ask_distribution = bin_zscores_to_percentages(ask_zscores);
+         var mid_distribution = bin_zscores_to_percentages(mid_zscores);
+
+         return new SpreadStatsResult {
+            bid_zscore_distribution = bid_distribution,
+            ask_zscore_distribution = ask_distribution,
+            mid_zscore_distribution = mid_distribution,
+            bid_stats = bid_stats,
+            ask_stats = ask_stats,
+            mid_stats = mid_stats
+         };
+      }
+
+      private static double zscore(double x, double mean, double std_dev) {
+         return std_dev == 0 ? 0 : (x - mean) / std_dev;
+      }
+
+      private static Dictionary<string, double> bin_zscores_to_percentages(List<double> zscores) {
+         var bins = _zscore_labels.ToDictionary(label => label, _ => 0);
+
+         foreach (var z in zscores) {
+            if (z < -2) bins["Z < -2σ"]++;
+            else if (z < -1) bins["-2σ ≤ Z < -1σ"]++;
+            else if (z < 0) bins["-1σ ≤ Z < 0σ"]++;
+            else if (z < 1) bins["0σ ≤ Z < 1σ"]++;
+            else if (z < 2) bins["1σ ≤ Z < 2σ"]++;
+            else bins["Z ≥ 2σ"]++;
+         }
+
+         double total = zscores.Count;
+         return bins.ToDictionary(kvp => kvp.Key, kvp => total == 0 ? 0 : (kvp.Value / total) * 100.0);
+      }
+
+      public static double score_spread(SpreadStatsResult result) {
+         var ask_bins = result.ask_zscore_distribution;
+         var bid_bins = result.bid_zscore_distribution;
+         var mid_bins = result.mid_zscore_distribution;
+
+         double buy_opportunity = ask_bins["Z < -2σ"] + ask_bins["-2σ ≤ Z < -1σ"];
+         double sell_opportunity = bid_bins["Z ≥ 2σ"] + bid_bins["1σ ≤ Z < 2σ"];
+         double opportunity_score = buy_opportunity + sell_opportunity;
+
+         double below_mean = mid_bins["Z < -2σ"] + mid_bins["-2σ ≤ Z < -1σ"] + mid_bins["-1σ ≤ Z < 0σ"];
+         double above_mean = mid_bins["Z ≥ 2σ"] + mid_bins["1σ ≤ Z < 2σ"] + mid_bins["0σ ≤ Z < 1σ"];
+         double symmetry_score = 1.0 - Math.Abs(below_mean - above_mean) / (below_mean + above_mean + 1e-6);
+
+         double amplitude_score = result.mid_stats.StandardDeviation;
+         double skew_penalty = 1.0 / (1.0 + Math.Abs(result.mid_stats.Skewness));
+         double kurtosis_bonus = Math.Min(result.mid_stats.Kurtosis, 6.0) / 6.0;
+
+         double score =
+             opportunity_score *
+             symmetry_score *
+             amplitude_score *
+             skew_penalty *
+             (1.0 + kurtosis_bonus * 0.1);
+
+         return score;
+      }
+   }
+
+
+
+
+
+/// <summary>
+/// Represents a single trade action (entry or exit) taken during the simulation.
+/// </summary>
+public struct TradeAction
+    {
+        public bool IsEntry { get; set; }    // True if this action opened a position
+        public bool IsLong { get; set; }     // True = Long, False = Short
+        public double Price { get; set; }    // Executed price
+        public int Index { get; set; }       // Tick index of the action
+
+        public override string ToString()
+        {
+            string side = IsLong ? "Long" : "Short";
+            string type = IsEntry ? "Open" : "Close";
+            return $"{type} {side} @ {Price:F4} (Tick {Index})";
+        }
+    }
+
+    /// <summary>
+/// Simulates a mean-reversion strategy using rolling, causal mid-price stats.
+/// ✅ Entry:
+///   - Buy at ask when ask < rollingMean - entryThreshold × rollingStdDev
+///   - Sell at bid when bid > rollingMean + entryThreshold × rollingStdDev
+/// ✅ Exit:
+///   - Close when mid returns within rollingMean ± exitThreshold × rollingStdDev
+///   - Long exits at bid, short exits at ask
+/// This version uses only past and current data at each step (no forward-looking bias).
+/// </summary>
+public static (double TotalPnL, List<TradeAction> TradeActions, int BuyTurns, int SellTurns) EvaluateSimulatedPnL(
+    List<Tuple<double, double, double, double>> spreadData,
+    int statWindow = 30,
+    double entryThresholdMultiplier = 0.5,
+    double exitThresholdMultiplier = 0.1)
+{
+    var trades = new List<TradeAction>();
+    var mids = spreadData.Select(x => (x.Item1 + x.Item3) / 2.0).ToList();
+
+    if (spreadData.Count < statWindow)
+        return (0.0, trades, 0, 0);
+
+    bool positionOpen = false;
+    bool isLong = false;
+    double entryPrice = 0.0;
+    double pnl = 0.0;
+    int buyTurns = 0;
+    int sellTurns = 0;
+
+    for (int i = statWindow; i < spreadData.Count; i++)
+    {
+        double bid = spreadData[i].Item1;
+        double ask = spreadData[i].Item3;
+        double mid = mids[i];
+
+        // Rolling window
+        var window = mids.Skip(i - statWindow).Take(statWindow).ToList();
+        double mean = window.Average();
+        double stddev = Math.Sqrt(window.Select(x => Math.Pow(x - mean, 2)).Average());
+
+        double lowerThreshold = mean - entryThresholdMultiplier * stddev;
+        double upperThreshold = mean + entryThresholdMultiplier * stddev;
+        double exitBand = exitThresholdMultiplier * stddev;
+
+        if (!positionOpen)
+        {
+            if (ask < lowerThreshold)
+            {
+                positionOpen = true;
+                isLong = true;
+                entryPrice = ask;
+                trades.Add(new TradeAction { IsEntry = true, IsLong = true, Price = entryPrice, Index = i });
+            }
+            else if (bid > upperThreshold)
+            {
+                positionOpen = true;
+                isLong = false;
+                entryPrice = bid;
+                trades.Add(new TradeAction { IsEntry = true, IsLong = false, Price = entryPrice, Index = i });
+            }
+        }
+        else
+        {
+            if (Math.Abs(mid - mean) < exitBand)
+            {
+                double exitPrice = isLong ? bid : ask;
+                pnl += isLong ? (exitPrice - entryPrice) : (entryPrice - exitPrice);
+
+                trades.Add(new TradeAction { IsEntry = false, IsLong = isLong, Price = exitPrice, Index = i });
+
+                if (isLong) buyTurns++;
+                else sellTurns++;
+
+                positionOpen = false;
+            }
+        }
+    }
+
+    return (pnl, trades, buyTurns, sellTurns);
+}
+
+
+
+    /// <summary>
+    /// Estimates the Hurst exponent of the spread mid-prices.
+    /// Assumes the input is a trailing window of recent data.
+    /// Hurst < 0.5 implies mean-reversion; ideal is 0.35–0.45.
+    /// </summary>
+    public static double EvaluateHurstExponent(List<Tuple<double, double, double, double>> spreadData)
+    {
+        const int MinLength = 21;
+
+        if (spreadData == null || spreadData.Count < MinLength)
+            return double.NaN;
+
+        var spreadMids = spreadData.Select(x => (x.Item1 + x.Item3) / 2.0).ToArray();
+        int n = spreadMids.Length;
+
+        var lags = new List<double>();
+        var tau = new List<double>();
+
+        for (int lag = 2; lag <= 20; lag++)
+        {
+            if (n - lag <= 1)
+                continue;
+
+            double[] diffs = new double[n - lag];
+            for (int i = 0; i < diffs.Length; i++)
+                diffs[i] = spreadMids[i + lag] - spreadMids[i];
+
+            double stddev = diffs.StandardDeviation();
+
+            if (stddev > 0)
+            {
+                lags.Add(Math.Log(lag));
+                tau.Add(Math.Log(stddev));
+            }
+        }
+
+        if (lags.Count < 2)
+            return double.NaN;
+
+        var (intercept, slope) = SimpleRegression.Fit(lags.ToArray(), tau.ToArray());
+        double hurst = slope * 0.5 + 0.5;
+        return hurst;
+    }
+
+    /// <summary>
+/// Computes the mean and standard deviation of the spread mid-prices
+/// over a trailing, causal window.
+/// </summary>
+/// <param name="spreadData">List of (bid, bidQty, ask, askQty)</param>
+/// <returns>Tuple of (Mean, StdDev). Returns (NaN, NaN) if input is too short.</returns>
+public static (double Mean, double StdDev) EvaluateMeanAndStdDev(List<Tuple<double, double, double, double>> spreadData)
+{
+    if (spreadData == null || spreadData.Count < 2)
+        return (double.NaN, double.NaN);
+
+    var mids = spreadData.Select(x => (x.Item1 + x.Item3) / 2.0).ToList();
+    double mean = mids.Average();
+    double stddev = Math.Sqrt(mids.Select(x => Math.Pow(x - mean, 2)).Average());
+
+    return (mean, stddev);
+}
+
+
+   /// <summary>
+   /// Counts how many times the spread ask dropped below (mean - threshold) [buy signal]
+   /// and how many times the spread bid rose above (mean + threshold) [sell signal].
+   /// Assumes spreadData is a trailing, causal window.
+   /// </summary>
+   /// <returns>(BuyExcursions, SellExcursions)</returns>
+   public static (uint BuyExcursions, double cumm_buy_price, uint SellExcursions, double cumm_sell_price) EvaluateExcursionCount(
+       List<Tuple<double, double, double, double>> spreadData,
+       double entryThresholdMultiplier = 0.5) {
+      if (spreadData == null || spreadData.Count < 2)
+         return (0, Double.NaN, 0, Double.NaN);
+
+      var midPrices = spreadData.Select(x => (x.Item1 + x.Item3) / 2.0).ToList();
+      double mean = midPrices.Average();
+      double stddev = midPrices.StandardDeviation();
+      double buy_threshold = mean - entryThresholdMultiplier * stddev;
+      double sell_threshold = mean + entryThresholdMultiplier * stddev;
+
+      int buyCount = 0;
+      int sellCount = 0;
+      //bool in_buy_zone = false;
+      //bool in_sell_zone = false;
+
+      double cumm_buy_price = 0.0, cumm_sell_price = 0.0;
+      foreach (var (bid, _, ask, _) in spreadData) {
+         if (ask <= buy_threshold) {
+            //if (!in_buy_zone) {
+            cumm_buy_price += ask;
+            buyCount++;
+            //in_buy_zone = true;
+         }
+         //} else {
+         //   in_buy_zone = false;
+         //}
+
+         if (bid >= sell_threshold) {
+            //if (!in_sell_zone) {
+            cumm_sell_price += bid;
+            sellCount++;
+            //in_sell_zone = true;
+         }
+         //} else {
+         //   in_sell_zone = false;
+         //}
+      }
+
+      return ((uint)buyCount, cumm_buy_price / buyCount, (uint)sellCount, cumm_sell_price / sellCount);
+   }
+
+/// <summary>
+/// Counts how many times the spread mid-price crosses its mean level.
+/// Each crossing indicates a potential oscillation turn (cycle midpoint).
+/// Assumes the input is a trailing, causal window of spread ticks.
+/// </summary>
+/// <param name="spreadData">List of (bid, bidQty, ask, askQty)</param>
+/// <returns>Number of zero crossings in the mid-price series</returns>
+public static UInt32 EvaluateZeroCrossingCount(List<Tuple<double, double, double, double>> spreadData)
+{
+    if (spreadData == null || spreadData.Count < 2)
+        return 0;
+
+    var mids = spreadData.Select(x => (x.Item1 + x.Item3) / 2.0).ToList();
+    double mean = mids.Average();
+    int zeroCrossings = 0;
+
+    for (int i = 1; i < mids.Count; i++)
+    {
+        double prevOffset = mids[i - 1] - mean;
+        double currOffset = mids[i] - mean;
+
+        if (prevOffset * currOffset < 0) // Sign changed
+            zeroCrossings++;
+    }
+
+    return (UInt32)zeroCrossings;
+}
+
+   /// <summary>
+   /// Computes the half-life of mean reversion.
+   /// Lower half-life means the spread reverts to its mean more quickly.
+   /// </summary>
+   public static double ComputeHalfLife(List<Tuple<double, double, double, double>> spreadData) {
+      var prices = spreadData.Select(x => (x.Item1 + x.Item3) / 2.0).ToList();
+      var lagged = prices.Take(prices.Count - 1).ToArray();
+      var delta = prices.Skip(1).Zip(lagged, (cur, prev) => cur - prev).ToArray();
+
+      if (lagged.Length < 2) return double.NaN;
+
+      var x = Vector<double>.Build.Dense(lagged);
+      var y = Vector<double>.Build.Dense(delta);
+      var beta = x.DotProduct(y) / x.DotProduct(x);
+
+      return -Math.Log(2) / beta;
+   }
+
+   /// <summary>
+   /// Computes the lag-1 autocorrelation of the spread.
+   /// Lower values indicate quicker mean-reversion.
+   /// </summary>
+   public static double ComputeLag1Autocorrelation(List<Tuple<double, double, double, double>> spreadData) {
+      var prices = spreadData.Select(x => (x.Item1 + x.Item3) / 2.0).ToList();
+      var mean = prices.Average();
+      var numerator = 0.0;
+      var denominator = 0.0;
+
+      for (int i = 1; i < prices.Count; i++) {
+         numerator += (prices[i] - mean) * (prices[i - 1] - mean);
+         denominator += Math.Pow(prices[i - 1] - mean, 2);
+      }
+
+      return denominator == 0 ? 0 : numerator / denominator;
+   }
+
+   /// <summary>
+   /// Estimates how quickly extreme Z-score values decay back toward the mean.
+   /// Shorter average decay time indicates stronger mean-reversion.
+   /// </summary>
+   public static double ComputeAverageZScoreDecay(List<Tuple<double, double, double, double>> spreadData, int zScoreWindow = 50) {
+      var prices = spreadData.Select(x => (x.Item1 + x.Item3) / 2.0).ToList();
+      if (prices.Count < zScoreWindow + 2) return double.NaN;
+
+      var decayTimes = new List<int>();
+      for (int i = zScoreWindow; i < prices.Count - 1; i++) {
+         var window = prices.Skip(i - zScoreWindow).Take(zScoreWindow).ToList();
+         double mean = window.Average();
+         double stdDev = Statistics.StandardDeviation(window);
+         if (stdDev == 0) continue;
+
+         double z = (prices[i] - mean) / stdDev;
+         if (Math.Abs(z) > 2) {
+            for (int j = 1; j < prices.Count - i; j++) {
+               double zNext = (prices[i + j] - mean) / stdDev;
+               if (Math.Abs(zNext) < 1) {
+                  decayTimes.Add(j);
+                  break;
+               }
+            }
+         }
+      }
+
+      return decayTimes.Count == 0 ? double.NaN : decayTimes.Average();
+   }
+
+   /// <summary>
+   /// Computes the variance ratio: short-term variance divided by long-term variance.
+   /// Lower ratio suggests stronger mean-reversion.
+   /// </summary>
+   public static double ComputeVarianceRatio(List<Tuple<double, double, double, double>> spreadData, int shortWindow, int longWindow) {
+      var prices = spreadData.Select(x => (x.Item1 + x.Item3) / 2.0).ToList();
+      if (prices.Count < longWindow) return double.NaN;
+
+      double shortVar = Statistics.Variance(prices.Skip(prices.Count - shortWindow).Take(shortWindow));
+      double longVar = Statistics.Variance(prices.Skip(prices.Count - longWindow).Take(longWindow));
+
+      return longVar == 0 ? double.NaN : shortVar / longVar;
+   }
+
+   /// <summary>
+   /// Computes the highest bid and the lowest ask from a spread tick series.
+   /// Useful for evaluating the best market opportunity range.
+   /// </summary>
+   /// <param name="spreadData">List of (bid, bidQty, ask, askQty)</param>
+   /// <returns>Tuple of (MaxBid, MinAsk)</returns>
+   public static (double MaxBid, double MinAsk) EvaluateMaxBidMinAsk(List<Tuple<double, double, double, double>> spreadData)
+{
+    if (spreadData == null || spreadData.Count == 0)
+        return (double.NaN, double.NaN);
+
+    double maxBid = double.NegativeInfinity;
+    double minAsk = double.PositiveInfinity;
+
+    foreach (var (bid, _, ask, _) in spreadData)
+    {
+        if (bid > maxBid) maxBid = bid;
+        if (ask < minAsk) minAsk = ask;
+    }
+
+    return (maxBid, minAsk);
+}
+
+/// <summary>
+/// Computes the distribution of spread mid-prices around the mean,
+/// grouped by caller-defined distance bands (in standard deviations).
+///
+/// This helps answer:
+/// - Is the spread tightly hugging the mean?
+/// - Are there wide excursions or heavy tails?
+///
+/// You define breakpoints like: [0.5, 1.0, 2.0] → it will return:
+/// - "≤ 0.5σ"
+/// - "0.5σ – 1.0σ"
+/// - "1.0σ – 2.0σ"
+/// - "> 2.0σ"
+///
+/// How to Interpret It:
+/// - ≤ 0.5σ        → Tight clustering near the mean (strong mean-reversion)
+/// - 0.5σ – 1.0σ   → Mild oscillation (common trade region)
+/// - 1.0σ – 2.0σ   → Moderate deviation (potential trade signals)
+/// - > 2.0σ        → Rare, extreme moves (tail events, potential entries/exits)
+///
+/// Use this to tune entry thresholds or detect regime shifts.
+/// </summary>
+/// <param name="spreadData">List of (bid, bidQty, ask, askQty)</param>
+/// <param name="regions">List of positive breakpoints in std deviations (e.g., [0.5, 1.0, 2.0])</param>
+/// <returns>Dictionary mapping label to % of mid-prices in that region</returns>
+public static Dictionary<string, double> EvaluateDistributionAroundMean(
+    List<Tuple<double, double, double, double>> spreadData,
+    List<double> regions)
+{
+    var result = new Dictionary<string, double>();
+
+    if (spreadData == null || spreadData.Count < 2 || regions == null || regions.Count == 0)
+        return result;
+
+    var mids = spreadData.Select(x => (x.Item1 + x.Item3) / 2.0).ToList();
+    double mean = mids.Average();
+    double stddev = Math.Sqrt(mids.Select(x => Math.Pow(x - mean, 2)).Average());
+
+    if (stddev == 0)
+    {
+        result["Zero Volatility"] = 100.0;
+        return result;
+    }
+
+    var buckets = new int[regions.Count + 1];
+    foreach (var mid in mids)
+    {
+        double distance = Math.Abs(mid - mean);
+        bool placed = false;
+
+        for (int i = 0; i < regions.Count; i++)
+        {
+            if (distance <= regions[i])
+            {
+                buckets[i]++;
+                placed = true;
+                break;
+            }
+        }
+
+        if (!placed)
+            buckets[regions.Count]++;
+    }
+
+    int total = mids.Count;
+    for (int i = 0; i < regions.Count; i++)
+    {
+        string label = i == 0
+            ? $"≤ {regions[i]:0.##}σ"
+            : $"{regions[i - 1]:0.##}σ – {regions[i]:0.##}σ";
+        result[label] = 100.0 * buckets[i] / total;
+    }
+
+    result[$"> {regions.Last():0.##}σ"] = 100.0 * buckets.Last() / total;
+
+    return result;
+}
+
+/// <summary>
+/// Computes two drift metrics indicating how much the spread's mean changes over time,
+/// based on the standard deviation of rolling mean values.
+///
+/// ✅ Absolute Drift: Standard deviation of the rolling mean
+/// ✅ Relative Drift: Drift normalized by the overall mean (unitless ratio)
+///
+/// This shows whether the center of the spread is stable — critical for mean-reversion.
+///
+/// How to Interpret the Output:
+/// - ✅ Relative Drift < 1% → Very stable mean — ideal
+/// - ⚠️ 1–5% → Mild drift — consider shorter holds or adaptive mean
+/// - ❌ >5% → Drifting mean — fixed mean may no longer work
+/// </summary>
+/// <param name="spreadData">List of (bid, bidQty, ask, askQty)</param>
+/// <param name="window">Window size for rolling mean (e.g., 20 ticks)</param>
+/// <returns>
+/// Tuple of (Absolute Drift, Relative Drift) — where:
+/// - Absolute Drift = StdDev of rolling means
+/// - Relative Drift = Drift / global mean
+/// </returns>
+public static (double AbsoluteDrift, double RelativeDrift) EvaluateMeanDriftScore(
+    List<Tuple<double, double, double, double>> spreadData,
+    int window = 20)
+{
+    if (spreadData == null || spreadData.Count < window)
+        return (double.NaN, double.NaN);
+
+    var mids = spreadData.Select(x => (x.Item1 + x.Item3) / 2.0).ToArray();
+    var rollingMeans = new List<double>();
+
+    for (int i = window; i <= mids.Length; i++)
+    {
+        double rollingMean = mids.Skip(i - window).Take(window).Average();
+        rollingMeans.Add(rollingMean);
+    }
+
+
+    double overallMean = mids.Average(); // use full mean for normalization
+    //double drift = Math.Sqrt(rollingMeans.Select(m => Math.Pow(m - rollingMeans.Average(), 2)).Average());
+    double drift = Math.Sqrt(rollingMeans.Select(m => Math.Pow(m - rollingMeans.Average(), 2)).Average());
+
+
+double relativeDrift = Math.Abs(overallMean) > 1e-8 ? drift / Math.Abs(overallMean) : double.NaN;
+    return (drift, relativeDrift);
+}
+
+/// <summary>
+/// Returns the latest Kalman Filter estimate of the true mean of the spread mid-prices.
+/// This tracks the spread's central value while allowing for gradual drift and suppressing noise.
+///
+/// Internally, this applies a recursive 1D Kalman filter over the input mid-prices.
+/// 
+/// Tuning Parameters:
+/// - Q: Process noise variance (higher = faster adaptation to drift)
+/// - R: Measurement noise variance (higher = more smoothing)
+/// 
+/// Use this instead of SMA if your mean drifts or if price series is noisy.
+///
+/// </summary>
+/// <param name="spreadData">List of (bid, bidQty, ask, askQty)</param>
+/// <param name="Q">Process noise variance (default = 0.0001)</param>
+/// <param name="R">Measurement noise variance (default = 0.01)</param>
+/// <returns>Latest estimated Kalman-filtered mean, or NaN if input is invalid</returns>
+public static double GetLatestKalmanMean(
+    List<Tuple<double, double, double, double>> spreadData,
+    double Q = 0.0001,
+    double R = 0.01)
+{
+    if (spreadData == null || spreadData.Count == 0)
+        return double.NaN;
+
+    var mids = spreadData.Select(x => (x.Item1 + x.Item3) / 2.0).ToList();
+
+    double x = mids[0];  // initial estimate of mean
+    double P = 1.0;       // initial uncertainty
+
+    for (int i = 1; i < mids.Count; i++)
+    {
+        // Predict step
+        double x_prior = x;
+        double P_prior = P + Q;
+
+        // Update step
+        double K = P_prior / (P_prior + R);
+        x = x_prior + K * (mids[i] - x_prior);
+        P = (1 - K) * P_prior;
+    }
+
+    return x; // the latest estimated mean
+}
+
+
+
+
+}
@@ -0,0 +1,137 @@
+namespace bts.SilverPine {
+   using BCG;
+
+   using System.Diagnostics;
+
+   public class SpreadPerformanceStream : ViewStream {
+      ViewStream _spread_price_stream = null;
+      private readonly IBlueTableSchema _schema;
+      private readonly int _window_secs;
+      private readonly double _buy_threshold, _sell_threshold, _flat_threshold;
+      private ViewStream _bbo_stream;
+      public SpreadPerformanceStream(string s1, string s2, double w1, double w2, int window_secs,
+                                       double buy_threshold, double sell_threshold, double flat_threshold)
+      : this(s1, s2, w1, w2, window_secs, buy_threshold, sell_threshold, flat_threshold, ViewStreamDatabase.instance) { }
+
+      public SpreadPerformanceStream(string s1, string s2, double w1, double w2, int window_secs,
+                                      double buy_threshold, double sell_threshold, double flat_threshold,
+                                      ViewStreamDatabase vsd) {
+         _bbo_stream = new BBOStream(vsd);
+         _schema = BlueDataUtility.form_schema("SpreadPerformance", Tuple.Create("id", TypeCode.Int32),                                                
+                                                Tuple.Create("t0", TypeCode.DateTime),
+                                                Tuple.Create("t1", TypeCode.DateTime),
+                                                Tuple.Create("n_ticks", TypeCode.UInt32),
+                                                Tuple.Create("curr_windows_seconds", TypeCode.Double),
+                                                // means / std dev                                          
+                                                Tuple.Create("stddev", TypeCode.Double),
+                                                Tuple.Create("mean", TypeCode.Double),
+                                                Tuple.Create("kalman_filter_mean", TypeCode.Double),
+                                                Tuple.Create("absolute_mean_drift", TypeCode.Double),
+                                                Tuple.Create("relative_mean_drift", TypeCode.Double),
+                                                // statistical measures
+                                                Tuple.Create("hurst_exponent", TypeCode.Double),
+                                                Tuple.Create("half_life", TypeCode.Double),
+                                                Tuple.Create("lag1_autocorrelation", TypeCode.Double),
+                                                Tuple.Create("average_zscore_decay", TypeCode.Double),
+                                                Tuple.Create("variance_ratio", TypeCode.Double),
+                                                Tuple.Create("n_zero_crossings", TypeCode.UInt32),                                                
+                                                // distributions
+                                                Tuple.Create("Z ≤ 0.5σ", TypeCode.Double),
+                                                Tuple.Create("0.5σ – 1.0σ", TypeCode.Double),
+                                                Tuple.Create("1.0σ – 2.0σ", TypeCode.Double),
+                                                Tuple.Create("> 2.0σ", TypeCode.Double),
+                                                ////////////////////////////////////////                                                   
+                                                Tuple.Create("max_bid", TypeCode.Double),
+                                                Tuple.Create("min_ask", TypeCode.Double),
+                                                Tuple.Create("n_buy_signals", TypeCode.UInt32),
+                                                Tuple.Create("cumm_buy_price", TypeCode.Double),
+                                                Tuple.Create("n_sell_signals", TypeCode.UInt32),
+                                                Tuple.Create("cumm_sell_price", TypeCode.Double)
+                                                //Tuple.Create("pnl", TypeCode.Double),
+                                                //Tuple.Create("buy_turns", TypeCode.UInt32),
+                                                //Tuple.Create("sell_turns", TypeCode.UInt32),
+                                                //Tuple.Create("trades", TypeCode.String)
+                                                );
+         _window_secs = window_secs;
+         _buy_threshold = buy_threshold;
+         _sell_threshold = sell_threshold;
+         _flat_threshold = flat_threshold;
+         _spread_price_stream = new SyntheticSpreadStream(s1, s2, w1, w2, _bbo_stream, vsd);
+         _spread_price_stream = new TimeSeriesStream(_spread_price_stream);
+         _spread_price_stream = new ThrottleStream(_spread_price_stream, TimeSpan.FromSeconds(5.0), vsd);
+      }
+
+      protected override void myOnFirstSubscriber() {
+         _spread_price_stream.onChange += _handle_spread_prices;
+      }
+
+      protected override void myOnLastSubscriber() {
+         _spread_price_stream.onChange -= _handle_spread_prices;
+      }
+
+      private void _handle_spread_prices(object dp_obj) {
+         DataPacket dp = dp_obj as DataPacket;
+         List<Tuple<DateTime, double, double, double, double>> spread_ts_prices = new List<Tuple<DateTime, double, double, double, double>>();
+         DateTime min_time = DateTime.MaxValue; DateTime max_time = DateTime.MinValue;
+         var sw = Stopwatch.StartNew();
+         sw.Start();
+         foreach (IBlueRow r in dp.newRows) {
+            DateTime ts = r.get_datetime("timestamp");
+            TimeSpan elapsed_time = DateTime.Now - ts;
+            if (elapsed_time.TotalSeconds >= _window_secs)
+               continue;
+            spread_ts_prices.Add(Tuple.Create(
+                                          r.get_datetime("timestamp"),
+                                          r.get<double>("spread_bid_price"),
+                                          r.get<double>("spread_bid_sz"),
+                                          r.get<double>("spread_ask_price"),
+                                          r.get<double>("spread_ask_sz")));
+            if (ts <= min_time) {
+               min_time = ts;
+            }
+            if (ts >= max_time) {
+               max_time = ts;
+            }
+         }
+         List<Tuple<double, double, double, double>> spread_prices = spread_ts_prices
+                                                                                             .OrderBy(t => t.Item1)
+                                                                                             .Select(t => Tuple.Create(t.Item2, t.Item3, t.Item4, t.Item5))
+                                                                                             .ToList();
+
+         uint n_ticks = (uint)spread_prices.Count;
+         double curr_windows_seconds = (max_time - min_time).TotalSeconds;
+         double hurst_exponent = SpreadPerformanceEvaluator.EvaluateHurstExponent(spread_prices);
+         double half_life = SpreadPerformanceEvaluator.ComputeHalfLife (spread_prices);
+         double lag1_autocorrelation = SpreadPerformanceEvaluator.ComputeLag1Autocorrelation(spread_prices);
+         double average_zscore_decay = SpreadPerformanceEvaluator.ComputeAverageZScoreDecay(spread_prices);
+         double variance_ratio = SpreadPerformanceEvaluator.ComputeVarianceRatio(spread_prices, 60, 360);
+
+         (double mean, double stddev) = SpreadPerformanceEvaluator.EvaluateMeanAndStdDev(spread_prices);
+         double kalman_filter_mean = SpreadPerformanceEvaluator.GetLatestKalmanMean(spread_prices);
+         uint n_zero_crossings = SpreadPerformanceEvaluator.EvaluateZeroCrossingCount(spread_prices);
+         (double max_bid, double min_ask) = SpreadPerformanceEvaluator.EvaluateMaxBidMinAsk(spread_prices);
+         (uint n_buy_signals, double cumm_buy_price, uint n_sell_signals, double cumm_sell_price) = SpreadPerformanceEvaluator.EvaluateExcursionCount(spread_prices, _buy_threshold);
+         //(double pnl, List<SpreadPerformanceEvaluator.TradeAction> trades, int buy_turns, int sell_turns) = SpreadPerformanceEvaluator.EvaluateSimulatedPnL(spread_prices, 30, _buy_threshold, _flat_threshold);
+         //string trade_str = String.Empty;
+         //foreach (var tr in trades) {
+         //   trade_str += tr.ToString() + " , ";
+         //}
+         Dictionary<string, double> dist_map = SpreadPerformanceEvaluator.EvaluateDistributionAroundMean(spread_prices,
+                                                                       new List<double>(new double[] { 0.5, 1.0, 2.0 }));
+         var dist_vals = dist_map.Count < 4 ? new double[] { Double.NaN, Double.NaN, Double.NaN, Double.NaN } : dist_map.Values.ToArray();
+         (double absolute_drift, double relative_drift) = SpreadPerformanceEvaluator.EvaluateMeanDriftScore(spread_prices);
+         IBlueRow nr = new BlueRow(_schema, 0, min_time, max_time, n_ticks, curr_windows_seconds, stddev, mean, kalman_filter_mean, absolute_drift, relative_drift,
+            hurst_exponent, half_life, lag1_autocorrelation, average_zscore_decay, variance_ratio, n_zero_crossings,
+                                    dist_vals[0], dist_vals[1], dist_vals[2], dist_vals[3],
+                                    max_bid, min_ask, n_buy_signals, cumm_buy_price, n_sell_signals, cumm_sell_price);
+         BlueTable t = new BlueTable(_schema);
+         t.addRow(nr);
+         var dp_out = BlueDataUtility.build_dp_inc(t, myLastValue as DataPacket, true);
+         var perf_msecs = sw.ElapsedMilliseconds;
+         sw.Stop();
+         Log.Gen.info($"SpreadPerformanceStream._handle_spread_prices comp_time: {perf_msecs} msecs");
+         myFireEvent(dp_out);
+
+      }
+   }
+}
@@ -0,0 +1,68 @@
+namespace bts.SilverPine{
+   using BCG;
+   public class SpreadStatArbsWeightLinearRegressionStream : ViewStream {
+      private IBlueTableSchema _schema;
+      private ViewStream _leg_price_stream;
+      private readonly double _window_secs;
+      private readonly string _s1, _s2;
+
+      public SpreadStatArbsWeightLinearRegressionStream(string s1, string s2, double window_secs, double sample_period_secs)
+         : this(s1, s2, window_secs, sample_period_secs, ViewStreamDatabase.instance) { }
+
+
+      public SpreadStatArbsWeightLinearRegressionStream(string s1, string s2, double window_secs, double sample_period_secs, ViewStreamDatabase vsd) {
+         _window_secs = window_secs;
+         _s1 = s1;
+         _s2 = s2;
+         _schema = BlueDataUtility.form_schema("LinearRegressionWeights",
+                                                Tuple.Create("id", TypeCode.Int32),
+                                                Tuple.Create("t0", TypeCode.DateTime),
+                                                Tuple.Create("t1", TypeCode.DateTime),
+                                                Tuple.Create("curr_windows_seconds", TypeCode.Double),
+                                                Tuple.Create("n_ticks", TypeCode.UInt32),
+                                                Tuple.Create("s1", TypeCode.String),
+                                                Tuple.Create("w1", TypeCode.Double),
+                                                Tuple.Create("s2", TypeCode.String),
+                                                Tuple.Create("w2", TypeCode.String));
+
+         _leg_price_stream = new TimeSeriesStream(new SyntheticSpreadLegsStream(s1, s2));
+         _leg_price_stream = new ThrottleStream(_leg_price_stream, TimeSpan.FromSeconds(1), vsd);
+      }
+
+      protected override void myOnFirstSubscriber() {
+         _leg_price_stream.onChange += _handle_leg_prices;
+      }
+
+      protected override void myOnLastSubscriber() {
+         _leg_price_stream.onChange -= _handle_leg_prices;
+      }
+
+      private void _handle_leg_prices(object dp_obj){
+         DataPacket dp = dp_obj as DataPacket;
+         List<Tuple<double, double, DateTime>> leg_prices = new List<Tuple<double, double, DateTime>>();
+         foreach (IBlueRow r in dp.newRows){
+            double s1_mid = (r.get<double>("s1_bid_price") + r.get<double>("s1_ask_price")) /2.0;
+            double s2_mid = (r.get<double>("s2_bid_price") + r.get<double>("s2_ask_price")) /2.0;
+            DateTime ts = r.get_datetime("timestamp");
+            leg_prices.Add(new Tuple<double, double, DateTime>(s1_mid, s2_mid, ts));
+         }
+         if (leg_prices.Count <= 2)
+            return;
+         leg_prices = leg_prices.Where(t => (DateTime.Now - t.Item3).TotalSeconds <= _window_secs).ToList();
+         DateTime t0 = leg_prices.Select(t => t.Item3).Min();
+         DateTime t1 = leg_prices.Select(t => t.Item3).Max();
+         double curr_windows_seconds = (t1 - t0).TotalSeconds;
+         int n_ticks = leg_prices.Count;
+         List<double> s1_mids = leg_prices.Select(t => t.Item1).ToList();
+         List<double> s2_mids = leg_prices.Select(t => t.Item2).ToList();
+         (double w1, double w2) = SpreadWeightCalculator.CalculateWeightsLinearRegression(s1_mids, s2_mids);
+         BlueTable state = new BlueTable(_schema);
+         IBlueRow nr = new BlueRow(_schema, 0, t0, t1, curr_windows_seconds, n_ticks,_s1, w1, _s2, w2);
+         state.addRow(nr);
+         var dp_out = BlueDataUtility.build_dp_inc(state, myLastValue as DataPacket, true);
+         myFireEvent(dp_out);
+
+      }
+   }
+}
+
@@ -0,0 +1,143 @@
+namespace bts.SilverPine {
+   using BCG;
+
+   using MathNet.Numerics.LinearAlgebra.Factorization;
+
+   using System;
+   using System.Collections.Generic;
+   using System.Diagnostics;
+   using System.Linq;
+   using System.Text;
+   using System.Threading.Tasks;
+   [DataStreamUserRespresentation("SpreadStatsStream")]
+   // This is the fundamental stream that computes the stats (including buy and sell zones) 
+   // of a spread. the legs and the weights are passed as args
+   public class SpreadStatsStream : ViewStream {
+      private ViewStream _spread_price_stream;
+      private readonly IBlueTableSchema _schema;
+      private readonly TimeSpan _window_sz;
+      private readonly string _spread;
+      public SpreadStatsStream(string s1, string s2, double w1, double w2, int rolling_window_sz_seconds, int window_sampling_sz_secs)
+         : this(s1, s2, w1, w2, rolling_window_sz_seconds, window_sampling_sz_secs, new BBOStream(ViewStreamDatabase.instance), ViewStreamDatabase.instance) { }
+      public SpreadStatsStream(string s1, string s2, double w1, double w2, int rolling_window_sz_seconds, int window_sampling_sz_secs, ViewStream bbo_stream, ViewStreamDatabase vsd) {
+         _window_sz = TimeSpan.FromSeconds(rolling_window_sz_seconds);
+         _spread = $"{w1} X {s1} - {w2} X {s2}";
+         _schema = new BlueTableSchema("SpreaddStats", new ColumnInfo ("id", 0, TypeCode.Int32));
+         _schema = _schema.add_column(TypeCode.String, "spread");
+         _schema = _schema.add_column(TypeCode.String, "window");
+         // basic stats
+         _schema = _schema.add_column(TypeCode.Double, "bid_mean");
+         _schema = _schema.add_column(TypeCode.Double, "ask_mean");
+         _schema = _schema.add_column(TypeCode.Double, "bid_stddev");
+         _schema = _schema.add_column(TypeCode.Double, "ask_stddev");
+         _schema = _schema.add_column(TypeCode.Double, "bid_skewness");
+         _schema = _schema.add_column(TypeCode.Double, "ask_skewness");
+         _schema = _schema.add_column(TypeCode.Double, "bid_kurtosis");
+         _schema = _schema.add_column(TypeCode.Double, "ask_kurtosis");
+         _schema = _schema.add_column(TypeCode.Double, "bid_max");
+         _schema = _schema.add_column(TypeCode.Double, "ask_max");
+         _schema = _schema.add_column(TypeCode.Double, "bid_min");
+         _schema = _schema.add_column(TypeCode.Double, "ask_min");
+         // bid bins
+         _schema = _schema.add_column(TypeCode.Double, "bid Z < -2σ");
+         _schema = _schema.add_column(TypeCode.Double, "bid -2σ ≤ Z < -1σ");
+         _schema = _schema.add_column(TypeCode.Double, "bid -1σ ≤ Z < 0σ");
+         // positive z's
+         _schema = _schema.add_column(TypeCode.Double, "bid 0σ ≤ Z < 1σ");
+         _schema = _schema.add_column(TypeCode.Double, "bid 1σ ≤ Z < 2σ");
+         _schema = _schema.add_column(TypeCode.Double, "bid Z ≥ 2σ");
+         // ask bins
+         _schema = _schema.add_column(TypeCode.Double, "ask Z < -2σ");
+         _schema = _schema.add_column(TypeCode.Double, "ask -2σ ≤ Z < -1σ");
+         _schema = _schema.add_column(TypeCode.Double, "ask -1σ ≤ Z < 0σ");
+         // positive z's
+         _schema = _schema.add_column(TypeCode.Double, "ask 0σ ≤ Z < 1σ");
+         _schema = _schema.add_column(TypeCode.Double, "ask 1σ ≤ Z < 2σ");
+         _schema = _schema.add_column(TypeCode.Double, "ask Z ≥ 2σ");
+         // opps
+         _schema = _schema.add_column(TypeCode.Int32, "n_buy_ticks");
+         _schema = _schema.add_column(TypeCode.Double, "cumm_buy_price");
+         _schema = _schema.add_column(TypeCode.Int32, "n_sell_ticks");
+         _schema = _schema.add_column(TypeCode.Double, "cumm_sell_price");
+         _schema = _schema.add_column(TypeCode.Int32, "n_trade_ticks");
+         _spread_price_stream = new SyntheticSpreadStream(s1, s2, w1, w2, bbo_stream, vsd);
+         _spread_price_stream = new TimeSeriesStream(_spread_price_stream);
+         double window_sampling_sz_smsecs = window_sampling_sz_secs * 1000.0 + new Random(DateTime.Now.Second).Next(-100, 100);
+         TimeSpan sampling_period = TimeSpan.FromMilliseconds(window_sampling_sz_smsecs);
+         _spread_price_stream = new ThrottleStream(_spread_price_stream, sampling_period, vsd);
+      }
+
+      protected override void myOnFirstSubscriber() {
+         _spread_price_stream.onChange += _handle_spread_prices;
+      }
+
+      protected override void myOnLastSubscriber() {
+         _spread_price_stream.onChange -= _handle_spread_prices;
+      }
+
+      private void _handle_spread_prices(object dp_obj) {
+         List<IBlueRow> rows = (dp_obj as DataPacket).newRows.to_list();
+         Func<IBlueRow, bool> row_filter = (r) => {
+            DateTime ts = r.get_datetime("timestamp");
+            DateTime start_time = DateTime.Now.Subtract(_window_sz);
+            return ts >= start_time;
+         };
+         var rolling_win_rows = rows.Where(r => row_filter(r));
+         var spread_prices = rolling_win_rows.Select(r => (
+                                          r.get_datetime("timestamp"),
+                                          r.get<double>("spread_bid_price"),
+                                          r.get<double>("spread_bid_sz"),
+                                          r.get<double>("spread_ask_price"),
+                                          r.get<double>("spread_ask_sz"))).ToList();
+         spread_prices.Sort((a,b) => a.Item1.CompareTo(b.Item1));
+         DateTime t0 = spread_prices.First().Item1, t1 = spread_prices.Last().Item1;
+         var tot_secs = (t1 - t0).TotalSeconds;
+         var stopwatch = Stopwatch.StartNew();
+         Log.Gen.info($"SpreadStatsStream.analyze_spread_quotes executed in {stopwatch.ElapsedMilliseconds} ms");
+         var result = SpreadPerformanceEvaluator.SpreadStatistics.analyze_spread_quotes(spread_prices);
+         (uint n_buy_signals, double cumm_buy_price, uint n_sell_signals, double cumm_sell_price) = 
+              SpreadPerformanceEvaluator.EvaluateExcursionCount(spread_prices.Select(t => Tuple.Create(t.Item2, t.Item3, t.Item4, t.Item5)).ToList(), 2.0);
+         IBlueRow nr = new BlueRow(_schema,
+            0,
+            _spread,
+            $"n_ticks: {spread_prices.Count} t0: {t0.ToLongTimeString()} -> t1: {t1.ToLongTimeString()} ({tot_secs}) secs",
+            result.bid_stats.Mean,
+            result.ask_stats.Mean,
+            result.bid_stats.StandardDeviation,
+            result.ask_stats.StandardDeviation,
+            result.bid_stats.Skewness,
+            result.ask_stats.Skewness,
+            result.bid_stats.Kurtosis,
+            result.ask_stats.Kurtosis,
+            result.bid_stats.Maximum,
+            result.ask_stats.Maximum,
+            result.bid_stats.Minimum,
+            result.ask_stats.Minimum,
+            // bid distribution
+            result.bid_zscore_distribution["Z < -2σ"],
+            result.bid_zscore_distribution["-2σ ≤ Z < -1σ"],
+            result.bid_zscore_distribution["-1σ ≤ Z < 0σ"],
+            result.bid_zscore_distribution["0σ ≤ Z < 1σ"],
+            result.bid_zscore_distribution["1σ ≤ Z < 2σ"],
+            result.bid_zscore_distribution["Z ≥ 2σ"],
+            // ask distributions
+            result.ask_zscore_distribution["Z < -2σ"],
+            result.ask_zscore_distribution["-2σ ≤ Z < -1σ"],
+            result.ask_zscore_distribution["-1σ ≤ Z < 0σ"],
+            result.ask_zscore_distribution["0σ ≤ Z < 1σ"],
+            result.ask_zscore_distribution["1σ ≤ Z < 2σ"],
+            result.ask_zscore_distribution["Z ≥ 2σ"],
+            n_buy_signals,
+            cumm_buy_price,
+            n_sell_signals,
+            cumm_sell_price,
+            n_buy_signals + n_sell_signals
+            );
+         stopwatch.Stop();
+         BlueTable t = new BlueTable(_schema);
+         t.addRow(nr);
+         var dp_out = BlueDataUtility.build_dp(t, myLastValue as DataPacket);
+         myFireEvent(dp_out);
+      }
+   }
+}
@@ -0,0 +1,49 @@
+using System;
+using System.Collections.Generic;
+using MathNet.Numerics.LinearRegression;
+
+namespace bts.SilverPine {
+
+   public static class SpreadWeightCalculator {
+      /// <summary>
+      /// Estimates hedge weights (w1, w2) such that the spread s = w1 * s1 - w2 * s2 
+      /// is potentially mean-reverting. Uses linear regression: s1 = beta * s2 + ε.
+      /// 
+      /// Assumes mid-prices are precomputed and aligned over the same time window.
+      /// </summary>
+      /// <param name="s1MidPrices">List of mid-prices for asset 1 (dependent variable)</param>
+      /// <param name="s2MidPrices">List of mid-prices for asset 2 (independent variable)</param>
+      /// <param name="includeIntercept">
+      /// Whether to include an intercept in the regression model (default: true).
+      /// </param>
+      /// <returns>(w1, w2): Tuple of hedge weights</returns>
+      public static (double w1, double w2) CalculateWeightsLinearRegression(
+          List<double> s1MidPrices,
+          List<double> s2MidPrices,
+          bool includeIntercept = true) {
+         if (s1MidPrices == null || s2MidPrices == null)
+            throw new ArgumentNullException("Price series cannot be null.");
+         if (s1MidPrices.Count != s2MidPrices.Count)
+            throw new ArgumentException("Price series must be of equal length.");
+         if (s1MidPrices.Count < 2)
+            throw new ArgumentException("Need at least two data points for regression.");
+
+         double[] x = s2MidPrices.ToArray(); // Independent variable
+         double[] y = s1MidPrices.ToArray(); // Dependent variable
+
+         double beta;
+
+         if (includeIntercept) {
+            // Returns (intercept, slope)
+            (double intercept, double slope) = SimpleRegression.Fit(x, y);
+            beta = slope;
+         } else {
+            // Returns only slope when forcing regression through origin
+            beta = SimpleRegression.FitThroughOrigin(x, y);
+         }
+
+         // Return fixed w1 = 1.0 and w2 = beta
+         return (1.0, beta);
+      }
+   }
+}
@@ -111,6 +111,7 @@ namespace bts.SilverPine {
                                                Tuple.Create("id", TypeCode.Int32),
                                                Tuple.Create("sec_key", TypeCode.UInt32),
                                                Tuple.Create("sec", TypeCode.String),
+                                                Tuple.Create("exchange", TypeCode.String),
                                                Tuple.Create("bid_price", TypeCode.Double),
                                                Tuple.Create("bid_sz", TypeCode.Single),
                                                Tuple.Create("ask_price", TypeCode.Double),
@@ -122,14 +123,16 @@ namespace bts.SilverPine {
      }

      public BBOStream(ViewStreamDatabase vsd) {
-         // bts.Log.Gen.info("BBOStream.cons");
-         // _vsd = vsd;
-         // _md_client = new MarketDataClient(new MDSource(TimeSpan.FromSeconds(0.5)));
-         FakeBBOStream rs = new FakeBBOStream("Future", "CME", "ES", 5000, 0.25, vsd);
-         mySetRealSource(rs);
+         bts.Log.Gen.info("BBOStream.cons");
+         _vsd = vsd;
+         _md_client = new MarketDataClient(new MDSource(TimeSpan.FromSeconds(0.5)));
+         //FakeBBOStream rs = new FakeBBOStream("Future", "CME", "ES", 5000, 0.25, vsd);
+         //mySetRealSource(rs);
      }

      protected override void myOnFirstSubscriber() {
+         _vsd.addWorker(this);
+         _md_client.books().Subscribe(_on_books);
         _vsd?.addWorker(this);
         _md_client?.books().Subscribe(_on_books);
      }
@@ -141,14 +144,14 @@ namespace bts.SilverPine {
         }
         foreach (Change<BookTuple> c in update.changes) {
            var bt = c.New;
-            bts.Log.Gen.info($"BBOStream changes: idx: {i} bid: {bt.book.bids[0].price} / {bt.book.bids[0].size} ask: {bt.book.asks[0].price} / {bt.book.asks[0].size}");
+            //bts.Log.Gen.info($"BBOStream changes: idx: {i} bid: {bt.book.bids[0].price} / {bt.book.bids[0].size} ask: {bt.book.asks[0].price} / {bt.book.asks[0].size}");
         }
      }

      private void _on_books(TableUpdate<BookTuple, uint> update) {
         _log_updates(update);
         // just cacching in here, will populate the rows in _poll_scan and fire the datapacket
-         bts.Log.Gen.info($"BBOStream._on_books state: {update.state.Count()} changes: {update.changes.Count()} added: {update.added.Count()}");
+         //bts.Log.Gen.info($"BBOStream._on_books state: {update.state.Count()} changes: {update.changes.Count()} added: {update.added.Count()}");
         _cached_state = update.state;
      }

@@ -158,7 +161,7 @@ namespace bts.SilverPine {

      private void _scan_bbo() {
         // this shouldn be made incremental using the TableUpdate
-         bts.Log.Gen.info("BBOStream._scan_bbo");
+         //bts.Log.Gen.info("BBOStream._scan_bbo");
         if (_cached_state == null)
            return;
         BlueTable new_state = new BlueTable(_bbo_schema);
@@ -167,7 +170,8 @@ namespace bts.SilverPine {
            var sec = kv.Value.security;
            int id = Convert.ToInt32(skey);
            BookTuple bt = kv.Value;
-            IBlueRow r = new BlueRow(_bbo_schema, id, skey, sec.ToString(), bt.book.bids[0].price, bt.book.bids[0].size,
+            var venue = sec.Exchange;
+            IBlueRow r = new BlueRow(_bbo_schema, id, skey, sec.ToString(), venue.ToString(), bt.book.bids[0].price, bt.book.bids[0].size,
                                     bt.book.asks[0].price, bt.book.asks[0].size);
            new_state.addRow(r);
         }
@@ -1,8 +1,9 @@
-namespace BCG {
+namespace bts.SilverPine {
   using BCG.Utilities.General;

   using bts.api;
   using bts.md;
+   using BCG;

   using Spark.Core.Reactive;

@@ -31,6 +32,7 @@ namespace BCG {
                                                Tuple.Create("sec_type", TypeCode.String),
                                                Tuple.Create("venue", TypeCode.String),
                                                Tuple.Create("symbol", TypeCode.String),
+                                                Tuple.Create("currency", TypeCode.String),
                                                Tuple.Create("expiration", TypeCode.DateTime),
                                                Tuple.Create("strike", TypeCode.Double),
                                                Tuple.Create("callput", TypeCode.String)
@@ -81,7 +83,7 @@ namespace BCG {
         foreach (var kv in _cached_state) {
            SecurityKey skey = kv.Key;
            int id = Convert.ToInt32(skey);
-            SecurityImmutable sec = kv.Value;
+            SecurityImmutable sec = kv.Value;            
            bts.api.SecurityType sec_type = sec.security_type();
            object exp = DBNull.Value;
            if (sec_type == bts.api.SecurityType.Future || sec_type == bts.api.SecurityType.Option)
@@ -92,7 +94,7 @@ namespace BCG {
            object cp = DBNull.Value;
            if (sec_type == bts.api.SecurityType.Option)
               cp = sec.Option.CallPut.ToString();
-            IBlueRow r = new BlueRow(_instrument_schema, id, skey, sec.security_desc(), sec.security_type().ToString(), sec.display_exchange().ToString(), sec.Symbol, exp, strike, cp);
+            IBlueRow r = new BlueRow(_instrument_schema, id, skey, sec.security_desc(), sec.security_type().ToString(), sec.display_exchange().ToString(), sec.Symbol, sec.Currency, exp, strike, cp);
            new_state.addRow(r);
         }
         DataPacket dp = null;
@@ -117,4 +119,4 @@ namespace BCG {
         }
      }
   }
-}
+}
@@ -174,10 +174,10 @@ namespace bts.SilverPine {
               // TODO: assuming 1 day only for historic               
               string table_name = get_table_from_barsize(ohlcv_params.bar_size, symbol);// $"{symbol}_OHLCV_1DAY";
                                                                                         //string table_name = $"{symbol}_{ohlcv_params.bar_size}";
-               string db_str = $"Data Source={_location}{_db_name};Version=3";
+               string db_name = _db_name;// $"Data Source={_location}{_db_name};Version=3";
               string sql_str = $"select * from {table_name}";
               // "Data Source=C:\\Users\\m_tah\\development\\data\\stream_data.db;Version=3;"
-               ViewStream rs = new SQLiteStream(db_str, sql_str);
+               ViewStream rs = new SQLiteStream(db_name, sql_str);
               rs = new SchemaGuesserStream(rs);
               if (ohlcv_params.bar_size == OHLCVBarSize.ONE_DAY)
                  rs = new DateStandardizationStream(rs, "ts_event", DateConversion.ChopTime);
@@ -207,4 +207,4 @@ namespace bts.SilverPine {


   }
-}
+}
@@ -0,0 +1,125 @@
+using System;
+using BCG;
+
+namespace bts.SilverPine
+{
+/*
+   ----------------------------------------------------------------------
+                               *Expression*
+   ----------------------------------------------------------------------
+       s1 = filter(bbo(), "sec = 'ETH-USDT-SWAP'")
+       s1 = addColumns(s1, "ljc:0")
+       s2 = filter(bbo(), "sec = 'ETH-USDT-SWAP'")
+       s2 = addColumns(s2, "rjc:0")
+       
+       s = fastJoin(s1, "ljc", "sec:s1 bid_price:s1_bid_price bid_sz:s1_bid_sz ask_price:s1_ask_price ask_sz:s1_ask_sz", 
+                    s2, "rjc", "sec:s2 bid_price:s2_bid_price bid_sz:s2_bid_sz ask_price:s2_ask_price ask_sz:s2_ask_sz")
+       
+
+   ----------------------------------------------------------------------
+                                 *Schema*
+   ----------------------------------------------------------------------
+       id                              System.Int32
+       s1                              System.String
+       s1_bid_price                    System.Double
+       s1_bid_sz                       System.Single
+       s1_ask_price                    System.Double
+       s1_ask_sz                       System.Single
+       s2                              System.String
+       s2_bid_price                    System.Double
+       s2_bid_sz                       System.Single
+       s2_ask_price                    System.Double
+       s2_ask_sz                       System.Single
+*/
+
+
+   // Combines the market data of spread legs into one stream, lined up nicely in a row.
+   // this is useful for operations like finding weights, that need leg market data organized together succintly.
+   public class SyntheticSpreadLegsStream : ViewStream 
+   {
+      public SyntheticSpreadLegsStream(string s1, string s2) : 
+         this(s1, s2, new BBOStream(ViewStreamDatabase.instance), ViewStreamDatabase.instance) {}
+
+      public SyntheticSpreadLegsStream(string s1, string s2, ViewStream bbo_stream, ViewStreamDatabase vsb)
+      {
+         ViewStream vs1;
+         ViewStream vs2;
+         ViewStream rs;
+
+         vs1 = new FilterStream (new BBOStream (),
+                                $"sec = '{s1}'");
+         vs1 = new AddColumnsStream (vs1,
+                                    "ljc:0");
+         vs2 = new FilterStream (bbo_stream,
+                                $"sec = '{s2}'");
+         vs2 = new AddColumnsStream (vs2,
+                                    "rjc:0");
+         rs = new NonIncJoinStream (vs1,
+                                   "ljc",
+                                   "sec:s1 bid_price:s1_bid_price bid_sz:s1_bid_sz ask_price:s1_ask_price ask_sz:s1_ask_sz",
+                                   vs2,
+                                   "rjc",
+                                   "sec:s2 bid_price:s2_bid_price bid_sz:s2_bid_sz ask_price:s2_ask_price ask_sz:s2_ask_sz");
+         mySetRealSource(rs);
+      }
+   }
+   public class SyntheticSpreadStream : ViewStream{
+      public SyntheticSpreadStream (string s1, string s2, double w1, double w2)
+      : this(s1, s2, w1, w2, new BBOStream(ViewStreamDatabase.instance), ViewStreamDatabase.instance){}
+      public SyntheticSpreadStream (string s1, string s2, double w1, double w2, ViewStream bbo_stream, ViewStreamDatabase vsd){
+         ViewStream rs = new SyntheticSpreadLegsStream(s1, s2, bbo_stream, vsd);
+
+         Func<IBlueTableSchema, IBlueTableSchema> get_schema = (inp_schema) => {
+            IBlueTableSchema rv = inp_schema;
+            rv = rv.add_column(TypeCode.Double, "spread_bid_price");
+            rv = rv.add_column(TypeCode.Double, "spread_bid_sz");
+            rv = rv.add_column(TypeCode.Double, "spread_ask_price");
+            rv = rv.add_column(TypeCode.Double, "spread_ask_sz");
+            return rv;
+         };
+
+         rs = new MapStream(rs, (inp_schema) => {
+           return get_schema(inp_schema);
+         }, 
+         (r) => {
+              double s1_bid_price = r.get<double>("s1_bid_price");
+              double s1_bid_sz = r.get<double>("s1_bid_sz");
+              double s1_ask_price = r.get<double>("s1_ask_price");
+              double s1_ask_sz = r.get<double>("s1_ask_sz");
+               
+              double s2_bid_price = r.get<double>("s2_bid_price");
+              double s2_bid_sz = r.get<double>("s2_bid_sz");
+              double s2_ask_price = r.get<double>("s2_ask_price");
+              double s2_ask_sz = r.get<double>("s2_ask_sz");
+               
+              (double spread_bid_price, double spread_ask_price, double spread_bid_sz, double spread_ask_sz) =   
+              ComputeSpreadBidAsk(s1_bid_price, s1_ask_price, s2_bid_price, s2_ask_price, s1_bid_sz, s1_ask_sz, s2_bid_sz, s2_ask_sz, w1, w2);
+              IBlueTableSchema out_schema = get_schema(r.schema);
+              MutableBlueRow rv = new MutableBlueRow(out_schema, r);
+              rv.setValue("spread_bid_price", spread_bid_price);
+              rv.setValue("spread_ask_price", spread_ask_price);
+              rv.setValue("spread_bid_sz", spread_bid_sz);
+              rv.setValue("spread_ask_sz", spread_ask_sz);
+              return rv;
+              }
+           );
+         mySetRealSource(rs);
+
+      }
+      public static (double SpreadBid, double SpreadAsk, double SpreadBidQty, double SpreadAskQty) ComputeSpreadBidAsk(
+        double s1Bid, double s1Ask,
+        double s2Bid, double s2Ask,
+        double s1BidQty, double s1AskQty,
+        double s2BidQty, double s2AskQty,
+        double w1, double w2)
+    {
+        double spreadBid = (w1 * s1Bid) - (w2 * s2Ask);
+        double spreadAsk = (w1 * s1Ask) - (w2 * s2Bid);
+
+        double spreadBidQty = Math.Min(s1BidQty / w1, s2AskQty / w2);
+        double spreadAskQty = Math.Min(s1AskQty / w1, s2BidQty / w2);
+
+        return (spreadBid, spreadAsk, spreadBidQty, spreadAskQty);
+    }
+   }
+}
@@ -43,7 +43,7 @@
      Condition="Exists($(ComputerSpecificConfig))" />

  <PropertyGroup Label="ProtoBuf">
-    <ProtocPluginBase>$(SparkRoot)\src\lib\api\service_compiler</ProtocPluginBase>
+    <ProtocPluginBase>$(SparkRoot)\src\lib_spark\spark\api\service_compiler</ProtocPluginBase>
  </PropertyGroup>
  <Choose>
    <When Condition="'$(OS)' == 'Windows_NT'">
@@ -24,7 +24,7 @@
    <PropertyGroup>
      <GenProtosCmd>$(protoc) %(Proto.FullPath)</GenProtosCmd>
      <GenProtosCmd>$(GenProtosCmd) --error_format=msvs</GenProtosCmd>
-      <GenProtosCmd>$(GenProtosCmd) --proto_path=$(SourceRoot)</GenProtosCmd>
+      <GenProtosCmd>$(GenProtosCmd) --proto_path=$(SourceRoot)/lib_spark</GenProtosCmd>
      <GenProtosCmd>$(GenProtosCmd) --plugin=&quot;protoc-gen-spark-csharp=$(ProtocPlugin)&quot;</GenProtosCmd>
      <GenProtosCmd>$(GenProtosCmd) --spark-csharp_out=lang=c#,templates=immutable.pb.cs,strip_out_path=1:%(Proto.OutDir)</GenProtosCmd>
      <GenProtosCmd>$(GenProtosCmd) --csharp_out=%(Proto.OutDir)</GenProtosCmd>
@@ -65,7 +65,7 @@
    <PropertyGroup>
      <GenProtosCmd>$(protoc) %(ProtoService.FullPath)</GenProtosCmd>
      <GenProtosCmd>$(GenProtosCmd) --error_format=msvs</GenProtosCmd>
-      <GenProtosCmd>$(GenProtosCmd) --proto_path=$(SourceRoot)</GenProtosCmd>
+      <GenProtosCmd>$(GenProtosCmd) --proto_path=$(SourceRoot)/lib_spark</GenProtosCmd>
      <GenProtosCmd>$(GenProtosCmd) --plugin=&quot;protoc-gen-service=$(ProtocPlugin)&quot;</GenProtosCmd>
      <GenProtosCmd>$(GenProtosCmd) --service_out=lang=c#,strip_out_path=1:%(ProtoService.OutDir)</GenProtosCmd>
      <GenProtosCmd>$(GenProtosCmd) --csharp_out=%(ProtoService.OutDir)</GenProtosCmd>
@@ -1,4 +1,4 @@
-cmake_minimum_required(VERSION 3.10)
+cmake_minimum_required(VERSION 3.5)
 project(Spark.Admin.WebApp)

 include(GNUInstallDirs)
@@ -10,11 +10,11 @@ include(${CMAKE_SOURCE_DIR}/cmake/ProtobufGenerateFs.cmake)
 file(
  GLOB_RECURSE ${PROJECT_NAME}_PROTO_SOURCES
  RELATIVE ${CMAKE_CURRENT_SOURCE_DIR}
-  ../lib/api/*.proto
-  ../lib/api/services/*.proto
-  ../lib/domain_model/*.proto
-  ../lib/sparkproto/*.proto
-  ../lib/md/protos/*.proto
+  ../lib_spark/spark/api/*.proto
+  ../lib_spark/spark/api/services/*.proto
+  ../lib_spark/spark/domain_model/*.proto
+  ../lib_spark/spark/sparkproto/*.proto
+  ../lib_spark/spark/md/protos/*.proto
  ../../externals/linux/include/google/protobuf/descriptor.proto
  ../../externals/linux/include/google/protobuf/timestamp.proto
 )
@@ -44,7 +44,7 @@ set(_build_admin_web_app_deps
  ${_build_admin_web_app_py}
  ${CMAKE_CURRENT_SOURCE_DIR}/src/Spark.Admin.WebApp.fsproj
  ${CMAKE_CURRENT_SOURCE_DIR}/webpack.config.js
-  ${CMAKE_SOURCE_DIR}/src/lib/api/service_compiler/templates/fs/fable.fs
+  ${CMAKE_SOURCE_DIR}/src/lib_spark/spark/api/service_compiler/templates/fs/fable.fs
  ${CMAKE_CURRENT_SOURCE_DIR}/package-lock.json
  ${${PROJECT_NAME}_PROTO_SOURCES}
  ${_api_ts_SOURCES}
@@ -73,7 +73,7 @@ add_custom_command(
 protobuf_generate_js(
  TARGET_NAME generate_js_protos
  PROTO_INCLUDE_DIRS
-    ..
+    ../lib_spark
    ../../externals/linux/include
  PROTO_SOURCES ${${PROJECT_NAME}_PROTO_SOURCES}
  OUTPUT ${CMAKE_CURRENT_BINARY_DIR}/proto.js
@@ -82,7 +82,7 @@ protobuf_generate_js(
 protobuf_generate_fs(
  TARGET_NAME generate_fs_protos
  PROTO_INCLUDE_DIRS
-    ..
+    ../lib_spark
    ../../externals/linux/include
  PROTO_SOURCES ${${PROJECT_NAME}_PROTO_SOURCES}
  OUT_DIR ${CMAKE_CURRENT_BINARY_DIR}/fable
@@ -184,4 +184,3 @@ install(
  DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}/public"
  DESTINATION "${CMAKE_INSTALL_BINDIR}/${PROJECT_NAME}"
 )
-
@@ -0,0 +1,186 @@
+cmake_minimum_required(VERSION 3.14)
+project(sparkd)
+
+include(FetchContent)
+include(${CMAKE_SOURCE_DIR}/cmake/safe_dep_glob.cmake)
+include(${CMAKE_SOURCE_DIR}/cmake/ProtobufGenerateCpp.cmake)
+include(${CMAKE_SOURCE_DIR}/cmake/InstallBasic.cmake)
+include(GNUInstallDirs)
+
+if (NOT DEFINED PROJECT_VERSION)
+  set(PROJECT_VERSION 1.0.0)
+endif()
+
+# dependencies
+file(
+  GLOB_RECURSE ${PROJECT_NAME}_PROTO_SOURCES
+  RELATIVE ${CMAKE_CURRENT_SOURCE_DIR}
+  *.proto
+)
+file(
+  GLOB_RECURSE ${PROJECT_NAME}_SOURCES
+  RELATIVE ${CMAKE_CURRENT_SOURCE_DIR}
+  *.cpp
+)
+update_deps_file("${${PROJECT_NAME}_PROTO_SOURCES};${${PROJECT_NAME}_SOURCES}")
+
+protobuf_generate_cpp(
+  CXX_SOURCES ${PROJECT_NAME}_PROTO_CXX_SOURCES
+  CXX_HEADERS ${PROJECT_NAME}_PROTO_CXX_HEADERS
+  CXX_OUT_DIR ${CMAKE_CURRENT_BINARY_DIR}/include
+  PROTO_INCLUDE_DIRS
+    ${EXTERNALS_INCLUDE}
+    ${CMAKE_SOURCE_DIR}/src/lib_spark
+    ${CMAKE_CURRENT_SOURCE_DIR}
+  PROTO_SOURCES
+    ${${PROJECT_NAME}_PROTO_SOURCES}
+)
+add_executable(sparkd ${${PROJECT_NAME}_SOURCES} ${${PROJECT_NAME}_PROTO_CXX_SOURCES})
+
+# include paths
+target_include_directories(
+  sparkd
+  PUBLIC
+    $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}>
+    $<BUILD_INTERFACE:${CMAKE_CURRENT_BINARY_DIR}>/include
+    $<BUILD_INTERFACE:${emumba-deribit-connector-library_INCLUDE_DIRS}>
+)
+
+find_package(PkgConfig REQUIRED)
+pkg_check_modules(emumba-deribit-connector-library REQUIRED emumba-deribit-connector)
+
+# This dir contains rapidjsonConfig.cmake
+list(APPEND CMAKE_PREFIX_PATH ${CMAKE_CURRENT_SOURCE_DIR}/cmake_modules)
+find_package(rapidjson REQUIRED)
+target_include_directories(
+  ${PROJECT_NAME} SYSTEM
+  PUBLIC
+    $<BUILD_INTERFACE:${rapidjson_INCLUDE_DIR}> 
+)
+add_dependencies(${PROJECT_NAME} rapidjson_project)
+
+# databento
+FetchContent_Declare(
+  databento
+  GIT_REPOSITORY https://github.com/databento/databento-cpp
+  GIT_TAG v0.31.0
+)
+FetchContent_MakeAvailable(databento)
+
+# deribit-demo log level
+if(NOT DEFINED DERIBIT_DEMO_LOG_LEVEL)
+  add_definitions(-DSPDLOG_ACTIVE_LEVEL=0)
+  message("-- Building of DERIBIT_DEMO_LOG_LEVEL=TRACE")
+elseif(DERIBIT_DEMO_LOG_LEVEL STREQUAL "TRACE")
+  add_definitions(-DSPDLOG_ACTIVE_LEVEL=0)
+  message("-- Building of DERIBIT_DEMO_LOG_LEVEL=TRACE")
+elseif(DERIBIT_DEMO_LOG_LEVEL STREQUAL "DEBUG")
+  add_definitions(-DSPDLOG_ACTIVE_LEVEL=1)
+  message("-- Building of DERIBIT_DEMO_LOG_LEVEL=DEBUG")
+elseif(DERIBIT_DEMO_LOG_LEVEL STREQUAL "INFO")
+  add_definitions(-DSPDLOG_ACTIVE_LEVEL=2)
+  message("-- Building of DERIBIT_DEMO_LOG_LEVEL=INFO")
+elseif(DERIBIT_DEMO_LOG_LEVEL STREQUAL "WARN")
+  add_definitions(-DSPDLOG_ACTIVE_LEVEL=3)
+  message("-- Building of DERIBIT_DEMO_LOG_LEVEL=WARN")
+elseif(DERIBIT_DEMO_LOG_LEVEL STREQUAL "ERROR")
+  add_definitions(-DSPDLOG_ACTIVE_LEVEL=4)
+  message("-- Building of DERIBIT_DEMO_LOG_LEVEL=ERROR")
+elseif(DERIBIT_DEMO_LOG_LEVEL STREQUAL "CRITICAL")
+  add_definitions(-DSPDLOG_ACTIVE_LEVEL=5)
+  message("-- Building of DERIBIT_DEMO_LOG_LEVEL=CRITICAL")
+elseif(DERIBIT_DEMO_LOG_LEVEL STREQUAL "OFF")
+  add_definitions(-DSPDLOG_ACTIVE_LEVEL=6)
+  message("-- Building of DERIBIT_DEMO_LOG_LEVEL=OFF")
+else()
+  add_definitions(-DSPDLOG_ACTIVE_LEVEL=0)
+  message("-- Building of DERIBIT_DEMO_LOG_LEVEL=TRACE")
+endif()
+
+target_link_directories(
+  ${PROJECT_NAME} PUBLIC
+  $<BUILD_INTERFACE:${CMAKE_CURRENT_BINARY_DIR}>
+  $<INSTALL_INTERFACE:${CMAKE_INSTALL_LIBDIR}>
+)
+
+# Set RPATH so sparkd knows where to find our shared libs (e.g., libspark).
+# The dirs are relative to sparkd's path, represented by "$ORIGIN".
+set_target_properties(
+  ${PROJECT_NAME} PROPERTIES
+  BUILD_RPATH "$ORIGIN/../lib_spark;$ORIGIN/../lib_python_md"
+  INSTALL_RPATH "$ORIGIN/../${CMAKE_INSTALL_LIBDIR}"
+)
+
+# Python interop (for python md providers)
+add_dependencies(${PROJECT_NAME}
+  python_md
+  python_md_module
+  python_md_providers
+  copy_python_md_providers
+)
+include(${CMAKE_SOURCE_DIR}/cmake/SymLink.cmake)
+
+sym_link_local_lib(python_md)
+sym_link_file(
+  SOURCE ${CMAKE_BINARY_DIR}/src/lib_python_md/python_md_module.cpython-310-x86_64-linux-gnu.so
+  DEST ${CMAKE_CURRENT_BINARY_DIR}/python_md_module.cpython-310-x86_64-linux-gnu.so
+  INSTALL_DEST ${CMAKE_INSTALL_BINDIR}/
+)
+get_target_property(providers python_md_providers INTERFACE_PYTHON_SCRIPTS_DIR)
+sym_link_dir(
+  TARGET_NAME link_python_md_providers
+  SOURCE ${providers}
+  DEST ${CMAKE_CURRENT_BINARY_DIR}/python_md_providers
+)
+
+target_link_libraries(
+  sparkd
+  spark
+  binance_execution
+  okx_execution
+  python_md
+  python_md_module
+  databento::databento
+  ${emumba-deribit-connector-library_LIBRARIES}
+  tbb
+  quickfix
+  benchmark
+  Common++
+  Packet++
+  Pcap++
+  lttng-ust
+  -lpcap
+  -static-libgcc
+  -static-libstdc++
+  -pthread)
+
+# Onixs
+target_link_libraries(
+  ${PROJECT_NAME}
+  onixs-cmemdh
+  OnixS.IceImpactMulticastPriceFeedHandlerCpp
+  # gcc doesn't resolve onixs-fix-engine and ACE in runtime path unless we specify
+  # --no-as-needed. These are onixs deps that aren't called directly by our code.
+  # Note: this isn't an issue for clang
+  -Wl,--no-as-needed
+  onixs-fix-engine
+  ACE
+)
+
+# Catch2
+find_package(Catch2 CONFIG REQUIRED)
+target_link_libraries(${PROJECT_NAME} Catch2::Catch2)
+
+install_basic()
+install(
+  FILES
+    "${EXTERNALS_LIB}/libonixs-cmemdh.so"
+    "${EXTERNALS_LIB}/libonixs-cmemdh.so.4"
+    "${EXTERNALS_LIB}/libonixs-cmemdh.so.4.8.0"
+    "${EXTERNALS_LIB}/libOnixS.IceImpactMulticastPriceFeedHandlerCpp.so.3.16.0.0"
+    "${EXTERNALS_LIB}/libonixs-fix-engine.so"
+    "${EXTERNALS_LIB}/libonixs-fix-engine.so.4"
+    "${EXTERNALS_LIB}/libonixs-fix-engine.so.4.10.1"
+    "${EXTERNALS_LIB}/libACE-5.7.5.so"
+  DESTINATION ${CMAKE_INSTALL_LIBDIR}
+)
@@ -0,0 +1,5 @@
+@PACKAGE_INIT@
+
+include("${CMAKE_CURRENT_LIST_DIR}/@PROJECT_NAME@Targets.cmake")
+
+check_required_components(@PROJECT_NAME@)
@@ -0,0 +1,19 @@
+FROM bluetradingsystems/linux_dotnet7
+
+ARG SOURCE_DIR="install/sparkd-release-gcc"
+ARG CONF_FILE="src/app_sparkd/spark.conf"
+
+USER "root"
+RUN apt-get update \
+  && apt-get install -y liblttng-ust-dev
+
+RUN apt-get update \
+  && apt-get install -y libpcap0.8-dev
+
+RUN apt-get update \
+  && apt-get install -y python3.10-dev
+
+USER "bts"
+COPY --chown=bts:bts "${SOURCE_DIR}" "."
+COPY --chown=bts:bts "${CONF_FILE}" ".local/share/bts/sparkd/conf/spark.conf"
+WORKDIR "/home/bts/bin"
@@ -0,0 +1,4 @@
+SPARKD__CONFIG_DIR=src/app_sparkd/configurations/alphaworx_prod
+SPARKD__SHM_SIZE=8gb
+SPARKD__MEMORY=8gb
+SPARKD__RUNTIME_CONF_DIR=/home/ubuntu/8000/sparkdata/conf/
@@ -1,4 +1,4 @@
-SPARKD__CONFIG_DIR=src/server/sparkd/configurations/alphaworx_zen_hk_live
+SPARKD__CONFIG_DIR=src/app_sparkd/configurations/alphaworx_zen_hk_live
 SPARKD__SHM_SIZE=8gb
 SPARKD__MEMORY=8gb
 SPARKD__IMAGE=sparkd_symbols
@@ -115,7 +115,7 @@ OkxExeCfg {
    ws_url: "wss://wsaws.okx.com:8443"
    rest_url: "https://www.okx.com"
    ws_ping_timeout_seconds: 10 
-    ws_ping_interval_seconds: 25
+    ws_ping_interval_seconds: 10
    ws_reconnect_interval_seconds: 5
    ws_reconnect_max_count: 5
 	  simulated_trading: false
@@ -0,0 +1,2 @@
+SPARKD__CONFIG_DIR=src/app_sparkd/configurations/binance_dev
+SPARKD__SHM_SIZE=8gb
@@ -0,0 +1 @@
+SPARKD__CONFIG_DIR=src/app_sparkd/configurations/py_dummy_md
@@ -0,0 +1,38 @@
+PublisherOptions {
+  value {
+    network_interface: "0.0.0.0"
+    heartbeat_ms: 5000
+    heartbeat_boot_seconds: 60
+    skip_options: true
+    hot_initial: 1024
+    hot_growth: 1024
+    cold_initial: 8192
+    cold_growth: 2048
+    channel_prefix: ""
+    disable_heartbeat: true
+  }
+}
+
+Logging {
+  value {
+    initial_queue_size: 120001
+    max_queue_growth: 4
+    file {
+      default_level: "Info"
+      subsystem {
+        name: "md"
+        level: "Diag"
+      }
+    }
+  }
+}
+
+MarketDataConfig {
+  value {
+    python_md_providers {
+      venue: VENUE_BYBIT
+      module_path: "python_md_providers"
+      provider_type: "bybit_md_provider.BybitMdProvider"
+    }
+  }
+}
@@ -0,0 +1,22 @@
+SymbolLists {
+  value {
+    list {
+      key: "Bybit"
+      symbol: "BTC"
+      symbol: "ETH"    
+      symbol: "BNB"
+      symbol: "XRP"
+      symbol: "ADA"
+      symbol: "PRIME"
+    }
+  }
+}
+
+Symbols {
+  value {
+    venues {
+      name: "VENUE_BYBIT"
+      spot: @SymbolLists.list[Bybit].symbol
+    }
+  }
+}
@@ -0,0 +1,2 @@
+SPARKD__CONFIG_DIR=src/app_sparkd/configurations/dev
+SPARKD__SHM_SIZE=8gb
@@ -0,0 +1 @@
+SPARKD__CONFIG_DIR=src/app_sparkd/configurations/py_dummy_md
@@ -0,0 +1,38 @@
+PublisherOptions {
+  value {
+    network_interface: "0.0.0.0"
+    heartbeat_ms: 5000
+    heartbeat_boot_seconds: 60
+    skip_options: true
+    hot_initial: 1024
+    hot_growth: 1024
+    cold_initial: 8192
+    cold_growth: 2048
+    channel_prefix: ""
+    disable_heartbeat: true
+  }
+}
+
+Logging {
+  value {
+    initial_queue_size: 120001
+    max_queue_growth: 4
+    file {
+      default_level: "Info"
+      subsystem {
+        name: "md"
+        level: "Diag"
+      }
+    }
+  }
+}
+
+MarketDataConfig {
+  value {
+    python_md_providers {
+      venue: VENUE_GATEIO
+      module_path: "python_md_providers"
+      provider_type: "gateio_md_provider.GateIoMdProvider"
+    }
+  }
+}
@@ -0,0 +1,47 @@
+  SymbolLists {
+  value {
+    list {
+      key: "Gateio"
+      symbol: "BTC"
+      symbol: "CAKE"      
+      symbol: "ETH"
+      symbol: "MANA"
+      symbol: "OMG"
+      symbol: "GMX"
+      symbol: "ICP"
+      symbol: "LRC"
+      symbol: "MINA"
+      symbol: "ZIL"
+      symbol: "TURBO"
+      symbol: "OP"
+      symbol: "ZRX"
+      symbol: "VET"
+      symbol: "AR"
+      symbol: "IMX"
+      symbol: "HIGH"
+      symbol: "ARB"
+      symbol: "YFI"
+      symbol: "NEAR"
+      symbol: "OXY"
+      symbol: "WAVES"
+      symbol: "AMP"
+      symbol: "CELO"
+      symbol: "CRV"
+      symbol: "FIL"
+      symbol: "PEOPLE"
+      symbol: "IOTX"
+      symbol: "ATOM"
+      symbol: "NOT"      
+      symbol: "GMT"            
+    }
+  }
+}
+
+Symbols {
+  value {
+    venues {
+      name: "VENUE_GATEIO"
+      spot: @SymbolLists.list[Gateio].symbol
+    }
+  }
+}
@@ -0,0 +1 @@
+SPARKD__CONFIG_DIR=src/app_sparkd/configurations/py_dummy_md
@@ -0,0 +1,38 @@
+PublisherOptions {
+  value {
+    network_interface: "0.0.0.0"
+    heartbeat_ms: 5000
+    heartbeat_boot_seconds: 60
+    skip_options: true
+    hot_initial: 1024
+    hot_growth: 1024
+    cold_initial: 8192
+    cold_growth: 2048
+    channel_prefix: ""
+    disable_heartbeat: true
+  }
+}
+
+Logging {
+  value {
+    initial_queue_size: 120001
+    max_queue_growth: 4
+    file {
+      default_level: "Info"
+      subsystem {
+        name: "md"
+        level: "Diag"
+      }
+    }
+  }
+}
+
+MarketDataConfig {
+  value {
+    python_md_providers {
+      venue: VENUE_GLBX
+      module_path: "python_md_providers"
+      provider_type: "dummy_md_provider.DummyMdProvider"
+    }
+  }
+}
@@ -0,0 +1,17 @@
+SymbolLists {
+  value {
+    list {
+      key: "Glbx"
+      symbol: "ES"
+    }
+  }
+}
+
+Symbols {
+  value {
+    venues {
+      name: "VENUE_GLBX"
+      future: @SymbolLists.list[Glbx].symbol
+    }
+  }
+}
@@ -0,0 +1,2 @@
+SPARKD__CONFIG_DIR=src/app_sparkd/configurations/sim_dev
+SPARKD__SHM_SIZE=8gb
@@ -1,5 +1,5 @@
 #include "AlgoDefinitionServiceServer.h"
-#include "lib/utils/PBUtils.h"
+#include "spark/utils/PBUtils.h"

 namespace bts {
 namespace api {
@@ -1,9 +1,17 @@
 #pragma once

-#include "lib/api/public/ServiceContext.h"
-#include "lib/api/services/AlgoDefinitionService.server.pb.h"
-#include "lib/persistence/IStore.h"
-#include "lib/persistence/SqliteStore.h"
+#include "spark/api/public/ServiceContext.h"
+#include "spark/api/services/AlgoDefinitionService.server.pb.h"
+#include "spark/api/services/OrderService.client.pb.h"
+#include "spark/api/services/OrderService.pb.h"
+#include "spark/logging/logging.h"
+#include "spark/persistence/IStore.h"
+#include "spark/persistence/SqliteStore.h"
+#include "spark/utils/PBUtils.h"
+#include "spark/utils/TimeStamp.h"
+#include "spark/utils/bts_assert.h"
+#include "spark/utils/bts_no_copy.h"
+#include <memory>

 #include <boost/tokenizer.hpp>
 #include <fmt/format.h>
@@ -1,14 +1,14 @@
 #include "AnalyticsServiceServer.h"
-#include "lib/api/apicore.pb.h"
-#include "lib/api/services/AnalyticsService.pb.h"
-#include "lib/api/services/AnalyticsService.server.pb.h"
-#include "lib/logging/logging.h"
-#include "lib/persistence/IStore.h"
-#include "lib/persistence/SqliteStore.h"
-#include "lib/utils/PBUtils.h"
-#include "lib/utils/TimeStamp.h"
-#include "lib/utils/bts_assert.h"
-#include "server/sparkd/algo/AlgoDefinitionServiceServer.h"
+#include "spark/api/apicore.pb.h"
+#include "spark/api/services/AnalyticsService.pb.h"
+#include "spark/api/services/AnalyticsService.server.pb.h"
+#include "spark/logging/logging.h"
+#include "spark/persistence/IStore.h"
+#include "spark/persistence/SqliteStore.h"
+#include "spark/utils/PBUtils.h"
+#include "spark/utils/TimeStamp.h"
+#include "spark/utils/bts_assert.h"
+#include "sparkd/algo/AlgoDefinitionServiceServer.h"

 #include <boost/algorithm/string.hpp>
 #include <boost/tokenizer.hpp>
@@ -1,6 +1,7 @@
 #pragma once

-#include "lib/api/services/AnalyticsService.server.pb.h"
+#include "spark/api/services/AnalyticsService.server.pb.h"
+#include "spark/utils/bts_no_copy.h"

 namespace bts {
 namespace api {
@@ -1,13 +1,13 @@
 #include "AdminReport.h"

-#include "lib/api/public/ServiceContext.h"
-#include "lib/config/config.h"
-#include "lib/config/protos/logging.pb.h"
-#include "lib/logging/logging.h"
-#include "lib/quickbus/quickbus.h"
+#include "spark/api/public/ServiceContext.h"
+#include "spark/config/config.h"
+#include "spark/config/protos/logging.pb.h"
+#include "spark/logging/logging.h"
+#include "spark/quickbus/quickbus.h"

-#include "lib/api/service_helpers/AdminServiceUtils.h"
-#include "lib/api/services/AdminService.client.pb.h"
+#include "spark/api/service_helpers/AdminServiceUtils.h"
+#include "spark/api/services/AdminService.client.pb.h"

 #include <algorithm>
 #include <boost/algorithm/string.hpp>
@@ -1,12 +1,12 @@
 #include "AutoCert.h"
-#include "lib/utils/PBUtils.h"
+#include "spark/utils/PBUtils.h"

-#include "lib/jetfix/common/InMsgBuffer.h"
-#include "lib/utils/bts_chrono.h"
-#include "lib/utils/datetime.h"
+#include "spark/jetfix/common/InMsgBuffer.h"
+#include "spark/utils/bts_chrono.h"
+#include "spark/utils/datetime.h"

-#include "lib/jetfix/codec/i32_toa.h"
-#include "lib/utils/SecureLogon.h"
+#include "spark/jetfix/codec/i32_toa.h"
+#include "spark/utils/SecureLogon.h"
 #include <boost/array.hpp>
 #include <boost/asio.hpp>
 #include <boost/filesystem.hpp>
@@ -1,13 +1,13 @@
 #include "CMECert.h"
-#include "lib/logging/logging.h"
-#include "lib/orderman/ilink3/IL3Dispatcher.h"
-#include "lib/orderman/ilink3/IL3GW.h"
-#include "lib/orderman/ilink3/IL3Reactor.h"
-#include "lib/orderman/ilink3/ILink3Utils.h"
-#include "lib/os/os.h"
-#include "lib/utils/Dispatcher.h"
-#include "lib/utils/SecureLogon.h"
-#include "lib/utils/TimeStamp.h"
+#include "spark/logging/logging.h"
+#include "spark/orderman/ilink3/IL3Dispatcher.h"
+#include "spark/orderman/ilink3/IL3GW.h"
+#include "spark/orderman/ilink3/IL3Reactor.h"
+#include "spark/orderman/ilink3/ILink3Utils.h"
+#include "spark/os/os.h"
+#include "spark/utils/Dispatcher.h"
+#include "spark/utils/SecureLogon.h"
+#include "spark/utils/TimeStamp.h"
 #include <chrono>
 #include <fstream>
 #include <future>
Author	SHA1	Message	Date
Adeel Khan	97b5ec49b4	CLEANUP: BYBIT Symbols.conf.	2025-05-26 11:38:19 +00:00
Adeel Khan	8aaadf1253	ADD: BYBIT MD Working, NaN's also Added.	2025-05-26 11:01:20 +00:00
Adeel Khan	13aa94863d	ADD: BYBIT Securities showing up on frontend. Default Contract Spec	2025-05-22 16:17:19 +00:00
Adeel Khan	29c951fbca	ADD: GATEIO md_provider with dynamic contract specs using Batch REST call	2025-05-22 16:17:19 +00:00
Adeel Khan	ae776cf75f	ADD: GATEIO md_provider with dynamic contract specs 1 REST call per Contract	2025-05-22 16:17:19 +00:00
Adeel Khan	f353a8e9fd	ADD: GATEIO md_provider with default contract specs	2025-05-22 16:17:19 +00:00
John Colvin	24f4c2bb27	Add gateio_dev config profile for Adeel	2025-05-21 04:15:30 +00:00
John Colvin	2960f19e60	cleanup: Remove unused spark.conf file Originally, this spark.conf was hardcoded into compose.yaml, and copied into the sparkd container on each deploy. It hasn't been used since config profiles were introduced.	2025-05-21 04:15:30 +00:00
John Colvin	653d96675b	Add py_dummy_md config profile to test py md plugin	2025-05-21 04:15:30 +00:00
John Colvin	ae2aacdb1c	deploy: Move configurations to new stutter dir "app_sparkd"	2025-05-21 04:15:30 +00:00
John Colvin	11a741eea5	Add protos for gateio	2025-05-21 04:15:30 +00:00
John Colvin	56d0217aad	Add protos for bybit	2025-05-21 04:15:30 +00:00
John Colvin	992b922d68	build: Move onix fix adapter to its own lib We should keep order execution dependencies out of lib_spark.	2025-05-21 04:14:12 +00:00
John Colvin	b1103d5edb	fix build: Move OnixsUtils to from lib_spark to app_sparkd They're only needed MDP3Main.cpp and ICEMain.cpp. We're one step closer to removing lib_spark's dependency on onixs.	2025-05-21 04:14:12 +00:00
John Colvin	1289992127	cleanup: Remove unnecessary onixs include from TrafixOrderStateHandler.h	2025-05-21 04:14:12 +00:00
John Colvin	239548774a	feat: Support python md plugins	2025-05-21 04:14:12 +00:00
John Colvin	d0b0ca42ab	Add "clear" fn to Book & BookSide Prelude to python market data bindings	2025-05-21 04:14:12 +00:00
John Colvin	51882570bd	Add "smart" parse fns for a couple proto enums Prelude to python market data bindings. Unlike the normal parse fns, the "smart" variants aren't case sensitive, and can match strings with or without TYPE_NAME_ prefixes.	2025-05-21 04:14:12 +00:00
John Colvin	3ad95cf0ae	build: Move okx execution to its own lib	2025-05-21 04:14:12 +00:00
John Colvin	02bcbb900a	cleanup: Remove unnecessary onixs include from IDGenerator.h	2025-05-21 04:14:12 +00:00
John Colvin	d8efc38eeb	cleanup: Remove unnecessary includes from MDP3Listener.h	2025-05-21 04:14:12 +00:00
John Colvin	a51143c94b	refactor: Organize libspark proj w/ stuttering include This new structure provides several benefits: * Allows us to install multiple CMake projects in the same destination dir (without include path conflicts). - This makes shipping the trading algo sdk a trivial excercise. Simply copy the lib and include dirs from the install dest. That's it! * Puts us in a position to break up the monolithic libspark into smaller modules, which are easier to manage. Windows specific notes: * Get rid of unnecessary script vs_filters.cmake * Define WebsocketConnectionGateway directly in cs (instead of proto) - The proto would no longer be in the include path passed to protoc. Moving it to c# because there wasn't a compelling reason for it to be a proto in the first place. * Set spark output prefix to "lib" on windows - So we get "libspark.dll" instead of "spark.dll", which conflicts with the csharp lib, "Spark.dll". * Copy libspark.dll to net8.0 & net8.0-windows in a single cmake build target fix build: Copy libspark to both dirs net8.0, net8.0-windows	2025-05-21 04:14:09 +00:00
John Colvin	5e401d49ac	cleanup: Move TradeSummarySignalDispatcher to libspark It is needed by TradeSummarySignal, which is in libspark. This prevents a circular dependency from libspark -> sparkd.	2025-05-20 20:20:07 +00:00
John Colvin	2761a52636	build: Clean up protobuf cmake scripts Make them more flexible, reusable. This is a precursor to porting over python md provider code from another branch.	2025-05-20 20:20:07 +00:00
John Colvin	b2f6421b1e	build: Rename external path vars w/ "EXTERNAL_" prefix in cmake The old "SPARK_" prefix felt like a misnomer.	2025-05-20 20:20:07 +00:00
John Colvin	236ab31fbc	Add a few arithmetic ops to TimeSpan	2025-05-20 20:20:07 +00:00
John Colvin	83b77c9fc0	diag: Log time it takes to start md sources So we can detect bottlenecks	2025-05-20 20:20:07 +00:00
John Colvin	9930b65ddc	refactor: Split MarketDataGateway up into providers/sources MarketDataGateway still exists, but is greatly simplified. It creates several `IMarketDataProvider`s, each of which generates a list of `IMarketDataSource`s. The market data sources provide market data to an `IMarketDataListener`. * EmumbaMarketDataProvider * CCApiMarketDataProvider * PyMarketDataProvider * OnixsMarketDataProvider * ReplayMarketDataProvider As a side effect, individual py market data sources are now explicitly limited to a single venue (it probably should have been this way all along, since the shared memory writers do not expect updates to involve multiple venues).	2025-05-20 20:20:07 +00:00
John Colvin	33093f3497	cleanup: Remove dependency on quickfix for string to_lower We shouldn't depend on external libs for trivial things like converting strings to lower case. Dependencies come at a cost. They add baggage in the form of internal dependencies and increased build time.	2025-05-20 20:20:07 +00:00
John Colvin	5ad7755b28	tool: Make open orders symbol param optional in binance.env	2025-05-20 20:08:34 +00:00
John Colvin	df4ca124c3	cleanup: Remove redundant script get_binance_md.sh I should have removed this when I added binance_rest_client.sh	2025-05-20 20:08:34 +00:00
John Colvin	2c8b9ee288	tool: Add envsource.fish to load .env files in fish	2025-05-20 20:08:34 +00:00
John Colvin	f02f28e347	tool: Add a few vars to binance.env	2025-05-20 20:08:34 +00:00
John Colvin	c8a89c1e8d	cleanup: Rename get_binance_md.py -> binance_client.py This is a more appropriate name (it's not just md anymore).	2025-05-20 20:08:34 +00:00
John Colvin	7ae9e07a30	tool: Use binance.env file in python client Like the bash impl (binance_rest_client.sh), it gets urls, params (templates), and api credentials from env vars.	2025-05-20 20:08:34 +00:00
Taha	a130e47218	Arb Scoring stream	2025-05-20 00:24:17 -04:00
John Colvin	a377f6de7f	improvement (ui): Use sec qty display settings in rec report	2025-05-19 12:35:25 -04:00
John Colvin	855373c9e6	cleanup: Remove unnecessary code	2025-05-19 15:57:35 +00:00
John Colvin	e1b535b633	feat: Sync exch positions in binance usds futs gw	2025-05-19 15:57:35 +00:00
John Colvin	3b9ac71460	refactor: Abstract out start/stop state machine for gen use * From binance usds futures gateway * Thread-safe * Handles re-entrancy gracefully	2025-05-19 15:57:35 +00:00
John Colvin	9a5075daaf	improvement: Use recursive_mutex in ws client to prevent deadlocks	2025-05-19 15:57:35 +00:00
John Colvin	4e7fe811a8	Add ConcurrentUnorderedMap (c++)	2025-05-19 15:57:35 +00:00
Taha	0a6733180d	Added SpreadStatsStream	2025-05-18 01:10:33 -04:00
Taha	f2169c6bd4	Adding more data to SpreadPerformanceStream	2025-05-16 01:42:32 -04:00
Taha	23ef478391	Spread analytics streams	2025-05-15 15:01:27 -04:00
John Colvin	236a5a9362	fix: Format binance order price up to 5 decimal places Bugs (fixed): * Binance JELLYUSDT order prices get rounded to 4 decimal places, even though the price tick is 0.00001	2025-05-13 15:02:02 +00:00
John Colvin	cb560bbe33	fix: Sum realized & unrealized pnl correctly in header rows	2025-05-08 14:13:02 -04:00
John Colvin	248f2644ca	deploy: Automatically restart "sparkd-symbols" service This is what we deploy to the alphaworx-zen-hk-live server	2025-05-07 14:49:53 +00:00
arham	b87d86f4d6	FIx: okx exe user cancel should be cxl not rjct	2025-05-07 08:38:28 +00:00
arham	8515196dcb	Add: okx exe fok orders working	2025-05-07 08:08:40 +00:00
Taha Afzal	e5e042f016	Spreader in RH mode now fires FOK, and hedges only whne the FOK is completely filled (ie not as we we might get partial fills)	2025-05-07 06:12:48 +00:00
John Colvin	da3593326d	Enable FOK orders at binance	2025-05-06 22:20:46 +00:00
John Colvin	313d3a2377	Enable FOK in EMNativeOrder.cpp This class was originally designed for Emumba execution at deribit (hence the "EM" prefix), but now serves all crypto exchanges.	2025-05-06 22:19:11 +00:00
Taha Afzal	22f0e126d0	Adding FillOrKill (FOK) to TimeInForce	2025-05-06 19:39:40 +00:00
arham	c76b061e20	fmt: okx exe	2025-05-02 09:00:00 +00:00
arham	0107b65281	add code for okx exe position updates	2025-05-02 08:59:46 +00:00
Taha Afzal	6b5496b799	OrderStore does not eleted all orders at sparkd boot-time now	2025-05-02 00:10:48 +00:00
arham	42f9daf9cb	Add: okx cancel all after code, not turned on for now	2025-05-01 09:48:06 +00:00
arham	bc5ddade7f	Add: okx exe small changes to maybe make ping better	2025-05-01 09:42:54 +00:00
arham	d8d198f7f4	Add: OKX only handle orders from this session	2025-05-01 09:41:20 +00:00
John Colvin	af9eb670c1	build: Update build scripts to target dotnet 8	2025-04-30 12:06:28 -04:00
				`@@ -0,0 +1 @@`
				`SPARKD__CONFIG_DIR=src/app_sparkd/configurations/py_dummy_md`