CLEANUP: BYBIT Symbols.conf.

Originally, this spark.conf was hardcoded into compose.yaml, and copied
into the sparkd container on each deploy. It hasn't been used since
config profiles were introduced.

.devcontainer/Dockerfile

@@ -65,7 +65,7 @@ RUN \
    && apt-get install -y cmake
 
 # Dotnet
-RUN apt-get update && apt-get install -y dotnet7
+RUN apt-get update && apt-get install -y dotnet8
 
 # Node
 ARG node_major=18

.env

@@ -39,6 +39,7 @@ SPARKD__CONFIG_DIR=src/server/sparkd/configurations/dev
 # Defaults to a volume named "sparkd_<config_port>" (e.g., sparkd_8000).
 # SPARKD__DATA_DIR=/home/bts/sparkdata/data
 SPARKD__SOURCE_DIR=install/sparkd-release-gcc
+SPARKD__RUNTIME_CONF_DIR=/home/ubuntu/8000/sparkdata/conf/
 SPARKD__CONFIG_PORT=8000
 SPARKD__QUICKBUS_PORT=8001
 SPARKD__SIM_FIX_PORT=8002

.gitignore

@@ -60,3 +60,4 @@ trace_data/
 *.Generated.cs
 *.lst
 .env.local
+.secrets

.vscode/launch.json

@@ -1,6 +1,14 @@
 {
   "version": "0.2.0",
   "configurations": [
+    {
+      "name": "Python Debugger: Current File with Arguments",
+      "type": "debugpy",
+      "request": "launch",
+      "program": "${file}",
+      "console": "integratedTerminal",
+      "args": "${command:pickArgs}"
+    },
     {
       "name": ".NET Core Attach",
       "type": "coreclr",
@@ -22,7 +30,7 @@
         "--config",
         "${workspaceFolder}/src/DataStreamSuite/DataStreamPublisher/Configurations/${input:dataStreamPublisherConfig}.json"
       ],
-      "cwd": "build/Debug/bin/net7.0",
+      "cwd": "build/Debug/bin/net8.0",
       "stopAtEntry": false,
       "console": "integratedTerminal"
     },
@@ -82,7 +90,7 @@
           "--config",
           "${workspaceFolder}/src/DataStreamSuite/DataStreamQueryAnalyzer/Configurations/DataStreamQueryAnalyzer.json"
         ],
-        "cwd": "build/Debug/bin/net7.0-windows",
+        "cwd": "build/Debug/bin/net8.0-windows",
         "env": {
           "PYTHON_EXE": "${workspaceFolder}/.venv/Scripts/python.exe"
         },
@@ -106,7 +114,7 @@
         "--config",
         "${workspaceFolder}/src/DataStreamSuite/PubSubServer/Configurations/${input:pubSubServerConfig}.json"
       ],
-      "cwd": "build/Debug/bin/net7.0",
+      "cwd": "build/Debug/bin/net8.0",
       "stopAtEntry": false,
       "console": "integratedTerminal"
     },
@@ -118,7 +126,7 @@
       "windows": {
         "preLaunchTask": "Build Spark (Debug)",
         "program": "Spark.exe",
-        "cwd": "build/Debug/bin/net7.0-windows",
+        "cwd": "build/Debug/bin/net8.0-windows",
       },
       "stopAtEntry": false,
       "console": "integratedTerminal"
@@ -130,7 +138,7 @@
       "program": "",
       "windows": {
         "program": "Spark.exe",
-        "cwd": "build/Debug/bin/net7.0-windows",
+        "cwd": "build/Debug/bin/net8.0-windows",
       },
       "stopAtEntry": false,
       "console": "integratedTerminal"
@@ -147,7 +155,7 @@
           "--sparkd",
           "${input:sparkConfigServer}"
         ],
-        "cwd": "build/Debug/bin/net7.0-windows",
+        "cwd": "build/Debug/bin/net8.0-windows",
       },
       "stopAtEntry": false,
       "console": "integratedTerminal"
@@ -160,7 +168,7 @@
       "windows": {
         "preLaunchTask": "Build Spark (Debug)",
         "program": "Spark.Console.exe",
-        "cwd": "build/Debug/bin/net7.0",
+        "cwd": "build/Debug/bin/net8.0",
         "args": [
           "test",
           "${input:sparkTest}"
@@ -176,7 +184,7 @@
       "program": "",
       "windows": {
         "program": "Spark.Console.exe",
-        "cwd": "build/Debug/bin/net7.0",
+        "cwd": "build/Debug/bin/net8.0",
         "args": [
           "test",
           "${input:sparkTest}"
@@ -191,24 +199,27 @@
       "request": "launch",
       "program": "",
       "linux": {
-        "preLaunchTask": "Build Sparkd (Debug)",
-        "program": "${workspaceFolder}/build/sparkd-debug-clang/server/sparkd",
-        "args": [
-          "run"
-        ],
-        "stopAtEntry": false,
-        "cwd": "${workspaceFolder}/build/sparkd-debug-clang/server",
-        "environment": [],
-        "externalConsole": false,
-        "MIMode": "gdb",
-        "setupCommands": [
-          {
-            "description": "Enable pretty-printing for gdb",
-            "text": "-enable-pretty-printing",
-            "ignoreFailures": true
-          }
-        ]
-      }
+        "program": "${workspaceFolder}/build/sparkd-debug-clang/src/app_sparkd/sparkd",
+      },
+      "args": [
+        "run"
+      ],
+      "cwd": "${workspaceFolder}/build/sparkd-debug-clang/src/app_sparkd",
+      "stopAtEntry": false,
+      "externalConsole": false,
+      "MIMode": "gdb",
+      "setupCommands": [
+        {
+          "description": "Enable pretty-printing for gdb",
+          "text": "-enable-pretty-printing",
+          "ignoreFailures": true
+        },
+        {
+          "description": "Set Disassembly Flavor to Intel",
+          "text": "-gdb-set disassembly-flavor intel",
+          "ignoreFailures": true
+        }
+      ]
     },
     {
       "name": "Sparkd (Skip Build)",
@@ -216,74 +227,27 @@
       "request": "launch",
       "program": "",
       "linux": {
-        "program": "${workspaceFolder}/build/sparkd-debug-clang/server/sparkd",
-        "args": [
-          "run"
-        ],
-        "stopAtEntry": false,
-        "cwd": "${workspaceFolder}/build/sparkd-debug-clang/server",
-        "environment": [],
-        "externalConsole": false,
-        "MIMode": "gdb",
-        "setupCommands": [
-          {
-            "description": "Enable pretty-printing for gdb",
-            "text": "-enable-pretty-printing",
-            "ignoreFailures": true
-          }
-        ]
-      }
-    },
-    {
-      "name": "Sparkd Test",
-      "type": "cppdbg",
-      "request": "launch",
-      "program": "",
-      "linux": {
-        "preLaunchTask": "Build Sparkd (Debug)",
-        "program": "${workspaceFolder}/build/sparkd-debug-clang/server/sparkd",
-        "args": [
-          "test",
-          "${input:sparkdTest}",
-        ],
-        "stopAtEntry": false,
-        "cwd": "${workspaceFolder}/build/sparkd-debug-clang/server",
-        "environment": [],
-        "externalConsole": false,
-        "MIMode": "gdb",
-        "setupCommands": [
-          {
-            "description": "Enable pretty-printing for gdb",
-            "text": "-enable-pretty-printing",
-            "ignoreFailures": true
-          }
-        ]
-      }
-    },
-    {
-      "name": "Sparkd Test (Skip Build)",
-      "type": "cppdbg",
-      "request": "launch",
-      "program": "",
-      "linux": {
-        "program": "${workspaceFolder}/build/sparkd-debug-clang/server/sparkd",
-        "args": [
-          "test",
-          "${input:sparkdTest}",
-        ],
-        "stopAtEntry": false,
-        "cwd": "${workspaceFolder}/build/sparkd-debug-clang/server",
-        "environment": [],
-        "externalConsole": false,
-        "MIMode": "gdb",
-        "setupCommands": [
-          {
-            "description": "Enable pretty-printing for gdb",
-            "text": "-enable-pretty-printing",
-            "ignoreFailures": true
-          }
-        ]
-      }
+        "program": "${workspaceFolder}/build/sparkd-debug-clang/src/app_sparkd/sparkd",
+      },
+      "args": [
+        "run"
+      ],
+      "cwd": "${workspaceFolder}/build/sparkd-debug-clang/src/app_sparkd",
+      "stopAtEntry": false,
+      "externalConsole": false,
+      "MIMode": "gdb",
+      "setupCommands": [
+        {
+          "description": "Enable pretty-printing for gdb",
+          "text": "-enable-pretty-printing",
+          "ignoreFailures": true
+        },
+        {
+          "description": "Set Disassembly Flavor to Intel",
+          "text": "-gdb-set disassembly-flavor intel",
+          "ignoreFailures": true
+        }
+      ]
     },
   ],
   "inputs": [

CMakeLists.txt

@@ -22,9 +22,9 @@ set(
 # can run simultaneously without fighting over the same shm files (e.g., sparkd,
 # api server, etc.).
 set(SHARED_MEMORY_PREFIX "bts" CACHE STRING "Prefix to use when creating shared memory files")
-set(SPARK_BIN "${EXTERNALS_ROOT}/bin")
-set(SPARK_INCLUDE "${EXTERNALS_ROOT}/include")
-set(SPARK_LIB "${EXTERNALS_ROOT}/lib/$<CONFIG>")
+set(EXTERNALS_BIN "${EXTERNALS_ROOT}/bin")
+set(EXTERNALS_INCLUDE "${EXTERNALS_ROOT}/include")
+set(EXTERNALS_LIB "${EXTERNALS_ROOT}/lib/$<CONFIG>")
 set(BUILD_DOCS OFF CACHE BOOL "Build documentation")
 set(BUILD_SDK OFF CACHE BOOL "Build the sdk")
 set(EXPORT_MD_CONSUMER OFF CACHE BOOL "Export (extern) C functions for MD consumer interop")
@@ -36,8 +36,8 @@ else()
   set(BUILD_ADMIN_WEB_APP ON CACHE BOOL "Build the Admin WebApp")
 endif()
 
-include_directories(BEFORE SYSTEM ${SPARK_INCLUDE} ${EXTERNALS_ROOT}/include/zlib)
-link_directories(${SPARK_LIB})
+include_directories(BEFORE SYSTEM ${EXTERNALS_INCLUDE} ${EXTERNALS_ROOT}/include/zlib)
+link_directories(${EXTERNALS_LIB})
 
 add_compile_definitions(SHARED_MEMORY_PREFIX="${SHARED_MEMORY_PREFIX}")
 add_compile_definitions(APP_DATA_DIR="${APP_DATA_DIR}")
@@ -47,12 +47,8 @@ endif()
 add_compile_definitions(EXPORT_MD_CONSUMER="${EXPORT_MD_CONSUMER}")
 if(WIN32)
   # subdirs
-  add_subdirectory(src/lib)
+  add_subdirectory(src/lib_spark)
 else()
-  # make sparkd look for libspark.so in its directory
-  set(CMAKE_BUILD_WITH_INSTALL_RPATH TRUE)
-  set(CMAKE_INSTALL_RPATH ".")
-
   if(BUILD_SPARKD)
     # Note: All macros that begin w/ 'CCAPI' are ccapi's macros (defined and/or
     # read by their code). We override some below.
@@ -118,11 +114,14 @@ else()
 
   # subdirs
   if (BUILD_SPARKD)
+    add_subdirectory(src/lib_onixs_fix)
     add_subdirectory(src/lib_binance_execution)
-    add_subdirectory(src/server/sparkd)
+    add_subdirectory(src/lib_okx_execution)
+    add_subdirectory(src/lib_python_md)
+    add_subdirectory(src/app_sparkd)
     add_subdirectory(src/data_files)
   endif()
-  add_subdirectory(src/lib)
+  add_subdirectory(src/lib_spark)
   if (BUILD_ADMIN_WEB_APP)
     add_subdirectory(src/Spark.Admin.WebApp)
   endif()
@@ -132,37 +131,4 @@ else()
   if(BUILD_SDK)
     add_subdirectory(src/sdk)
   endif()
-
-  if(BUILD_SPARKD)
-    add_custom_command(
-      OUTPUT ${CMAKE_BINARY_DIR}/server
-      COMMAND mkdir -p ${CMAKE_BINARY_DIR}/server
-      COMMAND ln ARGS -sf ${CMAKE_BINARY_DIR}/src/server/sparkd/sparkd ${CMAKE_BINARY_DIR}/server
-      COMMAND ln ARGS -sf ${CMAKE_BINARY_DIR}/data_files ${CMAKE_BINARY_DIR}/server
-      COMMAND ln ARGS -sf ${CMAKE_BINARY_DIR}/src/lib/libspark.so ${CMAKE_BINARY_DIR}/server
-      COMMAND ln ARGS -sf ${EXTERNALS_ROOT}/lib/${CMAKE_BUILD_TYPE}/libonixs-fix-engine.so.4 ${CMAKE_BINARY_DIR}/server
-      
-      COMMAND
-        ln ARGS -sf
-        ${EXTERNALS_ROOT}/lib/Release/libOnixS.IceImpactMulticastPriceFeedHandlerCpp.so.3.16.0.0
-        ${CMAKE_BINARY_DIR}/server
-      COMMAND ln ARGS -sf ${EXTERNALS_ROOT}/lib/Release/libACE-5.7.5.so ${CMAKE_BINARY_DIR}/server
-      COMMAND mkdir ARGS -p ${CMAKE_BINARY_DIR}/server/plugins
-      COMMENT "Creating server runtime layout"
-      DEPENDS sparkd data_files_dir)
-
-    add_custom_target(server_layout ALL DEPENDS ${CMAKE_BINARY_DIR}/server)
-
-    install(
-      DIRECTORY "${CMAKE_BINARY_DIR}/server/plugins"
-      DESTINATION "${CMAKE_INSTALL_BINDIR}")
-    install(
-      FILES
-      "${SPARK_LIB}/libACE-5.7.5.so"
-      "${SPARK_LIB}/libOnixS.IceImpactMulticastPriceFeedHandlerCpp.so.3.16.0.0"
-      "${SPARK_LIB}/libonixs-fix-engine.so.4.10.1"
-      "${SPARK_LIB}/libonixs-fix-engine.so.4"
-      "${SPARK_LIB}/libonixs-fix-engine.so"
-      DESTINATION "${CMAKE_INSTALL_BINDIR}")
-  endif()
 endif()

CMakePresets.json

@@ -31,9 +31,8 @@
         "CMAKE_SHARED_LINKER_FLAGS_RELEASE": "/OPT:REF /OPT:ICF /LTCG",
         "CMAKE_TOOLCHAIN_FILE": "${sourceDir}/vcpkg/scripts/buildsystems/vcpkg.cmake",
         "EXTERNALS_ROOT": "${sourceDir}/externals/windows",
-        "CONFIG_DEFAULTS_PATH": "config/profiles/default/*.conf",
-        "DOTNET_BINARY_DIR": "${sourceDir}/build/$<CONFIG>/bin/net7.0",
-        "DOTNET_WINDOWS_BINARY_DIR": "${sourceDir}/build/$<CONFIG>/bin/net7.0-windows",
+        "CONFIG_DEFAULTS_PATH": "spark/config/profiles/default/*.conf",
+        "DOTNET_BINARY_DIR": "${sourceDir}/build/$<CONFIG>/bin/net8.0;${sourceDir}/build/$<CONFIG>/bin/net8.0-windows",
         "EXPORT_MD_CONSUMER": {
           "type": "BOOL",
           "value": "ON"
@@ -58,12 +57,18 @@
         "APP_DATA_DIR_OVERRIDE": "${dollar}{SPARKD_DATA_DIR}",
         "CMAKE_BUILD_WITH_INSTALL_RPATH": {
           "type": "BOOL",
-          "value": "ON"
+          "value": "OFF"
         },
         "CMAKE_CXX_STANDARD": "20",
         "CMAKE_INSTALL_RPATH": ".",
         "CMAKE_INSTALL_PREFIX": "${sourceDir}/install/${presetName}",
+        "CMAKE_PREFIX_PATH": "${sourceDir}/cmake/;${sourceDir}/.venv/lib/python3.10/site-packages/pybind11/share/cmake/pybind11",
         "CMAKE_EXE_LINKER_FLAGS": "-Wno-undef",
+        "CMAKE_TOOLCHAIN_FILE": "${sourceDir}/vcpkg/scripts/buildsystems/vcpkg.cmake",
+        "DATABENTO_USE_EXTERNAL_JSON": {
+          "type": "BOOL",
+          "value": "ON"
+        },
         "EXTERNALS_ROOT": "${sourceDir}/externals/linux",
         "BUILD_SDK": {
           "type": "BOOL",
@@ -88,7 +93,7 @@
           "value": "ON"
         },
         "CCAPI_LOG_LEVEL": "Info",
-        "CONFIG_DEFAULTS_PATH": "config/profiles/md_dev/*.conf;config/profiles/default/*.conf"
+        "CONFIG_DEFAULTS_PATH": "spark/config/profiles/md_dev/*.conf;spark/config/profiles/default/*.conf"
       }
     },
 
@@ -99,7 +104,7 @@
       "cacheVariables": {
         "CMAKE_BUILD_TYPE": "Release",
         "CCAPI_LOG_LEVEL": "Warn",
-        "CONFIG_DEFAULTS_PATH": "config/profiles/md_prod/*.conf;config/profiles/default/*.conf"
+        "CONFIG_DEFAULTS_PATH": "spark/config/profiles/md_prod/*.conf;spark/config/profiles/default/*.conf"
       }
     },
 
@@ -181,8 +186,12 @@
         "CMAKE_CXX_FLAGS_DEBUG": "-g -fPIC -Wall -pedantic -Wextra -Werror -Wno-missing-braces -Wno-ignored-qualifiers --std=c++20 -fdiagnostics-color -DDEBUG",
         "CMAKE_CXX_FLAGS_RELEASE": "-fPIC -Wall -pedantic -Wextra -Werror -Wno-missing-braces -Wno-ignored-qualifiers --std=c++20 -fdiagnostics-color -fno-omit-frame-pointer -DNDEBUG -O3",
         "CMAKE_INSTALL_PREFIX": "${sourceDir}/install/${presetName}",
-        "CONFIG_DEFAULTS_PATH": "config/profiles/default/*.conf",
-        "DOTNET_BINARY_DIR": "${sourceDir}/build/$<CONFIG>/bin/net7.0",
+        "CMAKE_EXPORT_COMPILE_COMMANDS": {
+          "type": "BOOL",
+          "value": "ON"
+        },
+        "CONFIG_DEFAULTS_PATH": "spark/config/profiles/default/*.conf",
+        "DOTNET_BINARY_DIR": "${sourceDir}/build/$<CONFIG>/bin/net8.0",
         "EXPORT_MD_CONSUMER": {
           "type": "BOOL",
           "value": "ON"
@@ -222,7 +231,7 @@
       "cacheVariables": {
         "APP_DATA_DIR": "${dollar}{APPDATA}/bts/DataStream",
         "APP_DATA_DIR_OVERRIDE": "${dollar}{DATA_STREAM_DATA_DIR}",
-        "CONFIG_DEFAULTS_PATH": "config/profiles/default/*.conf;config/profiles/data_stream/*.conf",
+        "CONFIG_DEFAULTS_PATH": "spark/config/profiles/default/*.conf;spark/config/profiles/data_stream/*.conf",
         "SHARED_MEMORY_PREFIX": "data_stream"
       }
     },
@@ -257,8 +266,7 @@
         "CMAKE_CXX_COMPILER": "clang++",
         "CMAKE_CXX_FLAGS_DEBUG": "-g -fPIC -Wall -pedantic -Wextra -Werror -Wno-missing-braces -Wno-ignored-qualifiers --std=c++20 -fdiagnostics-color -DDEBUG",
         "CMAKE_CXX_FLAGS_RELEASE": "-fPIC -Wall -pedantic -Wextra -Werror -Wno-missing-braces -Wno-ignored-qualifiers --std=c++20 -fdiagnostics-color -fno-omit-frame-pointer -DNDEBUG -O3",
-        "CMAKE_INSTALL_PREFIX": "${sourceDir}/install/${presetName}",
-        "CONFIG_DEFAULTS_PATH": "config/profiles/default/*.conf;config/profiles/data_stream/*.conf",
+        "CONFIG_DEFAULTS_PATH": "spark/config/profiles/default/*.conf;spark/config/profiles/data_stream/*.conf",
         "DOTNET_BINARY_DIR": "${sourceDir}/build/$<CONFIG>/bin/net7.0",
         "EXPORT_MD_CONSUMER": {
           "type": "BOOL",

cmake/ExportToParent.cmake

@@ -0,0 +1,7 @@
+include_guard(GLOBAL)
+
+macro(export_to_parent)
+  foreach(var IN ITEMS ${ARGN})
+    set(${var} ${${var}} PARENT_SCOPE)
+  endforeach()
+endmacro()

cmake/InitLibrary.cmake

@@ -0,0 +1,142 @@
+include_guard(DIRECTORY)
+include(${CMAKE_SOURCE_DIR}/cmake/ExportToParent.cmake)
+include(${CMAKE_SOURCE_DIR}/cmake/InstallBasic.cmake)
+include(${CMAKE_SOURCE_DIR}/cmake/ProtobufGenerateCpp.cmake)
+include(${CMAKE_SOURCE_DIR}/cmake/SetFirstDefined.cmake)
+include(${CMAKE_SOURCE_DIR}/cmake/safe_dep_glob.cmake)
+
+function(init_library)
+  set(_options SHARED STATIC)
+  set(_one_value_args
+    NAME
+    SOURCE_DIR
+    INCLUDE_DIR
+  )
+  set(_multi_value_args)
+  cmake_parse_arguments(_ARGS "${_options}" "${_one_value_args}" "${_multi_value_args}" ${ARGN})
+  if (_ARGS_SHARED AND _ARGS_STATIC)
+    message(FATAL_ERROR "Invalid args passed to init_library: cannot specify STATIC and SHARED")
+  endif()
+  set_first_defined(_name _ARGS_NAME PROJECT_NAME)
+  set_first_defined(_source_dir _ARGS_SOURCE_DIR _name)
+  set_first_defined(_include_dir _ARGS_INCLUDE_DIR _source_dir)
+
+  # Cpp sources
+  file(
+    GLOB_RECURSE ${_name}_SOURCES
+    RELATIVE ${CMAKE_CURRENT_SOURCE_DIR}
+    ${_source_dir}/*.cpp
+  )
+  # Note: cpp files inside 'tests' dir(s) are considered tests.
+  # Remove tests from main sources, and add them to test sources
+  foreach(item ${${_name}_SOURCES})
+    if(${item} MATCHES "tests/.*\.cpp$")
+      list(APPEND ${_name}_TEST_SOURCES ${item})
+    endif()
+  endforeach()
+  foreach(item ${${_name}_TEST_SOURCES})
+    list(REMOVE_ITEM ${_name}_SOURCES ${item})
+  endforeach()
+
+  # Proto sources 
+  file(
+    GLOB_RECURSE ${_name}_PROTO_SOURCES
+    RELATIVE ${CMAKE_CURRENT_SOURCE_DIR}
+    ${_source_dir}/protobufs/*.proto
+  )
+  update_deps_file("${${_name}_PROTO_SOURCES};")
+  protobuf_generate_cpp(
+    CXX_SOURCES ${_name}_CXX_PROTO_SOURCES
+    CXX_HEADERS ${_name}_CXX_PROTO_HEADERS
+    CXX_OUT_DIR ${CMAKE_CURRENT_BINARY_DIR}
+    PROTO_INCLUDE_DIRS
+      ${EXTERNALS_INCLUDE}
+      ${CMAKE_CURRENT_SOURCE_DIR}
+    PROTO_SOURCES
+      ${${_name}_PROTO_SOURCES}
+  )
+
+  export_to_parent(
+    ${_name}_SOURCES
+    ${_name}_TEST_SOURCES
+    ${_name}_PROTO_SOURCES
+    ${_name}_CXX_PROTO_SOURCES
+    ${_name}_CXX_PROTO_HEADERS
+  )
+
+  # Add main lib
+  if (_ARGS_STATIC)
+    add_library(${_name} STATIC ${${_name}_SOURCES} ${${_name}_CXX_PROTO_SOURCES})
+  elseif(_ARGS_SHARED)
+    add_library(${_name} SHARED ${${_name}_SOURCES} ${${_name}_CXX_PROTO_SOURCES})
+  else()
+    add_library(${_name} ${${_name}_SOURCES} ${${_name}_CXX_PROTO_SOURCES})
+  endif()
+  target_include_directories(
+    ${_name}
+    SYSTEM
+    BEFORE
+    PUBLIC
+      $<BUILD_INTERFACE:${EXTERNALS_ROOT}/include>
+      $<BUILD_INTERFACE:${EXTERNALS_ROOT}/include/zlib>
+  )
+  target_include_directories(
+    ${_name}
+    PUBLIC
+      $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}>
+      $<INSTALL_INTERFACE:${CMAKE_INSTALL_INCLUDEDIR}>
+  )
+  # Include protos (if any)
+  if (${_name}_PROTO_SOURCES)
+    target_include_directories(
+      ${_name}
+      PUBLIC
+        $<BUILD_INTERFACE:${CMAKE_CURRENT_BINARY_DIR}>
+    )
+  endif()
+  target_link_directories(
+    ${_name}
+    PUBLIC
+      $<BUILD_INTERFACE:${CMAKE_CURRENT_BINARY_DIR}>
+      $<BUILD_INTERFACE:${EXTERNALS_ROOT}/lib/$<CONFIG>>
+      $<INSTALL_INTERFACE:${CMAKE_INSTALL_LIBDIR}>
+  )
+  # Install main lib
+  foreach(_hdr ${${_name}_CXX_PROTO_HEADERS})
+    file(RELATIVE_PATH _relative_path ${CMAKE_CURRENT_BINARY_DIR} ${_hdr})
+    install(FILES ${_hdr} DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}/${_relative_path})
+  endforeach()
+  install_basic(
+    NAME ${_name}
+    INCLUDE_DIR ${_include_dir}
+  )
+
+  # Add tests executable (if there are any)
+  if (${_name}_TEST_SOURCES AND NOT WIN32)
+    add_executable(${_name}-tests ${${_name}_TEST_SOURCES})
+
+    get_target_property(_lib_type ${_name} TYPE)
+    if (_lib_type STREQUAL "SHARED_LIBRARY")
+      # Set RPATH so the tests exe knows where to find the shared lib.
+      set_target_properties(
+        ${_name}-tests
+        PROPERTIES
+          BUILD_RPATH "$ORIGIN"
+          INSTALL_RPATH "$ORIGIN/../${CMAKE_INSTALL_LIBDIR}"
+      )
+    endif()
+
+    # Link main lib
+    target_link_directories(
+      ${_name}-tests
+      PRIVATE
+        $<BUILD_INTERFACE:${CMAKE_CURRENT_BINARY_DIR}>
+        $<INSTALL_INTERFACE:${CMAKE_INSTALL_LIBDIR}>
+    )
+    target_link_libraries(${_name}-tests PRIVATE ${_name})
+
+    # Catch2
+    find_package(Catch2 CONFIG REQUIRED COMPONENTS Catch2WithMain)
+    target_link_libraries(${_name}-tests PRIVATE Catch2::Catch2WithMain)
+  endif()
+endfunction()

cmake/InstallBasic.cmake

@@ -0,0 +1,39 @@
+include_guard(DIRECTORY)
+include(GNUInstallDirs)
+include(${CMAKE_SOURCE_DIR}/cmake/PackBasic.cmake)
+include(${CMAKE_SOURCE_DIR}/cmake/SetFirstDefined.cmake)
+
+function(install_basic)
+  set(_options)
+  set(_one_value_args NAME INCLUDE_DIR)
+  set(_multi_value_args)
+  cmake_parse_arguments(_ARGS "${_options}" "${_one_value_args}" "${_multi_value_args}" ${ARGN})
+  set_first_defined(_name _ARGS_NAME PROJECT_NAME)
+  set_first_defined(_include_dir _ARGS_INCLUDE_DIR _name)
+
+  file(
+    GLOB_RECURSE ${_name}_HEADERS
+    RELATIVE ${CMAKE_CURRENT_SOURCE_DIR}
+    ${_include_dir}/*.h)
+
+  target_sources(${_name}
+    PUBLIC
+      FILE_SET HEADERS FILES ${${_name}_HEADERS})
+
+  install(
+    TARGETS ${_name}
+    EXPORT ${_name}Targets
+    RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR}
+    LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR}
+    ARCHIVE DESTINATION ${CMAKE_INSTALL_LIBDIR}
+    FILE_SET HEADERS DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}
+  )
+
+  install(
+    EXPORT ${_name}Targets
+    FILE ${_name}Targets.cmake
+    NAMESPACE ${_name}::
+    DESTINATION ${CMAKE_INSTALL_LIBDIR}/cmake/${_name})
+
+  pack_basic()
+endfunction()

cmake/PackBasic.cmake

@@ -0,0 +1,44 @@
+include_guard(DIRECTORY)
+include(CMakePackageConfigHelpers)
+include(${CMAKE_SOURCE_DIR}/cmake/SetFirstDefined.cmake)
+
+function(pack_basic)
+  set(_options)
+  set(_one_value_args NAME)
+  set(_multi_value_args)
+  cmake_parse_arguments(_ARGS "${_options}" "${_one_value_args}" "${_multi_value_args}" ${ARGN})
+  set_first_defined(_name _ARGS_NAME PROJECT_NAME)
+
+  find_package(Git QUIET)
+
+  if(GIT_FOUND)
+    execute_process(
+      COMMAND ${GIT_EXECUTABLE} describe --tags --always --dirty
+      WORKING_DIRECTORY ${CMAKE_SOURCE_DIR}
+      OUTPUT_VARIABLE PROJECT_VERSION
+      OUTPUT_STRIP_TRAILING_WHITESPACE
+    )
+  endif()
+
+  if(NOT GIT_FOUND OR PROJECT_VERSION STREQUAL "")
+    set(PROJECT_VERSION "0.0.0")
+  endif()
+
+  write_basic_package_version_file(
+    "${CMAKE_CURRENT_BINARY_DIR}/${_name}ConfigVersion.cmake"
+    VERSION ${PROJECT_VERSION}
+    COMPATIBILITY AnyNewerVersion
+  )
+
+  configure_package_config_file(
+    "${CMAKE_CURRENT_SOURCE_DIR}/Config.cmake.in"
+    "${CMAKE_CURRENT_BINARY_DIR}/${_name}Config.cmake"
+    INSTALL_DESTINATION ${CMAKE_INSTALL_LIBDIR}/cmake/${_name}
+  )
+
+  install(
+    FILES
+      "${CMAKE_CURRENT_BINARY_DIR}/${_name}Config.cmake"
+      "${CMAKE_CURRENT_BINARY_DIR}/${_name}ConfigVersion.cmake"
+    DESTINATION ${CMAKE_INSTALL_LIBDIR}/cmake/${_name})
+endfunction()

cmake/Protobuf.cmake

@@ -1,12 +1,23 @@
-set(Protobuf_PROTOC_EXECUTABLE "${SPARK_BIN}/protoc")
-set(PROTOBUF_INCLUDE_DIRS "${SPARK_INCLUDE}")
+set(Protobuf_PROTOC_EXECUTABLE "${EXTERNALS_BIN}/protoc")
+set(PROTOBUF_INCLUDE_DIRS "${EXTERNALS_INCLUDE}")
 set(PROTOBUF_STATIC_LIB "libprotobuf")
+set(_protobuf_src_dir ${CMAKE_SOURCE_DIR}/src/lib_spark)
+set(_protobuf_bin_dir ${CMAKE_BINARY_DIR}/src/lib_spark)
 
-function(Protobuf_Generate_Cpp SRCS HDRS)
-  set(${SRCS})
-  set(${HDRS})
+function(Protobuf_Generate_Cpp)
+  set(_options)
+  set(_one_value_args CXX_SOURCES CXX_HEADERS CXX_OUT_DIR)
+  set(_multi_value_args PROTO_INCLUDE_DIRS PROTO_SOURCES)
+  cmake_parse_arguments(_ARGS "${_options}" "${_one_value_args}" "${_multi_value_args}" ${ARGN})
+  set(${_ARGS_CXX_SOURCES})
+  set(${_ARGS_CXX_HEADERS})
 
-  foreach(FIL ${ARGN})
+  set(_proto_include_cli_args)
+  foreach(_dir ${_ARGS_PROTO_INCLUDE_DIRS})
+    list(APPEND _proto_include_cli_args -I ${_dir})
+  endforeach()
+
+  foreach(FIL ${_ARGS_PROTO_SOURCES})
     get_filename_component(ABS_FIL ${FIL} ABSOLUTE)
     get_filename_component(FIL_WE ${FIL} NAME_WE)
     get_filename_component(FIL_DIR ${FIL} DIRECTORY)
@@ -14,48 +25,56 @@ function(Protobuf_Generate_Cpp SRCS HDRS)
       set(FIL_WE "${FIL_DIR}/${FIL_WE}")
     endif()
 
-    list(APPEND ${SRCS} "${CMAKE_CURRENT_BINARY_DIR}/${FIL_WE}.pb.cc")
-    list(APPEND ${HDRS} "${CMAKE_CURRENT_BINARY_DIR}/${FIL_WE}.pb.h")
+    list(APPEND ${_ARGS_CXX_SOURCES} "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.pb.cc")
+    list(APPEND ${_ARGS_CXX_HEADERS} "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.pb.h")
 
     add_custom_command(
-      OUTPUT "${CMAKE_CURRENT_BINARY_DIR}/${FIL_WE}.pb.cc"
-             "${CMAKE_CURRENT_BINARY_DIR}/${FIL_WE}.pb.h"
+      OUTPUT "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.pb.cc"
+             "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.pb.h"
       COMMAND
-        ${Protobuf_PROTOC_EXECUTABLE} ARGS --cpp_out=${CMAKE_BINARY_DIR}/src -I
-        ${PROTOBUF_INCLUDE_DIRS} -I ${CMAKE_SOURCE_DIR}/src ${ABS_FIL}
+        ${Protobuf_PROTOC_EXECUTABLE} ARGS --cpp_out=${_ARGS_CXX_OUT_DIR} ${_proto_include_cli_args} ${ABS_FIL}
       DEPENDS ${ABS_FIL}
       COMMENT "Running C++ protocol buffer compiler on ${FIL}"
       VERBATIM)
   endforeach()
 
-  set_source_files_properties(${${SRCS}} ${${HDRS}} PROPERTIES GENERATED TRUE)
+  set_source_files_properties(${${_ARGS_CXX_SOURCES}} ${${_ARGS_CXX_HEADERS}} PROPERTIES GENERATED TRUE)
 
   # remove some warnings from protobuf generated files
   if(WIN32)
-    set_source_files_properties(${${SRCS}} PROPERTIES COMPILE_FLAGS
+    set_source_files_properties(${${_ARGS_CXX_SOURCES}} PROPERTIES COMPILE_FLAGS
                                                       "/wd4309 /wd4838")
   else()
     set_source_files_properties(
-      ${${SRCS}}
+      ${${_ARGS_CXX_SOURCES}}
       PROPERTIES
         COMPILE_FLAGS
         "-Wno-unused-parameter -Wno-effc++ -Wno-pedantic -Wno-sign-compare -Wno-uninitialized"
     )
   endif()
 
-  set(${SRCS}
-      ${${SRCS}}
+  set(${_ARGS_CXX_SOURCES}
+      ${${_ARGS_CXX_SOURCES}}
       PARENT_SCOPE)
-  set(${HDRS}
-      ${${HDRS}}
+  set(${_ARGS_CXX_HEADERS}
+      ${${_ARGS_CXX_HEADERS}}
       PARENT_SCOPE)
 endfunction()
 
-function(Service_Compiler_Generate_Cpp SRCS HDRS)
-  set(${SRCS})
-  set(${HDRS})
+function(Service_Compiler_Generate_Cpp)
+  set(_options)
+  set(_one_value_args CXX_SOURCES CXX_HEADERS CXX_OUT_DIR)
+  set(_multi_value_args PROTO_INCLUDE_DIRS PROTO_SOURCES)
+  cmake_parse_arguments(_ARGS "${_options}" "${_one_value_args}" "${_multi_value_args}" ${ARGN})
+  set(${_ARGS_CXX_SOURCES})
+  set(${_ARGS_CXX_HEADERS})
 
-  foreach(FIL ${ARGN})
+  set(_proto_include_cli_args)
+  foreach(_dir ${_ARGS_PROTO_INCLUDE_DIRS})
+    list(APPEND _proto_include_cli_args -I ${_dir})
+  endforeach()
+
+  foreach(FIL ${_ARGS_PROTO_SOURCES})
     get_filename_component(ABS_FIL ${FIL} ABSOLUTE)
     get_filename_component(FIL_WE ${FIL} NAME_WE)
     get_filename_component(FIL_DIR ${FIL} DIRECTORY)
@@ -63,58 +82,58 @@ function(Service_Compiler_Generate_Cpp SRCS HDRS)
       set(FIL_WE "${FIL_DIR}/${FIL_WE}")
     endif()
 
-    list(APPEND ${SRCS} "${CMAKE_CURRENT_BINARY_DIR}/${FIL_WE}.pb.cc")
-    list(APPEND ${HDRS} "${CMAKE_CURRENT_BINARY_DIR}/${FIL_WE}.pb.h")
+    list(APPEND ${_ARGS_CXX_SOURCES} "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.pb.cc")
+    list(APPEND ${_ARGS_CXX_HEADERS} "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.pb.h")
 
-    list(APPEND ${SRCS} "${CMAKE_CURRENT_BINARY_DIR}/${FIL_WE}.client.pb.cc")
-    list(APPEND ${HDRS} "${CMAKE_CURRENT_BINARY_DIR}/${FIL_WE}.client.pb.h")
+    list(APPEND ${_ARGS_CXX_SOURCES} "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.client.pb.cc")
+    list(APPEND ${_ARGS_CXX_HEADERS} "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.client.pb.h")
 
-    list(APPEND ${SRCS} "${CMAKE_CURRENT_BINARY_DIR}/${FIL_WE}.server.pb.cc")
-    list(APPEND ${HDRS} "${CMAKE_CURRENT_BINARY_DIR}/${FIL_WE}.server.pb.h")
+    list(APPEND ${_ARGS_CXX_SOURCES} "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.server.pb.cc")
+    list(APPEND ${_ARGS_CXX_HEADERS} "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.server.pb.h")
 
     set(TEMPLATES
-        ${CMAKE_CURRENT_SOURCE_DIR}/api/service_compiler/templates/cpp/client.pb.cc
-        ${CMAKE_CURRENT_SOURCE_DIR}/api/service_compiler/templates/cpp/client.pb.h
-        ${CMAKE_CURRENT_SOURCE_DIR}/api/service_compiler/templates/cpp/server.pb.cc
-        ${CMAKE_CURRENT_SOURCE_DIR}/api/service_compiler/templates/cpp/server.pb.h
+        ${CMAKE_SOURCE_DIR}/src/lib_spark/spark/api/service_compiler/templates/cpp/client.pb.cc
+        ${CMAKE_SOURCE_DIR}/src/lib_spark/spark/api/service_compiler/templates/cpp/client.pb.h
+        ${CMAKE_SOURCE_DIR}/src/lib_spark/spark/api/service_compiler/templates/cpp/server.pb.cc
+        ${CMAKE_SOURCE_DIR}/src/lib_spark/spark/api/service_compiler/templates/cpp/server.pb.h
     )
 
+    set(_plugin_py ${CMAKE_SOURCE_DIR}/src/lib_spark/spark/api/service_compiler/plugin.py)
     if(WIN32)
+      set(_plugin_cmd ${CMAKE_SOURCE_DIR}/src/lib_spark/spark/api/service_compiler/plugin.cmd)
       add_custom_command(
-        OUTPUT "${CMAKE_CURRENT_BINARY_DIR}/${FIL_WE}.pb.cc"
-               "${CMAKE_CURRENT_BINARY_DIR}/${FIL_WE}.pb.h"
-               "${CMAKE_CURRENT_BINARY_DIR}/${FIL_WE}.client.pb.cc"
-               "${CMAKE_CURRENT_BINARY_DIR}/${FIL_WE}.client.pb.h"
-               "${CMAKE_CURRENT_BINARY_DIR}/${FIL_WE}.server.pb.cc"
-               "${CMAKE_CURRENT_BINARY_DIR}/${FIL_WE}.server.pb.h"
+        OUTPUT "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.pb.cc"
+               "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.pb.h"
+               "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.client.pb.cc"
+               "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.client.pb.h"
+               "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.server.pb.cc"
+               "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.server.pb.h"
         COMMAND SET PYTHONPATH=${SPARK_PYTHON}
         COMMAND ${CMAKE_SOURCE_DIR}/.venv/Scripts/activate.bat
         COMMAND
-          ${Protobuf_PROTOC_EXECUTABLE} ARGS --cpp_out=${CMAKE_BINARY_DIR}/src
-          --plugin=protoc-gen-service=${CMAKE_SOURCE_DIR}/src/lib/api/service_compiler/plugin.cmd
-          --service_out=lang=cpp:${CMAKE_BINARY_DIR}/src -I
-          ${PROTOBUF_INCLUDE_DIRS} -I ${CMAKE_SOURCE_DIR}/src ${ABS_FIL}
-        DEPENDS ${ABS_FIL} ${TEMPLATES}
-                ${CMAKE_CURRENT_SOURCE_DIR}/api/service_compiler/plugin.py
+          ${Protobuf_PROTOC_EXECUTABLE} ARGS --cpp_out=${_ARGS_CXX_OUT_DIR}
+          --plugin=protoc-gen-service=${_plugin_cmd}
+          --service_out=lang=cpp:${_ARGS_CXX_OUT_DIR} -I
+          ${PROTOBUF_INCLUDE_DIRS} -I ${_protobuf_src_dir} ${ABS_FIL}
+        DEPENDS ${ABS_FIL} ${TEMPLATES} ${_plugin_cmd} ${_plugin_py}
         COMMENT "Running C++ service compiler on ${FIL}"
         VERBATIM)
     else()
       add_custom_command(
-        OUTPUT "${CMAKE_CURRENT_BINARY_DIR}/${FIL_WE}.pb.cc"
-               "${CMAKE_CURRENT_BINARY_DIR}/${FIL_WE}.pb.h"
-               "${CMAKE_CURRENT_BINARY_DIR}/${FIL_WE}.client.pb.cc"
-               "${CMAKE_CURRENT_BINARY_DIR}/${FIL_WE}.client.pb.h"
-               "${CMAKE_CURRENT_BINARY_DIR}/${FIL_WE}.server.pb.cc"
-               "${CMAKE_CURRENT_BINARY_DIR}/${FIL_WE}.server.pb.h"
+        OUTPUT "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.pb.cc"
+               "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.pb.h"
+               "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.client.pb.cc"
+               "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.client.pb.h"
+               "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.server.pb.cc"
+               "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.server.pb.h"
         COMMAND ${CMAKE_SOURCE_DIR}/tools/venv_exec.sh
         ARGS
           ${Protobuf_PROTOC_EXECUTABLE}
-          --cpp_out=${CMAKE_BINARY_DIR}/src
-          --plugin=protoc-gen-service=${CMAKE_SOURCE_DIR}/src/lib/api/service_compiler/plugin.py
-          --service_out=lang=cpp,make_file_name=${ABS_FIL}:${CMAKE_BINARY_DIR}/src
-          -I ${PROTOBUF_INCLUDE_DIRS} -I ${CMAKE_SOURCE_DIR}/src ${ABS_FIL}
-        DEPENDS ${ABS_FIL} ${TEMPLATES}
-                ${CMAKE_CURRENT_SOURCE_DIR}/api/service_compiler/plugin.py
+          --cpp_out=${_ARGS_CXX_OUT_DIR}
+          --plugin=protoc-gen-service=${_plugin_py}
+          --service_out=lang=cpp,make_file_name=${ABS_FIL}:${_ARGS_CXX_OUT_DIR}
+          ${_proto_include_cli_args} ${ABS_FIL}
+        DEPENDS ${ABS_FIL} ${TEMPLATES} ${_plugin_py}
         COMMENT "Running C++ service compiler on ${FIL}"
         VERBATIM)
     endif()
@@ -123,22 +142,22 @@ function(Service_Compiler_Generate_Cpp SRCS HDRS)
 
   # remove some warnings from protobuf generated files
   if(WIN32)
-    set_source_files_properties(${${SRCS}} PROPERTIES COMPILE_FLAGS
+    set_source_files_properties(${${_ARGS_CXX_SOURCES}} PROPERTIES COMPILE_FLAGS
                                                       "/wd4309 /wd4838")
   else()
     set_source_files_properties(
-      ${${SRCS}}
+      ${${_ARGS_CXX_SOURCES}}
       PROPERTIES
         COMPILE_FLAGS
         "-Wno-unused-parameter -Wno-effc++ -Wno-pedantic -Wno-sign-compare -Wno-uninitialized"
     )
   endif()
 
-  set(${SRCS}
-      ${${SRCS}}
+  set(${_ARGS_CXX_SOURCES}
+      ${${_ARGS_CXX_SOURCES}}
       PARENT_SCOPE)
-  set(${HDRS}
-      ${${HDRS}}
+  set(${_ARGS_CXX_HEADERS}
+      ${${_ARGS_CXX_HEADERS}}
       PARENT_SCOPE)
 
 endfunction()

cmake/ProtobufGenerateCpp.cmake

@@ -0,0 +1,83 @@
+include_guard(GLOBAL)
+
+function(protobuf_generate_cpp)
+  set(_options)
+  set(_one_value_args
+    CXX_SOURCES
+    CXX_HEADERS
+    CXX_OUT_DIR
+    PROTOC
+  )
+  set(_multi_value_args
+    PROTO_INCLUDE_DIRS
+    PROTO_SOURCES
+  )
+  cmake_parse_arguments(_ARGS "${_options}" "${_one_value_args}" "${_multi_value_args}" ${ARGN})
+
+  if (NOT DEFINED _ARGS_PROTOC)
+    if (WIN32)
+      set(_ARGS_PROTOC "${EXTERNALS_BIN}/protoc.exe")
+    else()
+      set(_ARGS_PROTOC "${EXTERNALS_BIN}/protoc")
+    endif()
+  endif()
+  set(${_ARGS_CXX_SOURCES})
+  set(${_ARGS_CXX_HEADERS})
+
+  set(_proto_include_cli_args)
+  foreach(_dir ${_ARGS_PROTO_INCLUDE_DIRS})
+    list(APPEND _proto_include_cli_args -I ${_dir})
+  endforeach()
+
+  foreach(_fil ${_ARGS_PROTO_SOURCES})
+    get_filename_component(_abs_fil ${_fil} ABSOLUTE)
+    get_filename_component(_fil_we ${_fil} NAME_WE)
+    get_filename_component(_fil_dir ${_fil} DIRECTORY)
+    if(_fil_dir)
+      set(_fil_we "${_fil_dir}/${_fil_we}")
+    endif()
+    list(APPEND ${_ARGS_CXX_SOURCES} "${_ARGS_CXX_OUT_DIR}/${_fil_we}.pb.cc")
+    list(APPEND ${_ARGS_CXX_HEADERS} "${_ARGS_CXX_OUT_DIR}/${_fil_we}.pb.h")
+    message(STATUS "add_custom_command for ${_fil} -> ${_ARGS_CXX_OUT_DIR}/${_fil_we}.pb.h")
+    # TODO: Pass proto sources by relative path (so this fn can be called from
+    # arbitrary subdirs).
+    add_custom_command(
+      OUTPUT
+        "${_ARGS_CXX_OUT_DIR}/${_fil_we}.pb.cc"
+        "${_ARGS_CXX_OUT_DIR}/${_fil_we}.pb.h"
+      COMMAND
+        ${_ARGS_PROTOC}
+      ARGS
+        --cpp_out=${_ARGS_CXX_OUT_DIR} ${_proto_include_cli_args} ${_abs_fil}
+      DEPENDS ${_abs_fil}
+      COMMENT "Running C++ protocol buffer compiler on ${_fil}"
+      VERBATIM
+    )
+  endforeach()
+
+  set_source_files_properties(
+    ${${_ARGS_CXX_SOURCES}}
+    ${${_ARGS_CXX_HEADERS}}
+    PROPERTIES
+      GENERATED TRUE
+  )
+
+  # remove some warnings from protobuf generated files
+  if(WIN32)
+    set_source_files_properties(
+      ${${_ARGS_CXX_SOURCES}}
+      PROPERTIES
+        COMPILE_FLAGS "/wd4309 /wd4838"
+    )
+  else()
+    set_source_files_properties(
+      ${${_ARGS_CXX_SOURCES}}
+      PROPERTIES
+        COMPILE_FLAGS "-Wno-unused-parameter -Wno-effc++ -Wno-pedantic -Wno-sign-compare -Wno-uninitialized"
+    )
+  endif()
+
+  set(${_ARGS_CXX_SOURCES} ${${_ARGS_CXX_SOURCES}} PARENT_SCOPE)
+  set(${_ARGS_CXX_HEADERS} ${${_ARGS_CXX_HEADERS}} PARENT_SCOPE)
+endfunction()
+

cmake/ProtobufGenerateCppConfig.cmake

@@ -0,0 +1,61 @@
+include_guard(GLOBAL)
+
+function(protobuf_generate_cpp_config)
+  set(_options)
+  set(_one_value_args
+    PROTO_SOURCES_HEADER
+    PROTO_DEFAULTS_HEADER
+  )
+  set(_multi_value_args
+    PROTO_SOURCES
+    PROTO_DEFAULTS
+  )
+  cmake_parse_arguments(_ARGS "${_options}" "${_one_value_args}" "${_multi_value_args}" ${ARGN})
+
+  # Configure the 'config_proto_files' header. It provides an array of strings populated with the
+  # files in PROTO_SOURCES. At runtime, we use this array to generate a special 'config' proto
+  # message with fields for each message defined in said files. A global instance of this 'config'
+  # proto is then constructed, and populated with [compile-time] defaults, and [runtime] overrides
+  # (i.e., in spark.conf). Calls to `bts::config::get<T>()` retrieve the value from the field
+  # associated with type `T` in this global instance.
+  message(STATUS "PROTO_SOURCES: ${_ARGS_PROTO_SOURCES}")
+  message(STATUS "PROTO_SOURCES_HEADER: ${_ARGS_PROTO_SOURCES_HEADER}")
+  message(STATUS "PROTO_DEFAULTS_HEADER: ${_ARGS_PROTO_DEFAULTS_HEADER}")
+  if (_ARGS_PROTO_SOURCES_HEADER)
+    set(PROTO_SOURCES)
+    set(PROTO_SOURCES_CXX_ARRAY_ITEMS "")
+    foreach(_proto_path ${_ARGS_PROTO_SOURCES})
+      list(APPEND PROTO_SOURCES ${_proto_path})
+      string(APPEND PROTO_SOURCES_CXX_ARRAY_ITEMS "\"${_proto_path}\",\n")
+    endforeach()
+    list(LENGTH PROTO_SOURCES PROTO_SOURCES_LENGTH)
+    message(STATUS "CONFIGURING PROTO_SOURCES_HEADER): ${_ARGS_PROTO_SOURCES_HEADER}")
+    configure_file(${CMAKE_SOURCE_DIR}/cmake/config_proto_files.h.in ${_ARGS_PROTO_SOURCES_HEADER})
+  endif()
+
+  # Generate static defaults. Ideally, this would be a single string, but we use an array because
+  # the compiler complains if the string is longer than 65k.
+  set(_cfg_defaults_str "inline const char * config_static_defaults[] = { ")
+
+  foreach(_cfg_file_path ${_ARGS_PROTO_DEFAULTS})
+    file(READ ${_cfg_file_path} _cfg_file_content)
+    string(REPLACE "\"" "\\\"" _cfg_file_content_escaped ${_cfg_file_content})
+    string(REPLACE "\n" ";" _cfg_lines ${_cfg_file_content_escaped})
+
+    foreach(_line ${_cfg_lines})
+      string(APPEND _cfg_defaults_str "\n  \"${_line}\",")
+    endforeach(_line)
+  endforeach(_cfg_file_path)
+
+  string(APPEND _cfg_defaults_str "};")
+  set(_old_cfg_defaults_str "")
+
+  if(EXISTS ${_ARGS_PROTO_DEFAULTS_HEADER})
+    file(READ ${_ARGS_PROTO_DEFAULTS_HEADER} _old_cfg_defaults_str)
+  endif()
+
+  if(NOT _cfg_defaults_str STREQUAL _old_cfg_defaults_str)
+    file(WRITE "${_ARGS_PROTO_DEFAULTS_HEADER}" "${_cfg_defaults_str}")
+  endif()
+endfunction()
+

cmake/ProtobufGenerateFs.cmake

@@ -10,7 +10,7 @@ function(protobuf_generate_fs)
     set(_ARGS_TARGET_NAME "${PROJECT_NAME}_generate_fs_protos")
   endif()
   if (NOT DEFINED _ARGS_PROTOC)
-    set(_ARGS_PROTOC "${SPARK_BIN}/protoc")
+    set(_ARGS_PROTOC "${EXTERNALS_BIN}/protoc")
   endif()
 
   set(_proto_include_cli_args)
@@ -27,8 +27,8 @@ function(protobuf_generate_fs)
   endforeach()
 
   set(_venv_exec ${CMAKE_SOURCE_DIR}/tools/venv_exec.sh)
-  set(_plugin_py ${CMAKE_SOURCE_DIR}/src/lib/api/service_compiler/plugin.py)
-  set(_template_fs ${CMAKE_SOURCE_DIR}/src/lib/api/service_compiler/templates/fs/fable.fs)
+  set(_plugin_py ${CMAKE_SOURCE_DIR}/src/lib_spark/spark/api/service_compiler/plugin.py)
+  set(_template_fs ${CMAKE_SOURCE_DIR}/src/lib_spark/spark/api/service_compiler/templates/fs/fable.fs)
   add_custom_command(
     OUTPUT ${_outputs}
     WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}

cmake/SetFirstDefined.cmake

@@ -0,0 +1,10 @@
+include_guard(GLOBAL)
+
+macro(set_first_defined RESULT)
+  foreach(_arg ${ARGN})
+    if (DEFINED ${_arg})
+      set(${RESULT} ${${_arg}})
+      break()
+    endif()
+  endforeach()
+endmacro()

cmake/config_proto_files.h.in

@@ -0,0 +1,11 @@
+#pragma once
+
+#include <array>
+
+namespace bts::@PROJECT_NAME@::protobufs {
+
+inline std::array<const char *, @PROTO_SOURCES_LENGTH@> config_proto_files = {
+@PROTO_SOURCES_CXX_ARRAY_ITEMS@
+};
+
+} // namespace bts::@PROJECT_NAME@::protobufs

cmake/vs_filters.cmake

@@ -1,24 +0,0 @@
-cmake_minimum_required(VERSION 3.5)
-
-# Explicitly set policy to convert relative paths to absolute (although this is a default policy,
-# setting it explicitly prevents a compile-warning)
-cmake_policy(SET CMP0076 NEW)
-
-# add protos to the library to facilitate source_group
-target_sources(libspark PUBLIC ${Config_Protos} ${NonConfig_Protos} ${Service_Protos})
-
-function(assign_source_group)
-  foreach(_source IN ITEMS ${ARGN})
-    if(IS_ABSOLUTE "${_source}")
-      file(RELATIVE_PATH _source_rel "${CMAKE_CURRENT_SOURCE_DIR}" "${_source}")
-    else()
-      set(_source_rel "${_source}")
-    endif()
-    get_filename_component(_source_path "${_source_rel}" PATH)
-    string(REPLACE "/" "\\" _source_path_msvc "${_source_path}")
-    source_group("${_source_path_msvc}" FILES "${_source}")
-    set(SOURCE_FILES "${_source}")
-  endforeach()
-endfunction(assign_source_group)
-
-assign_source_group(${SparkLib_Sources} ${Config_Protos} ${NonConfig_Protos} ${Service_Protos})

compose.yaml

@@ -3,7 +3,7 @@ services:
     volumes:
       - ${SPARKD__DATA_DIR:-sparkd_data}:/home/bts/sparkdata/data
     build:
-      dockerfile: src/server/sparkd/Dockerfile
+      dockerfile: src/app_sparkd/Dockerfile
       args:
         - IMAGE=${SPARKD__IMAGE}
         - SOURCE_DIR=${SPARKD__SOURCE_DIR}
@@ -34,6 +34,47 @@ services:
         limits:
           memory: "${SPARKD__MEMORY:-4gb}"
 
+  sparkd-symbols:
+    volumes:
+      - ${SPARKD__DATA_DIR:-sparkd_data}:/home/bts/sparkdata/data
+      - ${SPARKD__RUNTIME_CONF_DIR:-sparkd_conf}:/home/bts/sparkdata/conf
+    build:
+      dockerfile: src/server/sparkd/Dockerfile
+      args:
+        - IMAGE=${SPARKD__IMAGE}
+        - SOURCE_DIR=${SPARKD__SOURCE_DIR}
+        - CONFIG_DIR=${SPARKD__CONFIG_DIR}
+    shm_size: "${SPARKD__SHM_SIZE:-512mb}"
+    command: >
+      sh -c "
+        echo bts | sudo -S chown -R bts:bts /home/bts/sparkdata
+        if [ -e "/home/bts/sparkdata/data/logging.sparkd/log" ]; then
+          mv "/home/bts/sparkdata/data/logging.sparkd/log" "/home/bts/sparkdata/data/logging.sparkd/log_$(date '+%Y%m%d-%H%M')"
+        fi
+        ls -alh /home/bts/sparkdata/conf/
+        cat /home/bts/sparkdata/conf/.env.local
+        if [ -e "/home/bts/sparkdata/conf/.env.local" ]; then
+          echo 'Sourcing /home/bts/sparkdata/conf/.env.local'
+          set -a
+          . /home/bts/sparkdata/conf/.env.local
+          set +a
+        fi
+        ./sparkd run"
+    env_file:
+      - .env
+      - ${SPARKD__CONFIG_DIR}/.env
+    ports:
+      - "${SPARKD__CONFIG_PORT}:${SPARKD__CONFIG_PORT}"
+      - "${SPARKD__QUICKBUS_PORT}:${SPARKD__QUICKBUS_PORT}"
+      - "${SPARKD__SIM_FIX_PORT}:${SPARKD__SIM_FIX_PORT}"
+    deploy:
+      restart_policy:
+        condition: any
+        delay: 3s
+      resources:
+        limits:
+          memory: "${SPARKD__MEMORY:-4gb}"
+
   sparkd-tail:
     image: bluetradingsystems/linux_dotnet7
     volumes:

doc/api/content/client/getting-started/services.md

@@ -124,8 +124,8 @@ service OrderService {
 ```
 ### C++ Example
 ```cpp
-#include "lib/api/public/ServiceContext.h"
-#include "lib/api/services/OrderService.client.pb.h"
+#include "spark/api/public/ServiceContext.h"
+#include "spark/api/services/OrderService.client.pb.h"
 using namespace bts::api;
 ServiceContext ctx;
 auto orders = ctx.locate<orderman::OrderService>();

doc/api/content/client/services/marketdata.md

@@ -29,9 +29,9 @@ namespace sample {
 ```
 
 ```cpp
-#include "lib/api/public/ServiceContext.h"
-#include "spark/lib/domain_model/Security.h"
-#include "lib/api/services/MarketDataService.client.pb.h"
+#include "spark/api/public/ServiceContext.h"
+#include "spark/domain_model/Security.h"
+#include "spark/api/services/MarketDataService.client.pb.h"
 #include <thread>
 
 using namespace bts::api;

src/DataStreamSuite/ArithmeticExpressions/ArithmeticExpressions.csproj

@@ -1,7 +1,7 @@
 <Project Sdk="Microsoft.NET.Sdk">
 
   <PropertyGroup>
-    <TargetFramework>net7.0</TargetFramework>
+    <TargetFramework>net8.0</TargetFramework>
     <ImplicitUsings>enable</ImplicitUsings>
     <Nullable>enable</Nullable>
   </PropertyGroup>

src/DataStreamSuite/BaseUtilities/BaseUtilities.csproj

@@ -1,7 +1,7 @@
 <Project Sdk="Microsoft.NET.Sdk">
 
   <PropertyGroup>
-    <TargetFramework>net7.0</TargetFramework>
+    <TargetFramework>net8.0</TargetFramework>
     <ImplicitUsings>enable</ImplicitUsings>
     <Nullable>enable</Nullable>
   </PropertyGroup>

src/DataStreamSuite/BlueRegistry/BlueRegistry.csproj

@@ -1,7 +1,7 @@
 <Project Sdk="Microsoft.NET.Sdk">
 
   <PropertyGroup>
-    <TargetFramework>net7.0</TargetFramework>
+    <TargetFramework>net8.0</TargetFramework>
     <ImplicitUsings>enable</ImplicitUsings>
     <Nullable>enable</Nullable>
   </PropertyGroup>

src/DataStreamSuite/BooleanExpressions/BooleanExpressions.csproj

@@ -1,14 +1,14 @@
 <Project Sdk="Microsoft.NET.Sdk">
 
   <PropertyGroup>
-    <TargetFramework>net7.0</TargetFramework>
+    <TargetFramework>net8.0</TargetFramework>
     <ImplicitUsings>enable</ImplicitUsings>
     <Nullable>enable</Nullable>
   </PropertyGroup>
 
   <ItemGroup>
     <PackageReference Include="Google.Protobuf" Version="3.24.3" />
-    <PackageReference Include="YaccLexTools" Version="1.2.0" />
+    <PackageReference Include="YaccLexTools" Version="1.2.3" />
   </ItemGroup>
 
   <ItemGroup>

src/DataStreamSuite/BooleanExpressions/GplexBuffers.cs

@@ -24,8 +24,6 @@ namespace QUT.GplexBuffers
         public BufferException(string message) : base(message) { }
         public BufferException(string message, Exception innerException)
             : base(message, innerException) { }
-        protected BufferException(SerializationInfo info, StreamingContext context)
-            : base(info, context) { }
     }
 
     internal abstract class ScanBuff
@@ -60,12 +58,10 @@ namespace QUT.GplexBuffers
             return new BuildBuffer(source);
         }
 
-#if (!BYTEMODE)
         public static ScanBuff GetBuffer(Stream source, int fallbackCodePage)
         {
             return new BuildBuffer(source, fallbackCodePage);
         }
-#endif // !BYTEMODE
 #endif // !NOFILES
     }
 
@@ -353,17 +349,15 @@ namespace QUT.GplexBuffers
         {
             FileStream fStrm = (stream as FileStream);
             if (fStrm != null) FileName = fStrm.Name;
-            NextBlk = BlockReaderFactory.Raw(stream);
+            NextBlk = BlockReaderFactory.Get(stream);
         }
 
-#if (!BYTEMODE)
         public BuildBuffer(Stream stream, int fallbackCodePage)
         {
             FileStream fStrm = (stream as FileStream);
             if (fStrm != null) FileName = fStrm.Name;
             NextBlk = BlockReaderFactory.Get(stream, fallbackCodePage);
         }
-#endif
 
         /// <summary>
         /// Marks a conservative lower bound for the buffer,
@@ -434,43 +428,19 @@ namespace QUT.GplexBuffers
     //
     internal static class BlockReaderFactory
     {
-        public static BlockReader Raw(Stream stream)
-        {
-            return delegate(char[] block, int index, int number)
-            {
-                byte[] b = new byte[number];
-                int count = stream.Read(b, 0, number);
-                int i = 0;
-                int j = index;
-                for (; i < count; i++, j++)
-                    block[j] = (char)b[i];
-                return count;
-            };
-        }
 
-#if (!BYTEMODE)
-        public static BlockReader Get(Stream stream, int fallbackCodePage)
+        public static BlockReader Get(Stream stream, int fallbackCodePage = -2)
         {
             Encoding encoding;
             int preamble = Preamble(stream);
 
             if (preamble != 0)  // There is a valid BOM here!
                 encoding = Encoding.GetEncoding(preamble);
-            else if (fallbackCodePage == -1) // Fallback is "raw" bytes
-                return Raw(stream);
-            else if (fallbackCodePage != -2) // Anything but "guess"
+            else if (fallbackCodePage >= 0) // Anything but "guess"
                 encoding = Encoding.GetEncoding(fallbackCodePage);
             else // This is the "guess" option
-            {
-                int guess = new Guesser(stream).GuessCodePage();
-                stream.Seek(0, SeekOrigin.Begin);
-                if (guess == -1) // ==> this is a 7-bit file
-                    encoding = Encoding.ASCII;
-                else if (guess == 65001)
-                    encoding = Encoding.UTF8;
-                else             // ==> use the machine default
-                    encoding = Encoding.Default;
-            }
+                encoding = GuessEncoding(stream);
+
             StreamReader reader = new StreamReader(stream, encoding);
             return reader.Read;
         }
@@ -488,6 +458,14 @@ namespace QUT.GplexBuffers
             int b2 = stream.ReadByte();
             if (b0 == 0xef && b1 == 0xbb && b2 == 0xbf)
                 return 65001; // UTF8
+
+            int b3 = stream.ReadByte();
+            // Check for UTF-32 BOM (Byte Order Mark)
+            if (b0 == 0x00 && b1 == 0x00 && b2 == 0xFE && b3 == 0xFF)
+                return 12001; // UTF-32BE
+            if (b0 == 0xFF && b1 == 0xFE && b2 == 0x00 && b3 == 0x00)
+                return 12000; // UTF-32LE
+
             //
             // There is no unicode preamble, so we
             // return denoter for the machine default.
@@ -495,7 +473,150 @@ namespace QUT.GplexBuffers
             stream.Seek(0, SeekOrigin.Begin);
             return 0;
         }
-#endif // !BYTEMODE
+
+        private static Encoding GuessEncoding(Stream stream)
+        {
+            stream.Seek(0, SeekOrigin.Begin);
+            if (IsUtf16(stream, out Encoding encoding))
+            {
+                stream.Seek(0, SeekOrigin.Begin);
+                return encoding;
+            }
+
+            stream.Seek(0, SeekOrigin.Begin);
+            if (IsUtf8(stream))
+            {
+                stream.Seek(0, SeekOrigin.Begin);
+                return Encoding.UTF8;
+            }
+
+            stream.Seek(0, SeekOrigin.Begin);
+            if (IsAscii(stream))
+            {
+                stream.Seek(0, SeekOrigin.Begin);
+                return Encoding.ASCII;
+            }
+
+#if !NET462_OR_GREATER
+            stream.Seek(0, SeekOrigin.Begin);
+            if (IsLatin1(stream))
+            {
+                stream.Seek(0, SeekOrigin.Begin);
+                return Encoding.Latin1;
+            }
+#endif
+            stream.Seek(0, SeekOrigin.Begin);
+            return Encoding.Default;
+        }
+
+        private static bool IsUtf8(Stream stream)
+        {
+            int b;
+            while ((b = stream.ReadByte()) != -1)
+            {
+                if ((b & 0x80) == 0) // 0xxxxxxx
+                {
+                    continue;
+                }
+                else if ((b & 0xE0) == 0xC0) // 110xxxxx 10xxxxxx
+                {
+                    if (!IsValidUtf8ContinuationByte(stream)) return false;
+                }
+                else if ((b & 0xF0) == 0xE0) // 1110xxxx 10xxxxxx 10xxxxxx
+                {
+                    if (!IsValidUtf8ContinuationByte(stream) || !IsValidUtf8ContinuationByte(stream)) return false;
+                }
+                else if ((b & 0xF8) == 0xF0) // 11110xxx 10xxxxxx 10xxxxxx 10xxxxxx
+                {
+                    if (!IsValidUtf8ContinuationByte(stream) || !IsValidUtf8ContinuationByte(stream) || !IsValidUtf8ContinuationByte(stream)) return false;
+                }
+                else
+                {
+                    return false;
+                }
+            }
+            return true;
+        }
+
+        private static bool IsValidUtf8ContinuationByte(Stream stream)
+        {
+            int b = stream.ReadByte();
+            return b != -1 && (b & 0xC0) == 0x80; // 10xxxxxx
+        }
+
+        private static bool IsUtf16(Stream stream, out Encoding encoding)
+        {
+            encoding = Encoding.Default;
+
+            if (stream.Length >= 2)
+            {
+                // Read the first two bytes
+                int b0 = stream.ReadByte();
+                int b1 = stream.ReadByte();
+
+                // Check for UTF-16 BOM (Byte Order Mark)
+                if (b0 == 0xFF && b1 == 0xFE)
+                {
+                    encoding = Encoding.Unicode;
+                    return true; // UTF-16LE
+                }
+                if (b0 == 0xFE && b1 == 0xFF)
+                {
+                    encoding = Encoding.BigEndianUnicode;
+                    return true; // UTF-16BE
+                }
+
+                // Reset the stream position
+                stream.Seek(0, SeekOrigin.Begin);
+
+                // Check for common UTF-16 patterns
+                byte[] buffer = new byte[2];
+                while (stream.Read(buffer, 0, 2) == 2)
+                {
+                    if (buffer[0] == 0x00 && buffer[1] != 0x00)
+                    {
+                        encoding = Encoding.BigEndianUnicode;
+                        return true; // Likely UTF-16BE
+                    }
+                    if (buffer[0] != 0x00 && buffer[1] == 0x00)
+                    {
+                        encoding = Encoding.Unicode;
+                        return true; // Likely UTF-16LE
+                    }
+                }
+            }
+
+            return false;
+        }
+
+        private static bool IsAscii(Stream stream)
+        {
+            int b;
+            while ((b = stream.ReadByte()) != -1)
+            {
+                // Latin1 includes byte values from 0x00 to 0xFF
+                if (b > 0x7F)
+                {
+                    return false;
+                }
+            }
+            return true;
+        }
+
+        private static bool IsLatin1(Stream stream)
+        {
+            int b;
+            while ((b = stream.ReadByte()) != -1)
+            {
+                // Latin1 includes byte values from 0x00 to 0xFF
+                if (b > 0xFF)
+                {
+                    return false;
+                }
+            }
+            return true;
+        }
+
     }
 #endif // !NOFILES
     #endregion Buffer classes
@@ -540,180 +661,6 @@ namespace QUT.GplexBuffers
             return 0;
         }
     }
-#region guesser
-#if (!BYTEMODE)
-    // ==============================================================
-    // ============          Encoding Guesser           =============
-    // ==============================================================
-
-    /// <summary>
-    /// This class provides a simple finite state automaton that
-    /// scans the file looking for (1) valid UTF-8 byte patterns,
-    /// (2) bytes >= 0x80 which are not part of a UTF-8 sequence.
-    /// The method then guesses whether it is UTF-8 or maybe some 
-    /// local machine default encoding.  This works well for the
-    /// various Latin encodings.
-    /// </summary>
-    internal class Guesser
-    {
-        ScanBuff buffer;
-
-        public int GuessCodePage() { return Scan(); }
-
-        const int maxAccept = 10;
-        const int initial = 0;
-        const int eofNum = 0;
-        const int goStart = -1;
-        const int INITIAL = 0;
-        const int EndToken = 0;
-
-        #region user code
-        /* 
-         *  Reads the bytes of a file to determine if it is 
-         *  UTF-8 or a single-byte code page file.
-         */
-        public long utfX;
-        public long uppr;
-        #endregion user code
-
-        int state;
-        int currentStart = startState[0];
-        int code;
-
-        #region ScannerTables
-        static int[] startState = new int[] { 11, 0 };
-
-        #region CharacterMap
-        static sbyte[] map = new sbyte[256] {
-/*     '\0' */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 
-/*   '\x10' */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 
-/*   '\x20' */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 
-/*      '0' */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 
-/*      '@' */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 
-/*      'P' */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 
-/*      '`' */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 
-/*      'p' */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 
-/*   '\x80' */ 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 
-/*   '\x90' */ 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 
-/*   '\xA0' */ 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 
-/*   '\xB0' */ 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 
-/*   '\xC0' */ 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 
-/*   '\xD0' */ 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 
-/*   '\xE0' */ 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 
-/*   '\xF0' */ 4, 4, 4, 4, 4, 4, 4, 4, 5, 5, 5, 5, 5, 5, 5, 5 };
-        #endregion
-
-        static sbyte[][] nextState = new sbyte[][] {
-            new sbyte[] {0, 0, 0, 0, 0, 0},
-            new sbyte[] {-1, -1, 10, -1, -1, -1},
-            new sbyte[] {-1, -1, -1, -1, -1, -1},
-            new sbyte[] {-1, -1, 8, -1, -1, -1},
-            new sbyte[] {-1, -1, 5, -1, -1, -1},
-            new sbyte[] {-1, -1, 6, -1, -1, -1},
-            new sbyte[] {-1, -1, 7, -1, -1, -1},
-            null,
-            new sbyte[] {-1, -1, 9, -1, -1, -1},
-            null,
-            null,
-            new sbyte[] {-1, 1, 2, 3, 4, 2}
-        };
-
-
-        [SuppressMessage("Microsoft.Performance", "CA1810:InitializeReferenceTypeStaticFieldsInline")]
-        // Reason for suppression: cannot have self-reference in array initializer.
-        static Guesser()
-        {
-            nextState[7] = nextState[2];
-            nextState[9] = nextState[2];
-            nextState[10] = nextState[2];
-        }
-
-        int NextState()
-        {
-            if (code == ScanBuff.EndOfFile)
-                return eofNum;
-            else
-                return nextState[state][map[code]];
-        }
-        #endregion
-
-        public Guesser(System.IO.Stream file) { SetSource(file); }
-
-        public void SetSource(System.IO.Stream source)
-        {
-            this.buffer = new BuildBuffer(source);
-            code = buffer.Read();
-        }
-
-        int Scan()
-        {
-            for (; ; )
-            {
-                int next;
-                state = currentStart;
-                while ((next = NextState()) == goStart)
-                    code = buffer.Read();
-
-                state = next;
-                code = buffer.Read();
-
-                while ((next = NextState()) > eofNum)
-                {
-                    state = next;
-                    code = buffer.Read();
-                }
-                if (state <= maxAccept)
-                {
-                    #region ActionSwitch
-#pragma warning disable 162
-                    switch (state)
-                    {
-                        case eofNum:
-                            switch (currentStart)
-                            {
-                                case 11:
-                                    if (utfX == 0 && uppr == 0) return -1; /* raw ascii */
-                                    else if (uppr * 10 > utfX) return 0;   /* default code page */
-                                    else return 65001;                     /* UTF-8 encoding */
-                                    break;
-                            }
-                            return EndToken;
-                        case 1: // Recognized '{Upper128}',	Shortest string "\xC0"
-                        case 2: // Recognized '{Upper128}',	Shortest string "\x80"
-                        case 3: // Recognized '{Upper128}',	Shortest string "\xE0"
-                        case 4: // Recognized '{Upper128}',	Shortest string "\xF0"
-                            uppr++;
-                            break;
-                        case 5: // Recognized '{Utf8pfx4}{Utf8cont}',	Shortest string "\xF0\x80"
-                            uppr += 2;
-                            break;
-                        case 6: // Recognized '{Utf8pfx4}{Utf8cont}{2}',	Shortest string "\xF0\x80\x80"
-                            uppr += 3;
-                            break;
-                        case 7: // Recognized '{Utf8pfx4}{Utf8cont}{3}',	Shortest string "\xF0\x80\x80\x80"
-                            utfX += 3;
-                            break;
-                        case 8: // Recognized '{Utf8pfx3}{Utf8cont}',	Shortest string "\xE0\x80"
-                            uppr += 2;
-                            break;
-                        case 9: // Recognized '{Utf8pfx3}{Utf8cont}{2}',	Shortest string "\xE0\x80\x80"
-                            utfX += 2;
-                            break;
-                        case 10: // Recognized '{Utf8pfx2}{Utf8cont}',	Shortest string "\xC0\x80"
-                            utfX++;
-                            break;
-                        default:
-                            break;
-                    }
-#pragma warning restore 162
-                    #endregion
-                }
-            }
-        }
-    } // end class Guesser
-    
-#endif // !BYTEMODE
-#endregion
 #endif // !NOFILES
 
 // End of code copied from embedded resource

src/DataStreamSuite/Bts.ColumnExpressions/Bts.ColumnExpressions.csproj

@@ -1,7 +1,7 @@
 <Project Sdk="Microsoft.NET.Sdk">
 
   <PropertyGroup>
-    <TargetFramework>net7.0</TargetFramework>
+    <TargetFramework>net8.0</TargetFramework>
     <ImplicitUsings>enable</ImplicitUsings>
     <Nullable>enable</Nullable>
   </PropertyGroup>

src/DataStreamSuite/Bts.ColumnSortExpressions/Bts.ColumnSortExpressions.csproj

@@ -1,7 +1,7 @@
 <Project Sdk="Microsoft.NET.Sdk">
 
   <PropertyGroup>
-    <TargetFramework>net7.0</TargetFramework>
+    <TargetFramework>net8.0</TargetFramework>
     <ImplicitUsings>enable</ImplicitUsings>
     <Nullable>enable</Nullable>
   </PropertyGroup>

src/DataStreamSuite/ColumnMapExpressions/ColumnMapExpressions.csproj

@@ -1,7 +1,7 @@
 <Project Sdk="Microsoft.NET.Sdk">
 
   <PropertyGroup>
-    <TargetFramework>net7.0</TargetFramework>
+    <TargetFramework>net8.0</TargetFramework>
     <ImplicitUsings>enable</ImplicitUsings>
     <Nullable>enable</Nullable>
   </PropertyGroup>

src/DataStreamSuite/DataStream/DataStream.csproj

@@ -1,7 +1,7 @@
 <Project Sdk="Microsoft.NET.Sdk">
 
   <PropertyGroup>
-    <TargetFramework>net7.0</TargetFramework>
+    <TargetFramework>net8.0</TargetFramework>
     <Nullable>enable</Nullable>
   </PropertyGroup>
 
@@ -23,6 +23,7 @@
     <PackageReference Include="MongoDB.Driver" Version="2.23.1" />
     <PackageReference Include="Parquet.Net" Version="5.0.1" />
     <PackageReference Include="System.Data.SQLite" Version="1.0.118" />
+    <PackageReference Include="MathNet.Numerics" Version="5.0.0" />
   </ItemGroup>
 
 </Project>

src/DataStreamSuite/DataStream/Primitives/GroupBy/GroupByBundle.cs

@@ -4,42 +4,46 @@ namespace bts.SilverPine {
    using System;
    using System.Collections.Generic;
    using System.Linq;
+   //using MathNet.Numerics.Statistics;
 
-   using ZstdSharp.Unsafe;
 
-   public enum GroupByBundleOp {
+   public enum MultiRowOp {
       Mean,
       Median,
       Mode,
+      Range,
       StdDev,
       Max,
       Min,
       First,
       Last,
-      Sum
+      Sum,
+      EdgeDiff,
+      EdgeDiffPercentage,
+      //IsEdgeDiff,
    }
-   public class GroupByBundle {
-      public readonly List<Tuple<string, GroupByBundleOp>> operations;
+   public class MultiRowOpHelper {
+      public readonly List<Tuple<string, MultiRowOp>> operations;
 
-      public static GroupByBundle parse(string text) {
+      public static MultiRowOpHelper parse(string text) {
          // text is inp_col: op, inp_col: op, inp_col:op
-         List<Tuple<string, GroupByBundleOp>> rv = new List<Tuple<string, GroupByBundleOp>>();
+         List<Tuple<string, MultiRowOp>> rv = new List<Tuple<string, MultiRowOp>>();
          string[] calcs = text.Split(',');
          foreach (string c in calcs) {
             string[] toks = c.Split(':');
             string inp_col = toks[0].Trim();
             string op_txt = toks[1].Trim();
-            GroupByBundleOp op = Enum.Parse<GroupByBundleOp>(op_txt, true);
+            MultiRowOp op = Enum.Parse<MultiRowOp>(op_txt, true);
             rv.Add(Tuple.Create(inp_col, op));
          }
-         return new GroupByBundle(rv);
+         return new MultiRowOpHelper(rv);
       }
 
-      public GroupByBundle(List<Tuple<string, GroupByBundleOp>> operations_) {
+      public MultiRowOpHelper(List<Tuple<string, MultiRowOp>> operations_) {
          operations = operations_;
       }
 
-      public static IBlueRow group_by(List<IBlueRow> rows, GroupByBundle parameters) {
+      public static IBlueRow group_by(List<IBlueRow> rows, MultiRowOpHelper parameters) {
          if (rows.Count() == 0) {
             return null;
          }
@@ -47,8 +51,8 @@ namespace bts.SilverPine {
          IBlueRow ir = rows.ElementAt(0);
          IBlueTableSchema in_schema = ir.schema;
          IBlueTableSchema out_schema = in_schema;
-         IDictionary<string, Tuple<string, GroupByBundleOp>> out_col_map =
-            new Dictionary<string, Tuple<string, GroupByBundleOp>>();
+         IDictionary<string, Tuple<string, MultiRowOp>> out_col_map =
+            new Dictionary<string, Tuple<string, MultiRowOp>>();
          foreach (var t in parameters.operations) {
             string out_col_name = $"{t.Item1}_{t.Item2}".ToLower();
             out_schema = out_schema.add_column(TypeCode.Double, out_col_name);
@@ -58,7 +62,7 @@ namespace bts.SilverPine {
          foreach (var kv in out_col_map) {
             string out_col_name = kv.Key;
             string in_col_name = kv.Value.Item1;
-            GroupByBundleOp op = kv.Value.Item2;
+            MultiRowOp op = kv.Value.Item2;
             IEnumerable<double> col_vals = BlueDataUtility.get_colvalues<double>(rows, in_col_name);
             double comp_res = compute_op(col_vals, op);
             or.setValue(out_col_name, comp_res);
@@ -66,20 +70,51 @@ namespace bts.SilverPine {
          return or;
       }
 
-      public static double compute_op(IEnumerable<double> col_vals, GroupByBundleOp op) {
+      // rolling_rows are the rolling window of rows. the 0th row is the one thats gettin
+      // calculated. rows 1 - n-1 are its window rows. the schema is based on the operations.
+
+      public IBlueRow rolling_calc(List<IBlueRow> rolling_rows) {
+         if (rolling_rows.Count() == 0) {
+            return null;
+         }
+         IBlueRow ir = rolling_rows.ElementAt(0);
+         IBlueTableSchema in_schema = ir.schema;
+         IBlueTableSchema out_schema = in_schema;
+         IDictionary<string, Tuple<string, MultiRowOp>> out_col_map =
+            new Dictionary<string, Tuple<string, MultiRowOp>>();
+         foreach (var t in operations) {
+            string out_col_name = $"{t.Item1}_{t.Item2}".ToLower();
+            out_schema = out_schema.add_column(TypeCode.Double, out_col_name);
+            out_col_map[out_col_name] = t;
+         }
+         MutableBlueRow or = new MutableBlueRow(out_schema, ir);
+         foreach (var kv in out_col_map) {
+            string out_col_name = kv.Key;
+            string in_col_name = kv.Value.Item1;
+            MultiRowOp op = kv.Value.Item2;
+            IEnumerable<double> col_vals = BlueDataUtility.get_colvalues<double>(rolling_rows, in_col_name);
+            double comp_res = compute_op(col_vals, op);
+            or.setValue(out_col_name, comp_res);
+         }
+         return or;
+      }
+
+      public static double compute_op(IEnumerable<double> col_vals, MultiRowOp op) {
          switch (op) {
-            case GroupByBundleOp.Max:
+            case MultiRowOp.Max:
                return col_vals.Max<double>();
-            case GroupByBundleOp.Min:
+            case MultiRowOp.Min:
                return col_vals.Min<double>();
-            case GroupByBundleOp.Mean:
+            case MultiRowOp.Mean:
                return col_vals.Sum() / col_vals.Count();
-            case GroupByBundleOp.Sum:
+            case MultiRowOp.Sum:
                return col_vals.Sum();
-            case GroupByBundleOp.First:
+            case MultiRowOp.First:
                return col_vals.First();
-            case GroupByBundleOp.Last:
+            case MultiRowOp.Last:
                return col_vals.Last();
+            case MultiRowOp.StdDev:
+               return MathNet.Numerics.Statistics.Statistics.StandardDeviation(col_vals);
             default:
                // TODO
                Log.assert(false);

src/DataStreamSuite/DataStream/Primitives/GroupBy/NonIncremental/AggregationSteam.cs

@@ -0,0 +1,20 @@
+namespace bts.SilverPine {
+   using BCG;
+
+   using System;
+
+   [DataStreamUserRespresentation("AggregationStream")]
+   public class AgregationStream : ViewStream {
+
+      public AgregationStream(ViewStream parent_str, string operation_txt) {
+         ViewStream rs = new NonIncGroupByStream(parent_str, String.Empty, operation_txt);
+         mySetRealSource(rs);
+      }
+
+      public AgregationStream(ViewStream parent_str, string groupby_col, string operation_txt) {
+         ViewStream rs = new NonIncGroupByStream(parent_str, groupby_col, operation_txt);
+         mySetRealSource(rs);
+      }
+   }
+
+}

src/DataStreamSuite/DataStream/Primitives/GroupBy/NonIncremental/NonIncGroupByStream.cs

@@ -8,7 +8,7 @@ using bts;
 using bts.SilverPine;
 
 namespace BCG {
-   [DataStreamUserRespresentation("aggregate")]
+   [DataStreamUserRespresentation("AggregateStream")]
    public class NonIncGroupByStream : ViewStream {
       private class DelNonIncGroupByEvaluator : INonIncGroupByEvaluator {
          Func<string[]> _groupby_cols_func;
@@ -53,15 +53,18 @@ namespace BCG {
                                  Func<List<IBlueRow>, List<IBlueRow>> evaluate_func)
                                  : this(main_stream, new DelNonIncGroupByEvaluator(groupby_cols_func, evaluate_func)) { }
 
-      public NonIncGroupByStream(ViewStream parent_str, string groupby_col, string standard_group_by_text, string optional_sort_col = null) {
-         GroupByBundle std_group_by = GroupByBundle.parse(standard_group_by_text);
+      public NonIncGroupByStream(ViewStream parent_str, string groupby_col, string standard_group_by_text)
+      : this(parent_str, groupby_col, standard_group_by_text, String.Empty) { }
+
+      public NonIncGroupByStream(ViewStream parent_str, string groupby_col, string standard_group_by_text, string optional_sort_col) {
+         MultiRowOpHelper std_group_by = MultiRowOpHelper.parse(standard_group_by_text);
          Func<string[]> groupby_cols_func = () => {
             return String.IsNullOrEmpty(groupby_col) ? null : new string[] { groupby_col };
          };
          Func<List<IBlueRow>, List<IBlueRow>> evaluate_func = (group_rows) => {
             if (!String.IsNullOrEmpty(optional_sort_col))
                group_rows.Sort((r1, r2) => BlueDataUtility.compare_rows(r2, r1, optional_sort_col));
-            IBlueRow rv = GroupByBundle.group_by(group_rows, std_group_by);
+            IBlueRow rv = MultiRowOpHelper.group_by(group_rows, std_group_by);
             return new IBlueRow[] { rv }.ToList();
          };
          ViewStream rs = new NonIncGroupByStream(parent_str, groupby_cols_func, evaluate_func);

src/DataStreamSuite/DataStream/Primitives/Join/NonIncremental/NonIncJoinBookkeeper.cs

@@ -74,7 +74,7 @@ namespace BCG {
       }
 
       public void _on_stream_cb(object dp_obj) {
-         Log.Gen.info($"NonIncJoinBookkeeper._on_stream_cb dp_obj: {dp_obj}");
+         //Log.Gen.info($"NonIncJoinBookkeeper._on_stream_cb dp_obj: {dp_obj}");
          var timer = Stopwatch.StartNew();
          DataPacket dp = (DataPacket)dp_obj;
          _fill_missing_data(dp);

src/DataStreamSuite/DataStream/Primitives/Join/NonIncremental/NonIncJoinStream.cs

@@ -50,7 +50,7 @@ namespace BCG {
          int data_col_no = 1;
          for (int jp = 0; jp < _join_specs.Length; jp++) {
             NonIncJoinBookkeeper s = new NonIncJoinBookkeeper(_join_specs[jp], jp, data_col_no, _row_hasher);
-            Log.Gen.info($"NonIncJoinStream.myOnFirstSubscriber jp: {jp} spec: {_join_specs[jp]}");
+            //Log.Gen.info($"NonIncJoinStream.myOnFirstSubscriber jp: {jp} spec: {_join_specs[jp]}");
             data_col_no += _join_specs[jp].data_col_names.Length;
             s.on_new_data += _stream_callback;
             _book_keeping_data.Add(s);
@@ -71,7 +71,7 @@ namespace BCG {
       }
 
       private void _stream_callback() {
-         Log.Gen.info("NonIncJoinStream._stream_callback received callback");
+         //Log.Gen.info("NonIncJoinStream._stream_callback received callback");
          _is_dirty = true;
          doWork();
       }
@@ -99,7 +99,7 @@ namespace BCG {
 
       private int _n_total_fires = 0, _n_expensive_fires = 0;
       private void _form_and_fire() {
-         Log.Gen.info($"NonIncJoinStream._form_and_fire _is_ready(): {_is_ready()}");
+         //Log.Gen.info($"NonIncJoinStream._form_and_fire _is_ready(): {_is_ready()}");
          if (_is_ready()) {
             _construct_schema();
             var timer = Stopwatch.StartNew();
@@ -126,11 +126,11 @@ namespace BCG {
             _n_total_fires++;
             //if (timer.ElapsedMilliseconds >= 250) {
             _n_expensive_fires++;
-            string log_msg = $"Join and map time for {_book_keeping_data.Count} "
-               + $"tables {timer.Elapsed.TotalMilliseconds} msecs, ratio {_n_expensive_fires}/{_n_total_fires}";
-            Log.Gen.info(log_msg);
+            //string log_msg = $"Join and map time for {_book_keeping_data.Count} "
+            //   + $"tables {timer.Elapsed.TotalMilliseconds} msecs, ratio {_n_expensive_fires}/{_n_total_fires}";
+            //Log.Gen.info(log_msg);
             //}
-            var dp_out = BlueDataUtility.build_dp_inc(dp_output_rows, myLastValue as DataPacket);
+            var dp_out = BlueDataUtility.build_dp_inc(dp_output_rows, myLastValue as DataPacket, true);
             myFireEvent(dp_out);
             //myFireEvent(new DataPacket(dp_output_rows, dp_output_rows, myLastValue == null ? new BlueTable(_output_schema) : (myLastValue as DataPacket).newRows));
          }
@@ -152,7 +152,7 @@ namespace BCG {
       }
 
       public void doWork() {
-         Log.Gen.info($"NonIncJoinStream.doWork _last_fire_time: {_last_fire_time} _is_dirty: {_is_dirty}");
+         //Log.Gen.info($"NonIncJoinStream.doWork _last_fire_time: {_last_fire_time} _is_dirty: {_is_dirty}");
          if (DateTime.Now - _last_fire_time >= _throttle_period && _is_dirty) {
             _is_dirty = false;
             _last_fire_time = DateTime.Now;

src/DataStreamSuite/DataStream/Primitives/Map/GlobalMapStream.cs

@@ -69,4 +69,6 @@ namespace bts {
          myFireEvent(dp_out);
       }
    }
+
+
 }

src/DataStreamSuite/DataStream/Primitives/Map/MapStream.cs

@@ -13,9 +13,8 @@ namespace BCG {
    /// desired row ie newRow = f(row), where f is the mapping fucntion. Note that the id 
    /// is preserved across the rows, ie row["id"] = f(row)["id"]
    /// </summary>
-   [DataStreamUserRespresentation (false)]
-   public class MapStream : MethodDrivenStream
-   {
+   [DataStreamUserRespresentation(false)]
+   public class MapStream : MethodDrivenStream {
       protected IStreamMapper myMapper;
       protected IChangeableStreamMapper myChangeableMapper = null;
       private bool myUseNewMap = Configuration.get<DataStreamConfig>().UseNewMap;

src/DataStreamSuite/DataStream/Primitives/Map/RollingComputationStream

@@ -0,0 +1,14 @@
+namespace bts.SilverPine
+{
+   using BCG;
+
+   [DataStreamUserRespresentation("RollingComputationStream")]
+   public class RollingComputationStream : ViewStream
+   {
+      private ViewStream _parent_str;
+      public RollingComputationStream(ViewStream parent_str, string operation_txt, string roll_column, uint window_sz)
+      {
+
+      }
+   }
+}

src/DataStreamSuite/DataStream/Primitives/Map/RollingComputationStream.cs

@@ -0,0 +1,10 @@
+namespace bts.SilverPine {
+   using BCG;
+
+
+   [DataStreamUserRespresentation("RollingComputationStream")]
+   public class RollingComputationStream : ViewStream {
+      public RollingComputationStream(ViewStream parent_str, string operation_txt, string roll_column, uint window_sz) {
+      }
+   }
+}

src/DataStreamSuite/DataStream/Primitives/Project/New/NewProjectStream.cs

@@ -108,6 +108,7 @@ namespace bts {
 
    }
 
+   [DataStreamUserRespresentation("Project")]
    public class NewProjectStream : ViewStream, IStreamMapper {
 
       private ViewStream _stream;

src/DataStreamSuite/DataStream/Primitives/Time/SecondStream.cs

@@ -37,4 +37,42 @@ namespace BCG
       }
          
    }
+
+   [DataStreamUserRespresentation("throttle")]
+   public class ThrottleStream : ViewStream{
+      private readonly ViewStream _main_stream;
+      private readonly TimeSpan _ts;
+      private readonly ViewStreamDatabase _vsd;
+      private DateTime _last_fire_time = DateTime.MinValue;
+
+      public ThrottleStream (ViewStream main_stream, int throttle_seconds)
+      :
+      this(main_stream, TimeSpan.FromSeconds(throttle_seconds), ViewStreamDatabase.instance)
+      {
+      }
+
+      public ThrottleStream (ViewStream main_stream, TimeSpan ts, ViewStreamDatabase vsd){
+         _main_stream = main_stream;
+         _ts = ts;
+         _vsd = vsd;
+      }
+
+       protected override void myOnFirstSubscriber() {
+         _main_stream.onChange += _handle_base_stream_update;
+      }
+
+      protected override void myOnLastSubscriber() {
+         _main_stream.onChange -= _handle_base_stream_update;
+      }
+
+      private void _handle_base_stream_update(object dp_obj){
+         if (DateTime.Now - _last_fire_time >= _ts){
+            DataPacket dp_in = dp_obj as DataPacket;
+            _last_fire_time = DateTime.Now;
+            DataPacket dp = BlueDataUtility.build_dp_inc(dp_in.newRows, myLastValue as DataPacket);
+            myFireEvent(dp);
+         }
+      }
+ }
+
 }

src/DataStreamSuite/DataStream/Primitives/Time/ThrottleStream

@@ -0,0 +1,40 @@
+namespace bts.SilverPine{
+   using BGC;
+   using System;
+   [DataStreamUserRespresentation("throttle")]
+   public class ThrottleStream : ViewStream{
+      private readonly ViewStream _main_stream;
+      private readonly TimeSpan _ts;
+      private readonly ViewStreamDatabase _vsd;
+      private DateTime _last_fire_time = DateTime.MinValue;
+
+      public ThrottleStream (ViewStream main_stream, int throttle_seconds)
+      :
+      this(main_stream, TimeSpan.FromSeconds(throttle_seconds), ViewStreamDatabase.instance)
+      {
+      }
+
+      public ThrottleStream (ViewStream main_stream, TimeSpan ts, ViewStreamDatabase vsd){
+         _main_stream = main_stream;
+         _ts = ts;
+         _vsd;
+      }
+
+       protected override void myOnFirstSubscriber() {
+         _main_stream.onChange += _handle_base_stream_update;
+      }
+
+      protected override void myOnLastSubscriber() {
+         _main_stream.onChange -= _handle_base_stream_update;
+      }
+
+      private void _handle_base_stream_update(object dp_obj){
+         if (DateTime.Now - _last_fire_time >= _ts){
+            DataPacket dp_in = dp_obj;
+            _last_fire_time = DateTime.Now;
+            DataPacket dp = BlueDataUtility.build_dp_inc(dp_in.newRow, myLastValue as DataPacket);
+            myFireEvent(dp);
+         }
+      }
+ }
+}

src/DataStreamSuite/DataStream/Primitives/Union/MutableAppendStream.cs

@@ -1,18 +1,15 @@
 using System;
 using System.Collections.Generic;
+
 using mbts;
 
-namespace BCG
-{
-   internal class MutableAppendTable : IBlueRowCollection
-   {
-      private class MyRow : IBlueRow
-      {
+namespace BCG {
+   internal class MutableAppendTable : IBlueRowCollection {
+      private class MyRow : IBlueRow {
          private IBlueRow myRow;
          private int myID;
 
-         public MyRow(IBlueRow r, int id)
-         {
+         public MyRow(IBlueRow r, int id) {
             myRow = r;
             myID = id;
          }
@@ -21,10 +18,8 @@ namespace BCG
 
          public IBlueTableSchema schema { get { return myRow.schema; } }
 
-         public object this[int index]
-         {
-            get
-            {
+         public object this[int index] {
+            get {
                if (myRow.schema[index].colName == "id")
                   return myID;
                else
@@ -52,20 +47,17 @@ namespace BCG
 
          public System.Collections.ICollection Keys { get { return myRow.Keys; } }
 
-         public void Remove(object key) {throw new Exception("The method or operation is not implemented.");}
+         public void Remove(object key) { throw new Exception("The method or operation is not implemented."); }
 
-         public System.Collections.ICollection Values
-         {
-            get
-            {
+         public System.Collections.ICollection Values {
+            get {
                object[] vals = new object[myRow.Count];
                CopyTo(vals, 0);
                return vals;
             }
          }
 
-         public object this[object key]
-         {
+         public object this[object key] {
             get { return key is string ? this[key as string] : this[(int)key]; }
             set { throw new Exception("The method or operation is not implemented."); }
          }
@@ -74,8 +66,7 @@ namespace BCG
 
          #region ICollection Members
 
-         public void CopyTo(Array array, int index)
-         {
+         public void CopyTo(Array array, int index) {
             myRow.CopyTo(array, index);
             array.SetValue(myID, index + myRow.schema["id"].colNo);
          }
@@ -90,8 +81,7 @@ namespace BCG
 
          #region IEnumerable Members
 
-         System.Collections.IEnumerator System.Collections.IEnumerable.GetEnumerator()
-         { return new BlueRowEnumerator(this); }
+         System.Collections.IEnumerator System.Collections.IEnumerable.GetEnumerator() { return new BlueRowEnumerator(this); }
 
          #endregion
       }
@@ -106,8 +96,7 @@ namespace BCG
 
       public MutableAppendTable(IBlueTableSchema schema) { myRows = new BlueTable(schema); }
 
-      public IBlueRow addRow(IBlueRow r, int tableNo)
-      {
+      public IBlueRow addRow(IBlueRow r, int tableNo) {
          if (!myIDMap.ContainsKey(tableNo))
             myIDMap.Add(tableNo, new Dictionary<int, IBlueRow>());
          IBlueRow nr = new MyRow(r, myID++); // BlueDataUtility.getNewReplacedRow(r, myID++, "id");
@@ -116,20 +105,17 @@ namespace BCG
          return nr;
       }
 
-      public IBlueRow removeRow(IBlueRow r, int tableNo)
-      {
+      public IBlueRow removeRow(IBlueRow r, int tableNo) {
          IBlueRow dr = myIDMap[tableNo][(int)r["id"]];
          myRows.deleteRow(dr);
          myIDMap[tableNo].Remove((int)r["id"]);
          return dr;
       }
 
-      public IBlueRowCollection removeTable(int tableNo)
-      {
+      public IBlueRowCollection removeTable(int tableNo) {
          if (!myIDMap.ContainsKey(tableNo))
             return null;
-         else
-         {
+         else {
             BlueTable rv = new BlueTable(myRows.schema);
             foreach (IBlueRow r in myIDMap[tableNo].Values)
                rv.addRow(r);
@@ -139,8 +125,7 @@ namespace BCG
          }
       }
 
-      public void clear()
-      {
+      public void clear() {
          myRows.clear();
          myIDMap.Clear();
       }
@@ -151,7 +136,7 @@ namespace BCG
 
       public IBlueTableSchema schema { get { return myRows.schema; } }
 
-      public bool containsID(int id) {return myRows.containsID(id);}
+      public bool containsID(int id) { return myRows.containsID(id); }
 
       #endregion
 
@@ -169,7 +154,7 @@ namespace BCG
 
       #region IEnumerable Members
 
-      public System.Collections.IEnumerator GetEnumerator() {return myRows.GetEnumerator();}
+      public System.Collections.IEnumerator GetEnumerator() { return myRows.GetEnumerator(); }
 
       #endregion
    }
@@ -179,8 +164,8 @@ namespace BCG
    /// to append (or remove) more streams at
    /// runtime
    /// </summary>
-   public class MutableAppendStream : ViewStream
-   {
+   [DataStreamUserRespresentation("UnionStream")]
+   public class MutableAppendStream : ViewStream {
       private bool mySubscriptionActive = false;
       private MutableAppendTable myTable;
       private int myMovingIndex;
@@ -188,22 +173,17 @@ namespace BCG
       private bool _allign_col_orders = true;
 
       public MutableAppendStream(params ViewStream[] streams)
-         : this(new List<ViewStream>(streams))
-      { }
+         : this(new List<ViewStream>(streams)) { }
 
       public MutableAppendStream(bool allign_col_order, params ViewStream[] streams)
-         : this(allign_col_order, new List<ViewStream>(streams))
-      { }
+         : this(allign_col_order, new List<ViewStream>(streams)) { }
 
       public MutableAppendStream(ICollection<ViewStream> streams)
-         : this(true, streams)
-      { }
+         : this(true, streams) { }
 
-      public MutableAppendStream(bool allign_col_orders, ICollection<ViewStream> streams)
-      {
+      public MutableAppendStream(bool allign_col_orders, ICollection<ViewStream> streams) {
          _allign_col_orders = allign_col_orders;
-         foreach (ViewStream vs in streams)
-         {
+         foreach (ViewStream vs in streams) {
             myMovingIndex++;
             ViewStream pj = new ChangeTableNameStream(vs, myMovingIndex.ToString());
             if (_allign_col_orders)
@@ -212,8 +192,7 @@ namespace BCG
          }
       }
 
-      public void addStream(ViewStream vs)
-      {
+      public void addStream(ViewStream vs) {
          myMovingIndex++;
          ///the change name is done to identify the callbacks, and the project to
          ///make all the schemas col orders the same
@@ -225,19 +204,14 @@ namespace BCG
             pj.onChange += new ChangeDelegate(myHandleStreamChanges);
       }
 
-      public void removeStream(ViewStream vs)
-      {
+      public void removeStream(ViewStream vs) {
          int index = -1;
-         foreach (KeyValuePair<int, Pair<ViewStream, ViewStream>> p in myStreamIndex)
-         {
-            if (p.Value.first == vs)
-            {
+         foreach (KeyValuePair<int, Pair<ViewStream, ViewStream>> p in myStreamIndex) {
+            if (p.Value.first == vs) {
                index = p.Key;
-               if (mySubscriptionActive)
-               {
+               if (mySubscriptionActive) {
                   p.Value.second.onChange -= new ChangeDelegate(myHandleStreamChanges);
-                  if (myTable != null)
-                  {
+                  if (myTable != null) {
                      IBlueRowCollection delRows = myTable.removeTable(p.Key);
                      if (delRows != null)
                         myFireEvent(new DataPacket(myTable, new BlueTable(myTable.schema), delRows));
@@ -250,15 +224,13 @@ namespace BCG
          }
       }
 
-      protected override void myOnFirstSubscriber()
-      {
+      protected override void myOnFirstSubscriber() {
          mySubscriptionActive = true;
          foreach (KeyValuePair<int, Pair<ViewStream, ViewStream>> p in myStreamIndex)
             p.Value.second.onChange += new ChangeDelegate(myHandleStreamChanges);
       }
 
-      protected override void myOnLastSubscriber()
-      {
+      protected override void myOnLastSubscriber() {
          mySubscriptionActive = false;
          foreach (KeyValuePair<int, Pair<ViewStream, ViewStream>> p in myStreamIndex)
             p.Value.second.onChange -= new ChangeDelegate(myHandleStreamChanges);
@@ -266,20 +238,17 @@ namespace BCG
             myTable.clear();
       }
 
-      private void myHandleStreamChanges(object newValue)
-      {         
+      private void myHandleStreamChanges(object newValue) {
          DataPacket dp = newValue as DataPacket;
          myTable = myTable ?? new MutableAppendTable(new BlueTableSchema("MutableAppend", dp.schema.columnInfo));
          BlueTable adds = new BlueTable(myTable.schema),
                      dels = new BlueTable(myTable.schema);
          int index = Int32.Parse(dp.schema.tableName);
-         foreach (IBlueRow r in dp.deletedRows)
-         {
+         foreach (IBlueRow r in dp.deletedRows) {
             IBlueRow dr = myTable.removeRow(r, index);
             dels.addRow(dr);
          }
-         foreach (IBlueRow r in dp.addedRows)
-         {
+         foreach (IBlueRow r in dp.addedRows) {
             IBlueRow nr = myTable.addRow(r, index);
             adds.addRow(nr);
          }

src/DataStreamSuite/DataStreamCore/DataStreamCore.csproj

@@ -2,7 +2,7 @@
 
   <PropertyGroup>
     <AllowUnsafeBlocks>true</AllowUnsafeBlocks>
-    <TargetFramework>net7.0</TargetFramework>
+    <TargetFramework>net8.0</TargetFramework>
     <Nullable>disable</Nullable>
   </PropertyGroup>
 
@@ -19,8 +19,8 @@
   </ItemGroup>
 
   <ItemGroup>
-    <Proto Include="../../lib/config/protos/data_stream_config.proto" CSName="DataStreamConfig">
-      <OutDir>$(ProtoOutputPath)/lib/config/protos</OutDir>
+    <Proto Include="../../lib_spark/spark/config/protos/data_stream_config.proto" CSName="DataStreamConfig">
+      <OutDir>$(ProtoOutputPath)/lib_spark/spark/config/protos</OutDir>
     </Proto>
   </ItemGroup>
 

src/DataStreamSuite/DataStreamExpressions/DataStreamExpressions.csproj

@@ -1,7 +1,7 @@
 <Project Sdk="Microsoft.NET.Sdk">
 
   <PropertyGroup>
-    <TargetFramework>net7.0</TargetFramework>
+    <TargetFramework>net8.0</TargetFramework>
     <ImplicitUsings>enable</ImplicitUsings>
     <Nullable>enable</Nullable>
   </PropertyGroup>

src/DataStreamSuite/DataStreamPublisher/DataStreamPublisher.csproj

@@ -2,7 +2,7 @@
 
   <PropertyGroup>
     <OutputType>Exe</OutputType>
-    <TargetFramework>net7.0</TargetFramework>
+    <TargetFramework>net8.0</TargetFramework>
     <Nullable>enable</Nullable>
     <AllowUnsafeBlocks>true</AllowUnsafeBlocks>
   </PropertyGroup>

src/DataStreamSuite/DataStreamQueryAnalyzer/DataStreamQueryAnalyzer.csproj

@@ -2,7 +2,7 @@
 
   <PropertyGroup>
     <OutputType>Exe</OutputType>
-    <TargetFramework>net7.0-windows</TargetFramework>
+    <TargetFramework>net8.0-windows</TargetFramework>
     <Nullable>enable</Nullable>
     <UseWindowsForms>true</UseWindowsForms>
   </PropertyGroup>

src/DataStreamSuite/DataStreamTests/DataStreamTests.csproj

@@ -1,7 +1,7 @@
 <Project Sdk="Microsoft.NET.Sdk">
 
   <PropertyGroup>
-    <TargetFramework>net7.0</TargetFramework>
+    <TargetFramework>net8.0</TargetFramework>
     <ImplicitUsings>enable</ImplicitUsings>
     <Nullable>enable</Nullable>
 

src/DataStreamSuite/ExpressionCore/ExpressionCore.csproj

@@ -1,7 +1,7 @@
 <Project Sdk="Microsoft.NET.Sdk">
 
   <PropertyGroup>
-    <TargetFramework>net7.0</TargetFramework>
+    <TargetFramework>net8.0</TargetFramework>
     <ImplicitUsings>enable</ImplicitUsings>
     <Nullable>enable</Nullable>
   </PropertyGroup>

src/DataStreamSuite/GuiUtilities/GuiUtilities.csproj

@@ -1,7 +1,7 @@
 <Project Sdk="Microsoft.NET.Sdk">
 
   <PropertyGroup>
-    <TargetFramework>net7.0-windows</TargetFramework>
+    <TargetFramework>net8.0-windows</TargetFramework>
     <ImplicitUsings>enable</ImplicitUsings>
     <Nullable>enable</Nullable>
     <UseWindowsForms>true</UseWindowsForms>

src/DataStreamSuite/PubSub/Client/PublishingUtilities/IPublisher.cs

@@ -6,12 +6,18 @@ namespace BCG
 	public interface IPublisher
 	{
       void publish( IPubItem pi );
+#pragma warning disable SYSLIB0011 // Type or member is obsolete
       void publish( IPubItem pi, IFormatter f );
+#pragma warning restore SYSLIB0011 // Type or member is obsolete
       void publish( IPubItem[] pi );
+#pragma warning disable SYSLIB0011 // Type or member is obsolete
       void publish( IPubItem[] pi, IFormatter f );
+#pragma warning restore SYSLIB0011 // Type or member is obsolete
    }
 
+#pragma warning disable SYSLIB0011 // Type or member is obsolete
    public interface IPublisher<T> where T : IFormatter
+#pragma warning restore SYSLIB0011 // Type or member is obsolete
    {
       void publish (IPubItem pi);
       void publish (IPubItem[] pi);

src/DataStreamSuite/PubSub/Misc/GenericFormatter.cs

@@ -5,23 +5,21 @@ using System.Runtime.Serialization.Formatters.Binary;
 
 namespace BCG
 {
+#pragma warning disable SYSLIB0011 // Type or member is obsolete
    public class GenericFormatter<F> : IGenericFormatter where F : IFormatter, new ()
    {
       IFormatter myFormatter = new F ();
 
       public T Deserialize<T> (Stream serializationStream)
       {
-         #pragma warning disable 
          return (T) myFormatter.Deserialize (serializationStream);
-         #pragma warning restore SYSLIB0011
       }
       public void Serialize<T> (Stream serializationStream, T graph)
       {
-         #pragma warning disable 
          myFormatter.Serialize (serializationStream, graph);
-         #pragma warning restore SYSLIB0011
       }
    }
 
    public class GenericBinaryFormatter : GenericFormatter<BinaryFormatter> { }
+#pragma warning restore SYSLIB0011 // Type or member is obsolete
 }

src/DataStreamSuite/PubSub/PubSub.csproj

@@ -6,7 +6,7 @@
 
   <PropertyGroup>
       <AllowUnsafeBlocks>true</AllowUnsafeBlocks>
-    <TargetFramework>net7.0</TargetFramework>
+    <TargetFramework>net8.0</TargetFramework>
     <Nullable>enable</Nullable>
   </PropertyGroup>
 

src/DataStreamSuite/PubSubServer/PubSubServer.csproj

@@ -2,7 +2,7 @@
 
   <PropertyGroup>
     <OutputType>Exe</OutputType>
-    <TargetFramework>net7.0</TargetFramework>
+    <TargetFramework>net8.0</TargetFramework>
     <Nullable>enable</Nullable>
   </PropertyGroup>
 

src/DataStreamSuite/SharedStreams/Business/MarketData/Analytics/SpatialArbScoreStream.cs

@@ -0,0 +1,17 @@
+namespace bts.SilverPine {
+   using BCG;
+   using System;
+
+   [DataStreamUserRespresentation("SpatialArbScoreStream")]
+   public class SpatialArbScoreStream : ViewStream {
+      public SpatialArbScoreStream(int window_sz_secs, int sample_period_sz)
+         : this(TimeSpan.FromSeconds(window_sz_secs), TimeSpan.FromSeconds(sample_period_sz), ViewStreamDatabase.instance) { }
+
+      public SpatialArbScoreStream(TimeSpan window_sz, TimeSpan sample_period, ViewStreamDatabase vsd) {
+         ViewStream rs = new SpatialArbStatsStream(window_sz, sample_period, vsd);
+         rs = new ThrottleStream(rs, TimeSpan.FromSeconds(1), vsd);
+         rs = new SortStream(rs, "-n_trade_ticks");
+         mySetRealSource(rs);
+      }
+   }
+}

src/DataStreamSuite/SharedStreams/Business/MarketData/Analytics/SpatialArbStatsStream.cs

@@ -0,0 +1,113 @@
+namespace bts.SilverPine {
+   using BCG;
+
+   using SharpCompress.Common.Rar;
+
+   using System;
+   using System.Collections.Generic;
+   using System.Linq;
+   using System.Text;
+   using System.Threading.Tasks;
+
+   // For all the pairs generated by SpatialFuturePairsStream, this stream 
+   // generates the statistics, by appending together a SpreadStatsStream
+   [DataStreamUserRespresentation("SpatialArbStatsStream")]
+   public class SpatialArbStatsStream : ViewStream{
+      private ViewStream _spatial_pairs_stream;
+      private ViewStreamDatabase _vsd;
+      private ViewStream _tot_str = null;
+      private HashSet<string> _key_map = new HashSet<string>();
+      private ViewStream _bbo_stream;
+      private readonly TimeSpan _window_sz, _sampling_period_sz;
+
+      public SpatialArbStatsStream(int window_sz_seconds, int sampling_period_sz_seconds)
+         : this(TimeSpan.FromSeconds(window_sz_seconds), TimeSpan.FromSeconds(sampling_period_sz_seconds), ViewStreamDatabase.instance) { }
+
+      public SpatialArbStatsStream(TimeSpan window_sz, TimeSpan sampling_period_sz, ViewStreamDatabase vsd) {
+         _vsd = vsd;
+         _window_sz = window_sz;
+         _sampling_period_sz = sampling_period_sz;
+         _spatial_pairs_stream = new SpatialFuturePairsStream(vsd);
+         _bbo_stream = new BBOStream(vsd);
+      }
+
+      protected override void myOnFirstSubscriber() {
+         _spatial_pairs_stream.onChange += _spatial_pairs_stream_onchange;
+      }
+
+      protected override void myOnLastSubscriber() {
+         _spatial_pairs_stream.onChange -= _spatial_pairs_stream_onchange;
+      }
+
+      private ViewStream _build_mapped_stream(ViewStream parent_str) {
+         Func<IBlueTableSchema, IBlueTableSchema> build_schema = (inp_schema) => {
+            IBlueTableSchema rv = new BlueTableSchema("SpreaddStats", new ColumnInfo("id", 0, TypeCode.Int32));
+            rv = rv.add_column(TypeCode.String, "spread");
+            rv = rv.add_column(TypeCode.String, "window");
+            // bids
+            rv = rv.add_column(TypeCode.Double, "bid_mean");
+            rv = rv.add_column(TypeCode.Double, "bid_stddev");
+            rv = rv.add_column(TypeCode.Double, "bid_skewness");
+            rv = rv.add_column(TypeCode.Double, "bid_kurtosis");
+            rv = rv.add_column(TypeCode.Double, "bid_max");
+            rv = rv.add_column(TypeCode.Double, "bid_min");
+            // asks
+            rv = rv.add_column(TypeCode.Double, "ask_mean");
+            rv = rv.add_column(TypeCode.Double, "ask_stddev");
+            rv = rv.add_column(TypeCode.Double, "ask_skewness");
+            rv = rv.add_column(TypeCode.Double, "ask_kurtosis");
+            rv = rv.add_column(TypeCode.Double, "ask_max");
+            rv = rv.add_column(TypeCode.Double, "ask_min");
+            // tail distribution
+            rv = rv.add_column(TypeCode.Double, "bid Z ≥ 2σ");
+            rv = rv.add_column(TypeCode.Double, "ask Z < -2σ");
+            // trading ticks
+            rv = rv.add_column(TypeCode.Int32, "n_buy_ticks");
+            rv = rv.add_column(TypeCode.Double, "cumm_buy_price");
+            rv = rv.add_column(TypeCode.Int32, "n_sell_ticks");
+            rv = rv.add_column(TypeCode.Double, "cumm_sell_price");
+            rv = rv.add_column(TypeCode.Int32, "n_trade_ticks");
+            return rv;
+         };
+         Func<IBlueRow, IBlueRow> build_row = (r) => {
+            IBlueTableSchema s = build_schema(r.schema);
+            return new BlueRow(s, r["id"], r["spread"], r["window"],
+                           r["bid_mean"], r["bid_stddev"], r["bid_skewness"], r["bid_kurtosis"], r["bid_max"], r["bid_min"],
+                           r["ask_mean"], r["ask_stddev"], r["ask_skewness"], r["ask_kurtosis"], r["ask_max"], r["ask_min"],
+                           r["bid Z ≥ 2σ"], r["ask Z < -2σ"],
+                           r["n_buy_ticks"], r["cumm_buy_price"], r["n_sell_ticks"], r["cumm_sell_price"], r["n_trade_ticks"]);
+         };
+         MapStream rs = new MapStream(parent_str, inp_schema => build_schema(inp_schema),
+            inp_r => build_row(inp_r));
+         return rs;
+      }
+
+      private void _spatial_pairs_stream_onchange(object dp_obj) {
+         if (_tot_str != null)
+            return;
+         MutableAppendStream append_str = new MutableAppendStream();
+         _tot_str = _build_mapped_stream(append_str);
+         //_tot_str = new ThrottleStream(_tot_str, 1000);
+         //_tot_str = new SortStream(_tot_str, "-n_trade_ticks");
+         _tot_str.onChange += _handle_tot_stream;
+
+         DataPacket dp = dp_obj as DataPacket;         
+         foreach (IBlueRow r in dp.newRows) {
+            string s1 = r.get<string>("s1");
+            string s2 = r.get<string>("s2");
+            string key = SpatialFuturePairsStream.get_key(s1, s2);
+            if (!_key_map.Contains(key)) {
+               _key_map.Add(key);
+               ViewStream spread_stats_stream = new SpreadStatsStream(s1, s2, 1, 1, Convert.ToInt32(_window_sz.TotalSeconds), 
+                                                                     Convert.ToInt32(_sampling_period_sz.TotalSeconds), _bbo_stream, _vsd);
+               append_str.addStream(spread_stats_stream);
+            }
+         }
+      }
+
+      private void _handle_tot_stream(object dp_obj) {
+         myFireEvent(dp_obj);
+      }
+   }
+
+}

src/DataStreamSuite/SharedStreams/Business/MarketData/Analytics/SpatialFuturePairsStream.cs

@@ -0,0 +1,89 @@
+namespace bts.SilverPine {
+   using BCG;
+
+   using bts.api;
+
+   using System;
+   using System.Collections.Generic;
+   using System.Linq;
+   using System.Text;
+   using System.Threading.Tasks;
+
+   [DataStreamUserRespresentation("SpatialFuturePairsStream")]
+   public class SpatialFuturePairsStream : ViewStream {
+      private ViewStream _main_stream;
+      private IBlueTableSchema _schema;
+      private HashSet<string> _key_map = new HashSet<string>();
+      public SpatialFuturePairsStream()
+         : this(ViewStreamDatabase.instance) { }
+
+      public SpatialFuturePairsStream(ViewStreamDatabase vsd) {
+         _main_stream = new InstrumentsStream(vsd);
+         _main_stream = new FilterStream(_main_stream, "sec_type='Future'");
+         _schema = new BlueTableSchema("SpatialFutures", new ColumnInfo("id", 0, TypeCode.Int32));
+         _schema = _schema.add_column(TypeCode.String, "s1");
+         _schema = _schema.add_column(TypeCode.String, "venue_1");
+         _schema = _schema.add_column(TypeCode.String, "s2");
+         _schema = _schema.add_column(TypeCode.String, "venue_2");
+
+      }
+
+      protected override void myOnFirstSubscriber() {
+         _main_stream.onChange += _main_stream_onchange;
+      }
+
+      protected override void myOnLastSubscriber() {
+         _main_stream.onChange -= _main_stream_onchange;
+      }
+
+      public static string get_key(string s1, string s2) {
+         string key = string.Compare(s1, s2, StringComparison.Ordinal) <= 0 ? s1 + s2 : s2 + s1;
+         return key;
+      }
+
+      public static string get_key(IBlueRow r1, IBlueRow r2) {
+         string s1 = r1.get<string>("sec");
+         string s2 = r2.get<string>("sec");
+         return get_key(s1, s2);
+      }
+
+      private bool _combine(IBlueRow r1, IBlueRow r2) {
+         string key = get_key(r1, r2);
+         if (_key_map.Contains(key))
+            return false;
+         if (r1.get<string>("symbol") != r2.get<string>("symbol"))
+            return false;
+         if (r1.get_enum<Venue>("venue") == r2.get_enum<Venue>("venue"))
+            return false;
+         if (r1.get<int>("sec_key") == r2.get<int>("sec_key")) {
+            return false;
+         }
+         DateTime e1 = r1.get_datetime("expiration");
+         DateTime e2 = r2.get_datetime("expiration");
+         bool exps_eq = e1 == e2 || (e1.Year >= 2100 && e2.Year >= 2100);
+         if (!exps_eq)
+            return false;         
+         return true;
+      }
+
+      private void _main_stream_onchange(object dp_obj) {
+         _key_map.Clear();
+         DataPacket dp = dp_obj as DataPacket;
+         BlueTable table = new BlueTable(_schema);
+         var rows = dp.newRows.to_list();
+         foreach (var r1 in rows) {
+            foreach (var r2 in rows) {
+               if (_combine(r1, r2)) {
+                  IBlueRow nr = new BlueRow(_schema, table.Count, r1.get<string>("sec"), r1.get<string>("venue"),
+                     r2.get<string>("sec"), r2.get<string>("venue"));
+                  table.addRow(nr);
+                  string key = get_key(r1, r2);
+                  _key_map.Add(key);
+               }
+            }
+         }
+         var dp_out = BlueDataUtility.build_dp(table, myLastValue as DataPacket);
+         myFireEvent(dp_out);
+      }
+   }
+}

src/DataStreamSuite/SharedStreams/Business/MarketData/Analytics/SpreadPerformanceEvaluator.cs

@@ -0,0 +1,656 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using MathNet.Numerics.Statistics;
+using MathNet.Numerics.LinearRegression;
+using System.Collections.Generic;
+using System.Linq;
+using MathNet.Numerics.Statistics;
+using MathNet.Numerics.LinearAlgebra;
+using Amazon.Runtime;
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using MathNet.Numerics.Statistics;
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using MathNet.Numerics.Statistics;
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using MathNet.Numerics.Statistics;
+// stats calculator
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using MathNet.Numerics.Statistics;
+
+/// <summary>
+/// Provides evaluation metrics to assess the quality of a synthetic spread for mean-reversion trading.
+/// Assumes the input spread data is a trailing, time-causal window of recent ticks (e.g., last 120 seconds).
+/// </summary>
+public static class SpreadPerformanceEvaluator
+{
+
+   public static class SpreadStatistics {
+      public class SpreadStatsResult {
+         public Dictionary<string, double> bid_zscore_distribution { get; set; }
+         public Dictionary<string, double> ask_zscore_distribution { get; set; }
+         public Dictionary<string, double> mid_zscore_distribution { get; set; }
+
+         public DescriptiveStatistics bid_stats { get; set; }
+         public DescriptiveStatistics ask_stats { get; set; }
+         public DescriptiveStatistics mid_stats { get; set; }
+      }
+
+      private static readonly string[] _zscore_labels = new[]
+      {
+        "Z < -2σ",
+        "-2σ ≤ Z < -1σ",
+        "-1σ ≤ Z < 0σ",
+        "0σ ≤ Z < 1σ",
+        "1σ ≤ Z < 2σ",
+        "Z ≥ 2σ"
+    };
+
+      public static SpreadStatsResult analyze_spread_quotes(List<(DateTime timestamp, double bid, double bid_qty, double ask, double ask_qty)> quotes) {
+         if (quotes == null || quotes.Count == 0)
+            throw new ArgumentException("Spread quote list cannot be null or empty.");
+
+         var bid_prices = quotes.Select(q => q.bid).ToList();
+         var ask_prices = quotes.Select(q => q.ask).ToList();
+         var mid_prices = quotes.Select(q => (q.bid + q.ask) / 2.0).ToList();
+
+         var bid_stats = new DescriptiveStatistics(bid_prices);
+         var ask_stats = new DescriptiveStatistics(ask_prices);
+         var mid_stats = new DescriptiveStatistics(mid_prices);
+
+         var bid_zscores = bid_prices.Select(x => zscore(x, bid_stats.Mean, bid_stats.StandardDeviation)).ToList();
+         var ask_zscores = ask_prices.Select(x => zscore(x, ask_stats.Mean, ask_stats.StandardDeviation)).ToList();
+         var mid_zscores = mid_prices.Select(x => zscore(x, mid_stats.Mean, mid_stats.StandardDeviation)).ToList();
+
+         var bid_distribution = bin_zscores_to_percentages(bid_zscores);
+         var ask_distribution = bin_zscores_to_percentages(ask_zscores);
+         var mid_distribution = bin_zscores_to_percentages(mid_zscores);
+
+         return new SpreadStatsResult {
+            bid_zscore_distribution = bid_distribution,
+            ask_zscore_distribution = ask_distribution,
+            mid_zscore_distribution = mid_distribution,
+            bid_stats = bid_stats,
+            ask_stats = ask_stats,
+            mid_stats = mid_stats
+         };
+      }
+
+      private static double zscore(double x, double mean, double std_dev) {
+         return std_dev == 0 ? 0 : (x - mean) / std_dev;
+      }
+
+      private static Dictionary<string, double> bin_zscores_to_percentages(List<double> zscores) {
+         var bins = _zscore_labels.ToDictionary(label => label, _ => 0);
+
+         foreach (var z in zscores) {
+            if (z < -2) bins["Z < -2σ"]++;
+            else if (z < -1) bins["-2σ ≤ Z < -1σ"]++;
+            else if (z < 0) bins["-1σ ≤ Z < 0σ"]++;
+            else if (z < 1) bins["0σ ≤ Z < 1σ"]++;
+            else if (z < 2) bins["1σ ≤ Z < 2σ"]++;
+            else bins["Z ≥ 2σ"]++;
+         }
+
+         double total = zscores.Count;
+         return bins.ToDictionary(kvp => kvp.Key, kvp => total == 0 ? 0 : (kvp.Value / total) * 100.0);
+      }
+
+      public static double score_spread(SpreadStatsResult result) {
+         var ask_bins = result.ask_zscore_distribution;
+         var bid_bins = result.bid_zscore_distribution;
+         var mid_bins = result.mid_zscore_distribution;
+
+         double buy_opportunity = ask_bins["Z < -2σ"] + ask_bins["-2σ ≤ Z < -1σ"];
+         double sell_opportunity = bid_bins["Z ≥ 2σ"] + bid_bins["1σ ≤ Z < 2σ"];
+         double opportunity_score = buy_opportunity + sell_opportunity;
+
+         double below_mean = mid_bins["Z < -2σ"] + mid_bins["-2σ ≤ Z < -1σ"] + mid_bins["-1σ ≤ Z < 0σ"];
+         double above_mean = mid_bins["Z ≥ 2σ"] + mid_bins["1σ ≤ Z < 2σ"] + mid_bins["0σ ≤ Z < 1σ"];
+         double symmetry_score = 1.0 - Math.Abs(below_mean - above_mean) / (below_mean + above_mean + 1e-6);
+
+         double amplitude_score = result.mid_stats.StandardDeviation;
+         double skew_penalty = 1.0 / (1.0 + Math.Abs(result.mid_stats.Skewness));
+         double kurtosis_bonus = Math.Min(result.mid_stats.Kurtosis, 6.0) / 6.0;
+
+         double score =
+             opportunity_score *
+             symmetry_score *
+             amplitude_score *
+             skew_penalty *
+             (1.0 + kurtosis_bonus * 0.1);
+
+         return score;
+      }
+   }
+
+
+
+
+
+/// <summary>
+/// Represents a single trade action (entry or exit) taken during the simulation.
+/// </summary>
+public struct TradeAction
+    {
+        public bool IsEntry { get; set; }    // True if this action opened a position
+        public bool IsLong { get; set; }     // True = Long, False = Short
+        public double Price { get; set; }    // Executed price
+        public int Index { get; set; }       // Tick index of the action
+
+        public override string ToString()
+        {
+            string side = IsLong ? "Long" : "Short";
+            string type = IsEntry ? "Open" : "Close";
+            return $"{type} {side} @ {Price:F4} (Tick {Index})";
+        }
+    }
+
+    /// <summary>
+/// Simulates a mean-reversion strategy using rolling, causal mid-price stats.
+/// ✅ Entry:
+///   - Buy at ask when ask < rollingMean - entryThreshold × rollingStdDev
+///   - Sell at bid when bid > rollingMean + entryThreshold × rollingStdDev
+/// ✅ Exit:
+///   - Close when mid returns within rollingMean ± exitThreshold × rollingStdDev
+///   - Long exits at bid, short exits at ask
+/// This version uses only past and current data at each step (no forward-looking bias).
+/// </summary>
+public static (double TotalPnL, List<TradeAction> TradeActions, int BuyTurns, int SellTurns) EvaluateSimulatedPnL(
+    List<Tuple<double, double, double, double>> spreadData,
+    int statWindow = 30,
+    double entryThresholdMultiplier = 0.5,
+    double exitThresholdMultiplier = 0.1)
+{
+    var trades = new List<TradeAction>();
+    var mids = spreadData.Select(x => (x.Item1 + x.Item3) / 2.0).ToList();
+
+    if (spreadData.Count < statWindow)
+        return (0.0, trades, 0, 0);
+
+    bool positionOpen = false;
+    bool isLong = false;
+    double entryPrice = 0.0;
+    double pnl = 0.0;
+    int buyTurns = 0;
+    int sellTurns = 0;
+
+    for (int i = statWindow; i < spreadData.Count; i++)
+    {
+        double bid = spreadData[i].Item1;
+        double ask = spreadData[i].Item3;
+        double mid = mids[i];
+
+        // Rolling window
+        var window = mids.Skip(i - statWindow).Take(statWindow).ToList();
+        double mean = window.Average();
+        double stddev = Math.Sqrt(window.Select(x => Math.Pow(x - mean, 2)).Average());
+
+        double lowerThreshold = mean - entryThresholdMultiplier * stddev;
+        double upperThreshold = mean + entryThresholdMultiplier * stddev;
+        double exitBand = exitThresholdMultiplier * stddev;
+
+        if (!positionOpen)
+        {
+            if (ask < lowerThreshold)
+            {
+                positionOpen = true;
+                isLong = true;
+                entryPrice = ask;
+                trades.Add(new TradeAction { IsEntry = true, IsLong = true, Price = entryPrice, Index = i });
+            }
+            else if (bid > upperThreshold)
+            {
+                positionOpen = true;
+                isLong = false;
+                entryPrice = bid;
+                trades.Add(new TradeAction { IsEntry = true, IsLong = false, Price = entryPrice, Index = i });
+            }
+        }
+        else
+        {
+            if (Math.Abs(mid - mean) < exitBand)
+            {
+                double exitPrice = isLong ? bid : ask;
+                pnl += isLong ? (exitPrice - entryPrice) : (entryPrice - exitPrice);
+
+                trades.Add(new TradeAction { IsEntry = false, IsLong = isLong, Price = exitPrice, Index = i });
+
+                if (isLong) buyTurns++;
+                else sellTurns++;
+
+                positionOpen = false;
+            }
+        }
+    }
+
+    return (pnl, trades, buyTurns, sellTurns);
+}
+
+
+
+    /// <summary>
+    /// Estimates the Hurst exponent of the spread mid-prices.
+    /// Assumes the input is a trailing window of recent data.
+    /// Hurst < 0.5 implies mean-reversion; ideal is 0.35–0.45.
+    /// </summary>
+    public static double EvaluateHurstExponent(List<Tuple<double, double, double, double>> spreadData)
+    {
+        const int MinLength = 21;
+
+        if (spreadData == null || spreadData.Count < MinLength)
+            return double.NaN;
+
+        var spreadMids = spreadData.Select(x => (x.Item1 + x.Item3) / 2.0).ToArray();
+        int n = spreadMids.Length;
+
+        var lags = new List<double>();
+        var tau = new List<double>();
+
+        for (int lag = 2; lag <= 20; lag++)
+        {
+            if (n - lag <= 1)
+                continue;
+
+            double[] diffs = new double[n - lag];
+            for (int i = 0; i < diffs.Length; i++)
+                diffs[i] = spreadMids[i + lag] - spreadMids[i];
+
+            double stddev = diffs.StandardDeviation();
+
+            if (stddev > 0)
+            {
+                lags.Add(Math.Log(lag));
+                tau.Add(Math.Log(stddev));
+            }
+        }
+
+        if (lags.Count < 2)
+            return double.NaN;
+
+        var (intercept, slope) = SimpleRegression.Fit(lags.ToArray(), tau.ToArray());
+        double hurst = slope * 0.5 + 0.5;
+        return hurst;
+    }
+
+    /// <summary>
+/// Computes the mean and standard deviation of the spread mid-prices
+/// over a trailing, causal window.
+/// </summary>
+/// <param name="spreadData">List of (bid, bidQty, ask, askQty)</param>
+/// <returns>Tuple of (Mean, StdDev). Returns (NaN, NaN) if input is too short.</returns>
+public static (double Mean, double StdDev) EvaluateMeanAndStdDev(List<Tuple<double, double, double, double>> spreadData)
+{
+    if (spreadData == null || spreadData.Count < 2)
+        return (double.NaN, double.NaN);
+
+    var mids = spreadData.Select(x => (x.Item1 + x.Item3) / 2.0).ToList();
+    double mean = mids.Average();
+    double stddev = Math.Sqrt(mids.Select(x => Math.Pow(x - mean, 2)).Average());
+
+    return (mean, stddev);
+}
+
+
+   /// <summary>
+   /// Counts how many times the spread ask dropped below (mean - threshold) [buy signal]
+   /// and how many times the spread bid rose above (mean + threshold) [sell signal].
+   /// Assumes spreadData is a trailing, causal window.
+   /// </summary>
+   /// <returns>(BuyExcursions, SellExcursions)</returns>
+   public static (uint BuyExcursions, double cumm_buy_price, uint SellExcursions, double cumm_sell_price) EvaluateExcursionCount(
+       List<Tuple<double, double, double, double>> spreadData,
+       double entryThresholdMultiplier = 0.5) {
+      if (spreadData == null || spreadData.Count < 2)
+         return (0, Double.NaN, 0, Double.NaN);
+
+      var midPrices = spreadData.Select(x => (x.Item1 + x.Item3) / 2.0).ToList();
+      double mean = midPrices.Average();
+      double stddev = midPrices.StandardDeviation();
+      double buy_threshold = mean - entryThresholdMultiplier * stddev;
+      double sell_threshold = mean + entryThresholdMultiplier * stddev;
+
+      int buyCount = 0;
+      int sellCount = 0;
+      //bool in_buy_zone = false;
+      //bool in_sell_zone = false;
+
+      double cumm_buy_price = 0.0, cumm_sell_price = 0.0;
+      foreach (var (bid, _, ask, _) in spreadData) {
+         if (ask <= buy_threshold) {
+            //if (!in_buy_zone) {
+            cumm_buy_price += ask;
+            buyCount++;
+            //in_buy_zone = true;
+         }
+         //} else {
+         //   in_buy_zone = false;
+         //}
+
+         if (bid >= sell_threshold) {
+            //if (!in_sell_zone) {
+            cumm_sell_price += bid;
+            sellCount++;
+            //in_sell_zone = true;
+         }
+         //} else {
+         //   in_sell_zone = false;
+         //}
+      }
+
+      return ((uint)buyCount, cumm_buy_price / buyCount, (uint)sellCount, cumm_sell_price / sellCount);
+   }
+
+/// <summary>
+/// Counts how many times the spread mid-price crosses its mean level.
+/// Each crossing indicates a potential oscillation turn (cycle midpoint).
+/// Assumes the input is a trailing, causal window of spread ticks.
+/// </summary>
+/// <param name="spreadData">List of (bid, bidQty, ask, askQty)</param>
+/// <returns>Number of zero crossings in the mid-price series</returns>
+public static UInt32 EvaluateZeroCrossingCount(List<Tuple<double, double, double, double>> spreadData)
+{
+    if (spreadData == null || spreadData.Count < 2)
+        return 0;
+
+    var mids = spreadData.Select(x => (x.Item1 + x.Item3) / 2.0).ToList();
+    double mean = mids.Average();
+    int zeroCrossings = 0;
+
+    for (int i = 1; i < mids.Count; i++)
+    {
+        double prevOffset = mids[i - 1] - mean;
+        double currOffset = mids[i] - mean;
+
+        if (prevOffset * currOffset < 0) // Sign changed
+            zeroCrossings++;
+    }
+
+    return (UInt32)zeroCrossings;
+}
+
+   /// <summary>
+   /// Computes the half-life of mean reversion.
+   /// Lower half-life means the spread reverts to its mean more quickly.
+   /// </summary>
+   public static double ComputeHalfLife(List<Tuple<double, double, double, double>> spreadData) {
+      var prices = spreadData.Select(x => (x.Item1 + x.Item3) / 2.0).ToList();
+      var lagged = prices.Take(prices.Count - 1).ToArray();
+      var delta = prices.Skip(1).Zip(lagged, (cur, prev) => cur - prev).ToArray();
+
+      if (lagged.Length < 2) return double.NaN;
+
+      var x = Vector<double>.Build.Dense(lagged);
+      var y = Vector<double>.Build.Dense(delta);
+      var beta = x.DotProduct(y) / x.DotProduct(x);
+
+      return -Math.Log(2) / beta;
+   }
+
+   /// <summary>
+   /// Computes the lag-1 autocorrelation of the spread.
+   /// Lower values indicate quicker mean-reversion.
+   /// </summary>
+   public static double ComputeLag1Autocorrelation(List<Tuple<double, double, double, double>> spreadData) {
+      var prices = spreadData.Select(x => (x.Item1 + x.Item3) / 2.0).ToList();
+      var mean = prices.Average();
+      var numerator = 0.0;
+      var denominator = 0.0;
+
+      for (int i = 1; i < prices.Count; i++) {
+         numerator += (prices[i] - mean) * (prices[i - 1] - mean);
+         denominator += Math.Pow(prices[i - 1] - mean, 2);
+      }
+
+      return denominator == 0 ? 0 : numerator / denominator;
+   }
+
+   /// <summary>
+   /// Estimates how quickly extreme Z-score values decay back toward the mean.
+   /// Shorter average decay time indicates stronger mean-reversion.
+   /// </summary>
+   public static double ComputeAverageZScoreDecay(List<Tuple<double, double, double, double>> spreadData, int zScoreWindow = 50) {
+      var prices = spreadData.Select(x => (x.Item1 + x.Item3) / 2.0).ToList();
+      if (prices.Count < zScoreWindow + 2) return double.NaN;
+
+      var decayTimes = new List<int>();
+      for (int i = zScoreWindow; i < prices.Count - 1; i++) {
+         var window = prices.Skip(i - zScoreWindow).Take(zScoreWindow).ToList();
+         double mean = window.Average();
+         double stdDev = Statistics.StandardDeviation(window);
+         if (stdDev == 0) continue;
+
+         double z = (prices[i] - mean) / stdDev;
+         if (Math.Abs(z) > 2) {
+            for (int j = 1; j < prices.Count - i; j++) {
+               double zNext = (prices[i + j] - mean) / stdDev;
+               if (Math.Abs(zNext) < 1) {
+                  decayTimes.Add(j);
+                  break;
+               }
+            }
+         }
+      }
+
+      return decayTimes.Count == 0 ? double.NaN : decayTimes.Average();
+   }
+
+   /// <summary>
+   /// Computes the variance ratio: short-term variance divided by long-term variance.
+   /// Lower ratio suggests stronger mean-reversion.
+   /// </summary>
+   public static double ComputeVarianceRatio(List<Tuple<double, double, double, double>> spreadData, int shortWindow, int longWindow) {
+      var prices = spreadData.Select(x => (x.Item1 + x.Item3) / 2.0).ToList();
+      if (prices.Count < longWindow) return double.NaN;
+
+      double shortVar = Statistics.Variance(prices.Skip(prices.Count - shortWindow).Take(shortWindow));
+      double longVar = Statistics.Variance(prices.Skip(prices.Count - longWindow).Take(longWindow));
+
+      return longVar == 0 ? double.NaN : shortVar / longVar;
+   }
+
+   /// <summary>
+   /// Computes the highest bid and the lowest ask from a spread tick series.
+   /// Useful for evaluating the best market opportunity range.
+   /// </summary>
+   /// <param name="spreadData">List of (bid, bidQty, ask, askQty)</param>
+   /// <returns>Tuple of (MaxBid, MinAsk)</returns>
+   public static (double MaxBid, double MinAsk) EvaluateMaxBidMinAsk(List<Tuple<double, double, double, double>> spreadData)
+{
+    if (spreadData == null || spreadData.Count == 0)
+        return (double.NaN, double.NaN);
+
+    double maxBid = double.NegativeInfinity;
+    double minAsk = double.PositiveInfinity;
+
+    foreach (var (bid, _, ask, _) in spreadData)
+    {
+        if (bid > maxBid) maxBid = bid;
+        if (ask < minAsk) minAsk = ask;
+    }
+
+    return (maxBid, minAsk);
+}
+
+/// <summary>
+/// Computes the distribution of spread mid-prices around the mean,
+/// grouped by caller-defined distance bands (in standard deviations).
+///
+/// This helps answer:
+/// - Is the spread tightly hugging the mean?
+/// - Are there wide excursions or heavy tails?
+///
+/// You define breakpoints like: [0.5, 1.0, 2.0] → it will return:
+/// - "≤ 0.5σ"
+/// - "0.5σ – 1.0σ"
+/// - "1.0σ – 2.0σ"
+/// - "> 2.0σ"
+///
+/// How to Interpret It:
+/// - ≤ 0.5σ        → Tight clustering near the mean (strong mean-reversion)
+/// - 0.5σ – 1.0σ   → Mild oscillation (common trade region)
+/// - 1.0σ – 2.0σ   → Moderate deviation (potential trade signals)
+/// - > 2.0σ        → Rare, extreme moves (tail events, potential entries/exits)
+///
+/// Use this to tune entry thresholds or detect regime shifts.
+/// </summary>
+/// <param name="spreadData">List of (bid, bidQty, ask, askQty)</param>
+/// <param name="regions">List of positive breakpoints in std deviations (e.g., [0.5, 1.0, 2.0])</param>
+/// <returns>Dictionary mapping label to % of mid-prices in that region</returns>
+public static Dictionary<string, double> EvaluateDistributionAroundMean(
+    List<Tuple<double, double, double, double>> spreadData,
+    List<double> regions)
+{
+    var result = new Dictionary<string, double>();
+
+    if (spreadData == null || spreadData.Count < 2 || regions == null || regions.Count == 0)
+        return result;
+
+    var mids = spreadData.Select(x => (x.Item1 + x.Item3) / 2.0).ToList();
+    double mean = mids.Average();
+    double stddev = Math.Sqrt(mids.Select(x => Math.Pow(x - mean, 2)).Average());
+
+    if (stddev == 0)
+    {
+        result["Zero Volatility"] = 100.0;
+        return result;
+    }
+
+    var buckets = new int[regions.Count + 1];
+    foreach (var mid in mids)
+    {
+        double distance = Math.Abs(mid - mean);
+        bool placed = false;
+
+        for (int i = 0; i < regions.Count; i++)
+        {
+            if (distance <= regions[i])
+            {
+                buckets[i]++;
+                placed = true;
+                break;
+            }
+        }
+
+        if (!placed)
+            buckets[regions.Count]++;
+    }
+
+    int total = mids.Count;
+    for (int i = 0; i < regions.Count; i++)
+    {
+        string label = i == 0
+            ? $"≤ {regions[i]:0.##}σ"
+            : $"{regions[i - 1]:0.##}σ – {regions[i]:0.##}σ";
+        result[label] = 100.0 * buckets[i] / total;
+    }
+
+    result[$"> {regions.Last():0.##}σ"] = 100.0 * buckets.Last() / total;
+
+    return result;
+}
+
+/// <summary>
+/// Computes two drift metrics indicating how much the spread's mean changes over time,
+/// based on the standard deviation of rolling mean values.
+///
+/// ✅ Absolute Drift: Standard deviation of the rolling mean
+/// ✅ Relative Drift: Drift normalized by the overall mean (unitless ratio)
+///
+/// This shows whether the center of the spread is stable — critical for mean-reversion.
+///
+/// How to Interpret the Output:
+/// - ✅ Relative Drift < 1% → Very stable mean — ideal
+/// - ⚠️ 1–5% → Mild drift — consider shorter holds or adaptive mean
+/// - ❌ >5% → Drifting mean — fixed mean may no longer work
+/// </summary>
+/// <param name="spreadData">List of (bid, bidQty, ask, askQty)</param>
+/// <param name="window">Window size for rolling mean (e.g., 20 ticks)</param>
+/// <returns>
+/// Tuple of (Absolute Drift, Relative Drift) — where:
+/// - Absolute Drift = StdDev of rolling means
+/// - Relative Drift = Drift / global mean
+/// </returns>
+public static (double AbsoluteDrift, double RelativeDrift) EvaluateMeanDriftScore(
+    List<Tuple<double, double, double, double>> spreadData,
+    int window = 20)
+{
+    if (spreadData == null || spreadData.Count < window)
+        return (double.NaN, double.NaN);
+
+    var mids = spreadData.Select(x => (x.Item1 + x.Item3) / 2.0).ToArray();
+    var rollingMeans = new List<double>();
+
+    for (int i = window; i <= mids.Length; i++)
+    {
+        double rollingMean = mids.Skip(i - window).Take(window).Average();
+        rollingMeans.Add(rollingMean);
+    }
+
+
+    double overallMean = mids.Average(); // use full mean for normalization
+    //double drift = Math.Sqrt(rollingMeans.Select(m => Math.Pow(m - rollingMeans.Average(), 2)).Average());
+    double drift = Math.Sqrt(rollingMeans.Select(m => Math.Pow(m - rollingMeans.Average(), 2)).Average());
+
+
+double relativeDrift = Math.Abs(overallMean) > 1e-8 ? drift / Math.Abs(overallMean) : double.NaN;
+    return (drift, relativeDrift);
+}
+
+/// <summary>
+/// Returns the latest Kalman Filter estimate of the true mean of the spread mid-prices.
+/// This tracks the spread's central value while allowing for gradual drift and suppressing noise.
+///
+/// Internally, this applies a recursive 1D Kalman filter over the input mid-prices.
+/// 
+/// Tuning Parameters:
+/// - Q: Process noise variance (higher = faster adaptation to drift)
+/// - R: Measurement noise variance (higher = more smoothing)
+/// 
+/// Use this instead of SMA if your mean drifts or if price series is noisy.
+///
+/// </summary>
+/// <param name="spreadData">List of (bid, bidQty, ask, askQty)</param>
+/// <param name="Q">Process noise variance (default = 0.0001)</param>
+/// <param name="R">Measurement noise variance (default = 0.01)</param>
+/// <returns>Latest estimated Kalman-filtered mean, or NaN if input is invalid</returns>
+public static double GetLatestKalmanMean(
+    List<Tuple<double, double, double, double>> spreadData,
+    double Q = 0.0001,
+    double R = 0.01)
+{
+    if (spreadData == null || spreadData.Count == 0)
+        return double.NaN;
+
+    var mids = spreadData.Select(x => (x.Item1 + x.Item3) / 2.0).ToList();
+
+    double x = mids[0];  // initial estimate of mean
+    double P = 1.0;       // initial uncertainty
+
+    for (int i = 1; i < mids.Count; i++)
+    {
+        // Predict step
+        double x_prior = x;
+        double P_prior = P + Q;
+
+        // Update step
+        double K = P_prior / (P_prior + R);
+        x = x_prior + K * (mids[i] - x_prior);
+        P = (1 - K) * P_prior;
+    }
+
+    return x; // the latest estimated mean
+}
+
+
+
+
+}

src/DataStreamSuite/SharedStreams/Business/MarketData/Analytics/SpreadPerformanceStream.cs

@@ -0,0 +1,137 @@
+namespace bts.SilverPine {
+   using BCG;
+
+   using System.Diagnostics;
+
+   public class SpreadPerformanceStream : ViewStream {
+      ViewStream _spread_price_stream = null;
+      private readonly IBlueTableSchema _schema;
+      private readonly int _window_secs;
+      private readonly double _buy_threshold, _sell_threshold, _flat_threshold;
+      private ViewStream _bbo_stream;
+      public SpreadPerformanceStream(string s1, string s2, double w1, double w2, int window_secs,
+                                       double buy_threshold, double sell_threshold, double flat_threshold)
+      : this(s1, s2, w1, w2, window_secs, buy_threshold, sell_threshold, flat_threshold, ViewStreamDatabase.instance) { }
+
+      public SpreadPerformanceStream(string s1, string s2, double w1, double w2, int window_secs,
+                                      double buy_threshold, double sell_threshold, double flat_threshold,
+                                      ViewStreamDatabase vsd) {
+         _bbo_stream = new BBOStream(vsd);
+         _schema = BlueDataUtility.form_schema("SpreadPerformance", Tuple.Create("id", TypeCode.Int32),                                                
+                                                Tuple.Create("t0", TypeCode.DateTime),
+                                                Tuple.Create("t1", TypeCode.DateTime),
+                                                Tuple.Create("n_ticks", TypeCode.UInt32),
+                                                Tuple.Create("curr_windows_seconds", TypeCode.Double),
+                                                // means / std dev                                          
+                                                Tuple.Create("stddev", TypeCode.Double),
+                                                Tuple.Create("mean", TypeCode.Double),
+                                                Tuple.Create("kalman_filter_mean", TypeCode.Double),
+                                                Tuple.Create("absolute_mean_drift", TypeCode.Double),
+                                                Tuple.Create("relative_mean_drift", TypeCode.Double),
+                                                // statistical measures
+                                                Tuple.Create("hurst_exponent", TypeCode.Double),
+                                                Tuple.Create("half_life", TypeCode.Double),
+                                                Tuple.Create("lag1_autocorrelation", TypeCode.Double),
+                                                Tuple.Create("average_zscore_decay", TypeCode.Double),
+                                                Tuple.Create("variance_ratio", TypeCode.Double),
+                                                Tuple.Create("n_zero_crossings", TypeCode.UInt32),                                                
+                                                // distributions
+                                                Tuple.Create("Z ≤ 0.5σ", TypeCode.Double),
+                                                Tuple.Create("0.5σ – 1.0σ", TypeCode.Double),
+                                                Tuple.Create("1.0σ – 2.0σ", TypeCode.Double),
+                                                Tuple.Create("> 2.0σ", TypeCode.Double),
+                                                ////////////////////////////////////////                                                   
+                                                Tuple.Create("max_bid", TypeCode.Double),
+                                                Tuple.Create("min_ask", TypeCode.Double),
+                                                Tuple.Create("n_buy_signals", TypeCode.UInt32),
+                                                Tuple.Create("cumm_buy_price", TypeCode.Double),
+                                                Tuple.Create("n_sell_signals", TypeCode.UInt32),
+                                                Tuple.Create("cumm_sell_price", TypeCode.Double)
+                                                //Tuple.Create("pnl", TypeCode.Double),
+                                                //Tuple.Create("buy_turns", TypeCode.UInt32),
+                                                //Tuple.Create("sell_turns", TypeCode.UInt32),
+                                                //Tuple.Create("trades", TypeCode.String)
+                                                );
+         _window_secs = window_secs;
+         _buy_threshold = buy_threshold;
+         _sell_threshold = sell_threshold;
+         _flat_threshold = flat_threshold;
+         _spread_price_stream = new SyntheticSpreadStream(s1, s2, w1, w2, _bbo_stream, vsd);
+         _spread_price_stream = new TimeSeriesStream(_spread_price_stream);
+         _spread_price_stream = new ThrottleStream(_spread_price_stream, TimeSpan.FromSeconds(5.0), vsd);
+      }
+
+      protected override void myOnFirstSubscriber() {
+         _spread_price_stream.onChange += _handle_spread_prices;
+      }
+
+      protected override void myOnLastSubscriber() {
+         _spread_price_stream.onChange -= _handle_spread_prices;
+      }
+
+      private void _handle_spread_prices(object dp_obj) {
+         DataPacket dp = dp_obj as DataPacket;
+         List<Tuple<DateTime, double, double, double, double>> spread_ts_prices = new List<Tuple<DateTime, double, double, double, double>>();
+         DateTime min_time = DateTime.MaxValue; DateTime max_time = DateTime.MinValue;
+         var sw = Stopwatch.StartNew();
+         sw.Start();
+         foreach (IBlueRow r in dp.newRows) {
+            DateTime ts = r.get_datetime("timestamp");
+            TimeSpan elapsed_time = DateTime.Now - ts;
+            if (elapsed_time.TotalSeconds >= _window_secs)
+               continue;
+            spread_ts_prices.Add(Tuple.Create(
+                                          r.get_datetime("timestamp"),
+                                          r.get<double>("spread_bid_price"),
+                                          r.get<double>("spread_bid_sz"),
+                                          r.get<double>("spread_ask_price"),
+                                          r.get<double>("spread_ask_sz")));
+            if (ts <= min_time) {
+               min_time = ts;
+            }
+            if (ts >= max_time) {
+               max_time = ts;
+            }
+         }
+         List<Tuple<double, double, double, double>> spread_prices = spread_ts_prices
+                                                                                             .OrderBy(t => t.Item1)
+                                                                                             .Select(t => Tuple.Create(t.Item2, t.Item3, t.Item4, t.Item5))
+                                                                                             .ToList();
+
+         uint n_ticks = (uint)spread_prices.Count;
+         double curr_windows_seconds = (max_time - min_time).TotalSeconds;
+         double hurst_exponent = SpreadPerformanceEvaluator.EvaluateHurstExponent(spread_prices);
+         double half_life = SpreadPerformanceEvaluator.ComputeHalfLife (spread_prices);
+         double lag1_autocorrelation = SpreadPerformanceEvaluator.ComputeLag1Autocorrelation(spread_prices);
+         double average_zscore_decay = SpreadPerformanceEvaluator.ComputeAverageZScoreDecay(spread_prices);
+         double variance_ratio = SpreadPerformanceEvaluator.ComputeVarianceRatio(spread_prices, 60, 360);
+
+         (double mean, double stddev) = SpreadPerformanceEvaluator.EvaluateMeanAndStdDev(spread_prices);
+         double kalman_filter_mean = SpreadPerformanceEvaluator.GetLatestKalmanMean(spread_prices);
+         uint n_zero_crossings = SpreadPerformanceEvaluator.EvaluateZeroCrossingCount(spread_prices);
+         (double max_bid, double min_ask) = SpreadPerformanceEvaluator.EvaluateMaxBidMinAsk(spread_prices);
+         (uint n_buy_signals, double cumm_buy_price, uint n_sell_signals, double cumm_sell_price) = SpreadPerformanceEvaluator.EvaluateExcursionCount(spread_prices, _buy_threshold);
+         //(double pnl, List<SpreadPerformanceEvaluator.TradeAction> trades, int buy_turns, int sell_turns) = SpreadPerformanceEvaluator.EvaluateSimulatedPnL(spread_prices, 30, _buy_threshold, _flat_threshold);
+         //string trade_str = String.Empty;
+         //foreach (var tr in trades) {
+         //   trade_str += tr.ToString() + " , ";
+         //}
+         Dictionary<string, double> dist_map = SpreadPerformanceEvaluator.EvaluateDistributionAroundMean(spread_prices,
+                                                                       new List<double>(new double[] { 0.5, 1.0, 2.0 }));
+         var dist_vals = dist_map.Count < 4 ? new double[] { Double.NaN, Double.NaN, Double.NaN, Double.NaN } : dist_map.Values.ToArray();
+         (double absolute_drift, double relative_drift) = SpreadPerformanceEvaluator.EvaluateMeanDriftScore(spread_prices);
+         IBlueRow nr = new BlueRow(_schema, 0, min_time, max_time, n_ticks, curr_windows_seconds, stddev, mean, kalman_filter_mean, absolute_drift, relative_drift,
+            hurst_exponent, half_life, lag1_autocorrelation, average_zscore_decay, variance_ratio, n_zero_crossings,
+                                    dist_vals[0], dist_vals[1], dist_vals[2], dist_vals[3],
+                                    max_bid, min_ask, n_buy_signals, cumm_buy_price, n_sell_signals, cumm_sell_price);
+         BlueTable t = new BlueTable(_schema);
+         t.addRow(nr);
+         var dp_out = BlueDataUtility.build_dp_inc(t, myLastValue as DataPacket, true);
+         var perf_msecs = sw.ElapsedMilliseconds;
+         sw.Stop();
+         Log.Gen.info($"SpreadPerformanceStream._handle_spread_prices comp_time: {perf_msecs} msecs");
+         myFireEvent(dp_out);
+
+      }
+   }
+}

src/DataStreamSuite/SharedStreams/Business/MarketData/Analytics/SpreadStatArbsWeightLinearRegressionStream.cs

@@ -0,0 +1,68 @@
+namespace bts.SilverPine{
+   using BCG;
+   public class SpreadStatArbsWeightLinearRegressionStream : ViewStream {
+      private IBlueTableSchema _schema;
+      private ViewStream _leg_price_stream;
+      private readonly double _window_secs;
+      private readonly string _s1, _s2;
+
+      public SpreadStatArbsWeightLinearRegressionStream(string s1, string s2, double window_secs, double sample_period_secs)
+         : this(s1, s2, window_secs, sample_period_secs, ViewStreamDatabase.instance) { }
+
+
+      public SpreadStatArbsWeightLinearRegressionStream(string s1, string s2, double window_secs, double sample_period_secs, ViewStreamDatabase vsd) {
+         _window_secs = window_secs;
+         _s1 = s1;
+         _s2 = s2;
+         _schema = BlueDataUtility.form_schema("LinearRegressionWeights",
+                                                Tuple.Create("id", TypeCode.Int32),
+                                                Tuple.Create("t0", TypeCode.DateTime),
+                                                Tuple.Create("t1", TypeCode.DateTime),
+                                                Tuple.Create("curr_windows_seconds", TypeCode.Double),
+                                                Tuple.Create("n_ticks", TypeCode.UInt32),
+                                                Tuple.Create("s1", TypeCode.String),
+                                                Tuple.Create("w1", TypeCode.Double),
+                                                Tuple.Create("s2", TypeCode.String),
+                                                Tuple.Create("w2", TypeCode.String));
+
+         _leg_price_stream = new TimeSeriesStream(new SyntheticSpreadLegsStream(s1, s2));
+         _leg_price_stream = new ThrottleStream(_leg_price_stream, TimeSpan.FromSeconds(1), vsd);
+      }
+
+      protected override void myOnFirstSubscriber() {
+         _leg_price_stream.onChange += _handle_leg_prices;
+      }
+
+      protected override void myOnLastSubscriber() {
+         _leg_price_stream.onChange -= _handle_leg_prices;
+      }
+
+      private void _handle_leg_prices(object dp_obj){
+         DataPacket dp = dp_obj as DataPacket;
+         List<Tuple<double, double, DateTime>> leg_prices = new List<Tuple<double, double, DateTime>>();
+         foreach (IBlueRow r in dp.newRows){
+            double s1_mid = (r.get<double>("s1_bid_price") + r.get<double>("s1_ask_price")) /2.0;
+            double s2_mid = (r.get<double>("s2_bid_price") + r.get<double>("s2_ask_price")) /2.0;
+            DateTime ts = r.get_datetime("timestamp");
+            leg_prices.Add(new Tuple<double, double, DateTime>(s1_mid, s2_mid, ts));
+         }
+         if (leg_prices.Count <= 2)
+            return;
+         leg_prices = leg_prices.Where(t => (DateTime.Now - t.Item3).TotalSeconds <= _window_secs).ToList();
+         DateTime t0 = leg_prices.Select(t => t.Item3).Min();
+         DateTime t1 = leg_prices.Select(t => t.Item3).Max();
+         double curr_windows_seconds = (t1 - t0).TotalSeconds;
+         int n_ticks = leg_prices.Count;
+         List<double> s1_mids = leg_prices.Select(t => t.Item1).ToList();
+         List<double> s2_mids = leg_prices.Select(t => t.Item2).ToList();
+         (double w1, double w2) = SpreadWeightCalculator.CalculateWeightsLinearRegression(s1_mids, s2_mids);
+         BlueTable state = new BlueTable(_schema);
+         IBlueRow nr = new BlueRow(_schema, 0, t0, t1, curr_windows_seconds, n_ticks,_s1, w1, _s2, w2);
+         state.addRow(nr);
+         var dp_out = BlueDataUtility.build_dp_inc(state, myLastValue as DataPacket, true);
+         myFireEvent(dp_out);
+
+      }
+   }
+}
+

src/DataStreamSuite/SharedStreams/Business/MarketData/Analytics/SpreadStatsStream.cs

@@ -0,0 +1,143 @@
+namespace bts.SilverPine {
+   using BCG;
+
+   using MathNet.Numerics.LinearAlgebra.Factorization;
+
+   using System;
+   using System.Collections.Generic;
+   using System.Diagnostics;
+   using System.Linq;
+   using System.Text;
+   using System.Threading.Tasks;
+   [DataStreamUserRespresentation("SpreadStatsStream")]
+   // This is the fundamental stream that computes the stats (including buy and sell zones) 
+   // of a spread. the legs and the weights are passed as args
+   public class SpreadStatsStream : ViewStream {
+      private ViewStream _spread_price_stream;
+      private readonly IBlueTableSchema _schema;
+      private readonly TimeSpan _window_sz;
+      private readonly string _spread;
+      public SpreadStatsStream(string s1, string s2, double w1, double w2, int rolling_window_sz_seconds, int window_sampling_sz_secs)
+         : this(s1, s2, w1, w2, rolling_window_sz_seconds, window_sampling_sz_secs, new BBOStream(ViewStreamDatabase.instance), ViewStreamDatabase.instance) { }
+      public SpreadStatsStream(string s1, string s2, double w1, double w2, int rolling_window_sz_seconds, int window_sampling_sz_secs, ViewStream bbo_stream, ViewStreamDatabase vsd) {
+         _window_sz = TimeSpan.FromSeconds(rolling_window_sz_seconds);
+         _spread = $"{w1} X {s1} - {w2} X {s2}";
+         _schema = new BlueTableSchema("SpreaddStats", new ColumnInfo ("id", 0, TypeCode.Int32));
+         _schema = _schema.add_column(TypeCode.String, "spread");
+         _schema = _schema.add_column(TypeCode.String, "window");
+         // basic stats
+         _schema = _schema.add_column(TypeCode.Double, "bid_mean");
+         _schema = _schema.add_column(TypeCode.Double, "ask_mean");
+         _schema = _schema.add_column(TypeCode.Double, "bid_stddev");
+         _schema = _schema.add_column(TypeCode.Double, "ask_stddev");
+         _schema = _schema.add_column(TypeCode.Double, "bid_skewness");
+         _schema = _schema.add_column(TypeCode.Double, "ask_skewness");
+         _schema = _schema.add_column(TypeCode.Double, "bid_kurtosis");
+         _schema = _schema.add_column(TypeCode.Double, "ask_kurtosis");
+         _schema = _schema.add_column(TypeCode.Double, "bid_max");
+         _schema = _schema.add_column(TypeCode.Double, "ask_max");
+         _schema = _schema.add_column(TypeCode.Double, "bid_min");
+         _schema = _schema.add_column(TypeCode.Double, "ask_min");
+         // bid bins
+         _schema = _schema.add_column(TypeCode.Double, "bid Z < -2σ");
+         _schema = _schema.add_column(TypeCode.Double, "bid -2σ ≤ Z < -1σ");
+         _schema = _schema.add_column(TypeCode.Double, "bid -1σ ≤ Z < 0σ");
+         // positive z's
+         _schema = _schema.add_column(TypeCode.Double, "bid 0σ ≤ Z < 1σ");
+         _schema = _schema.add_column(TypeCode.Double, "bid 1σ ≤ Z < 2σ");
+         _schema = _schema.add_column(TypeCode.Double, "bid Z ≥ 2σ");
+         // ask bins
+         _schema = _schema.add_column(TypeCode.Double, "ask Z < -2σ");
+         _schema = _schema.add_column(TypeCode.Double, "ask -2σ ≤ Z < -1σ");
+         _schema = _schema.add_column(TypeCode.Double, "ask -1σ ≤ Z < 0σ");
+         // positive z's
+         _schema = _schema.add_column(TypeCode.Double, "ask 0σ ≤ Z < 1σ");
+         _schema = _schema.add_column(TypeCode.Double, "ask 1σ ≤ Z < 2σ");
+         _schema = _schema.add_column(TypeCode.Double, "ask Z ≥ 2σ");
+         // opps
+         _schema = _schema.add_column(TypeCode.Int32, "n_buy_ticks");
+         _schema = _schema.add_column(TypeCode.Double, "cumm_buy_price");
+         _schema = _schema.add_column(TypeCode.Int32, "n_sell_ticks");
+         _schema = _schema.add_column(TypeCode.Double, "cumm_sell_price");
+         _schema = _schema.add_column(TypeCode.Int32, "n_trade_ticks");
+         _spread_price_stream = new SyntheticSpreadStream(s1, s2, w1, w2, bbo_stream, vsd);
+         _spread_price_stream = new TimeSeriesStream(_spread_price_stream);
+         double window_sampling_sz_smsecs = window_sampling_sz_secs * 1000.0 + new Random(DateTime.Now.Second).Next(-100, 100);
+         TimeSpan sampling_period = TimeSpan.FromMilliseconds(window_sampling_sz_smsecs);
+         _spread_price_stream = new ThrottleStream(_spread_price_stream, sampling_period, vsd);
+      }
+
+      protected override void myOnFirstSubscriber() {
+         _spread_price_stream.onChange += _handle_spread_prices;
+      }
+
+      protected override void myOnLastSubscriber() {
+         _spread_price_stream.onChange -= _handle_spread_prices;
+      }
+
+      private void _handle_spread_prices(object dp_obj) {
+         List<IBlueRow> rows = (dp_obj as DataPacket).newRows.to_list();
+         Func<IBlueRow, bool> row_filter = (r) => {
+            DateTime ts = r.get_datetime("timestamp");
+            DateTime start_time = DateTime.Now.Subtract(_window_sz);
+            return ts >= start_time;
+         };
+         var rolling_win_rows = rows.Where(r => row_filter(r));
+         var spread_prices = rolling_win_rows.Select(r => (
+                                          r.get_datetime("timestamp"),
+                                          r.get<double>("spread_bid_price"),
+                                          r.get<double>("spread_bid_sz"),
+                                          r.get<double>("spread_ask_price"),
+                                          r.get<double>("spread_ask_sz"))).ToList();
+         spread_prices.Sort((a,b) => a.Item1.CompareTo(b.Item1));
+         DateTime t0 = spread_prices.First().Item1, t1 = spread_prices.Last().Item1;
+         var tot_secs = (t1 - t0).TotalSeconds;
+         var stopwatch = Stopwatch.StartNew();
+         Log.Gen.info($"SpreadStatsStream.analyze_spread_quotes executed in {stopwatch.ElapsedMilliseconds} ms");
+         var result = SpreadPerformanceEvaluator.SpreadStatistics.analyze_spread_quotes(spread_prices);
+         (uint n_buy_signals, double cumm_buy_price, uint n_sell_signals, double cumm_sell_price) = 
+              SpreadPerformanceEvaluator.EvaluateExcursionCount(spread_prices.Select(t => Tuple.Create(t.Item2, t.Item3, t.Item4, t.Item5)).ToList(), 2.0);
+         IBlueRow nr = new BlueRow(_schema,
+            0,
+            _spread,
+            $"n_ticks: {spread_prices.Count} t0: {t0.ToLongTimeString()} -> t1: {t1.ToLongTimeString()} ({tot_secs}) secs",
+            result.bid_stats.Mean,
+            result.ask_stats.Mean,
+            result.bid_stats.StandardDeviation,
+            result.ask_stats.StandardDeviation,
+            result.bid_stats.Skewness,
+            result.ask_stats.Skewness,
+            result.bid_stats.Kurtosis,
+            result.ask_stats.Kurtosis,
+            result.bid_stats.Maximum,
+            result.ask_stats.Maximum,
+            result.bid_stats.Minimum,
+            result.ask_stats.Minimum,
+            // bid distribution
+            result.bid_zscore_distribution["Z < -2σ"],
+            result.bid_zscore_distribution["-2σ ≤ Z < -1σ"],
+            result.bid_zscore_distribution["-1σ ≤ Z < 0σ"],
+            result.bid_zscore_distribution["0σ ≤ Z < 1σ"],
+            result.bid_zscore_distribution["1σ ≤ Z < 2σ"],
+            result.bid_zscore_distribution["Z ≥ 2σ"],
+            // ask distributions
+            result.ask_zscore_distribution["Z < -2σ"],
+            result.ask_zscore_distribution["-2σ ≤ Z < -1σ"],
+            result.ask_zscore_distribution["-1σ ≤ Z < 0σ"],
+            result.ask_zscore_distribution["0σ ≤ Z < 1σ"],
+            result.ask_zscore_distribution["1σ ≤ Z < 2σ"],
+            result.ask_zscore_distribution["Z ≥ 2σ"],
+            n_buy_signals,
+            cumm_buy_price,
+            n_sell_signals,
+            cumm_sell_price,
+            n_buy_signals + n_sell_signals
+            );
+         stopwatch.Stop();
+         BlueTable t = new BlueTable(_schema);
+         t.addRow(nr);
+         var dp_out = BlueDataUtility.build_dp(t, myLastValue as DataPacket);
+         myFireEvent(dp_out);
+      }
+   }
+}

src/DataStreamSuite/SharedStreams/Business/MarketData/Analytics/SpreadWeightCalculator.cs

@@ -0,0 +1,49 @@
+using System;
+using System.Collections.Generic;
+using MathNet.Numerics.LinearRegression;
+
+namespace bts.SilverPine {
+
+   public static class SpreadWeightCalculator {
+      /// <summary>
+      /// Estimates hedge weights (w1, w2) such that the spread s = w1 * s1 - w2 * s2 
+      /// is potentially mean-reverting. Uses linear regression: s1 = beta * s2 + ε.
+      /// 
+      /// Assumes mid-prices are precomputed and aligned over the same time window.
+      /// </summary>
+      /// <param name="s1MidPrices">List of mid-prices for asset 1 (dependent variable)</param>
+      /// <param name="s2MidPrices">List of mid-prices for asset 2 (independent variable)</param>
+      /// <param name="includeIntercept">
+      /// Whether to include an intercept in the regression model (default: true).
+      /// </param>
+      /// <returns>(w1, w2): Tuple of hedge weights</returns>
+      public static (double w1, double w2) CalculateWeightsLinearRegression(
+          List<double> s1MidPrices,
+          List<double> s2MidPrices,
+          bool includeIntercept = true) {
+         if (s1MidPrices == null || s2MidPrices == null)
+            throw new ArgumentNullException("Price series cannot be null.");
+         if (s1MidPrices.Count != s2MidPrices.Count)
+            throw new ArgumentException("Price series must be of equal length.");
+         if (s1MidPrices.Count < 2)
+            throw new ArgumentException("Need at least two data points for regression.");
+
+         double[] x = s2MidPrices.ToArray(); // Independent variable
+         double[] y = s1MidPrices.ToArray(); // Dependent variable
+
+         double beta;
+
+         if (includeIntercept) {
+            // Returns (intercept, slope)
+            (double intercept, double slope) = SimpleRegression.Fit(x, y);
+            beta = slope;
+         } else {
+            // Returns only slope when forcing regression through origin
+            beta = SimpleRegression.FitThroughOrigin(x, y);
+         }
+
+         // Return fixed w1 = 1.0 and w2 = beta
+         return (1.0, beta);
+      }
+   }
+}

src/DataStreamSuite/SharedStreams/Business/MarketData/Analytics/VWAPStream.cs

@@ -100,9 +100,9 @@ namespace bts.SilverPine {
       this(symbol_or_contract, beg_dt, end_dt, ViewStreamDatabase.instance) { }
 
       public VWAPStream(string symbol_or_contract, DateTime beg_dt, DateTime end_dt, ViewStreamDatabase db) {
-         var fc = new ContractUtils.FutureContract(symbol_or_contract);
-         ViewStream r = new FutureHistoricalBarsStream(fc, BarSize.ONE_DAY, new DateTimeRange(beg_dt, end_dt));
-
+         //var fc = new ContractUtils.FutureContract(symbol_or_contract);
+         ViewStream r = new FutureHistoricalBarsStream(symbol_or_contract, beg_dt.ToString(), end_dt.ToString());
+         // since this is already filtered, we can groupBy iit without a groupBy column
          r = new MathStream(r,
                  "vprice = volume * close");
          r = new NonIncGroupByStream(r,

src/DataStreamSuite/SharedStreams/Business/MarketData/BBOStream.cs

@@ -13,7 +13,6 @@ namespace bts.SilverPine {
    using Spark.Api.MarketData;
 
    using SecurityKey = System.UInt32;
-   using System.Data.Entity.Infrastructure;
 
    public class FakeBBOStream : ViewStream, IWorker {
       private string _symbol;
@@ -105,14 +104,14 @@ namespace bts.SilverPine {
    public class BBOStream : ViewStream, IWorker {
       private readonly TimeSpan _poll_period = TimeSpan.FromMilliseconds(500);
       private DateTime _last_poll_time = DateTime.MinValue;
-      private ViewStreamDatabase _vsd;
-      private int _id = 0;
-      IReadOnlyDictionary<SecurityKey, BookTuple> _cached_state = null;
-      MarketDataClient _md_client;
+      private ViewStreamDatabase? _vsd;
+      IReadOnlyDictionary<SecurityKey, BookTuple>? _cached_state = null;
+      MarketDataClient? _md_client;
       internal static IBlueTableSchema _bbo_schema = BlueDataUtility.form_schema("bbo",
                                                 Tuple.Create("id", TypeCode.Int32),
                                                 Tuple.Create("sec_key", TypeCode.UInt32),
                                                 Tuple.Create("sec", TypeCode.String),
+                                                Tuple.Create("exchange", TypeCode.String),
                                                 Tuple.Create("bid_price", TypeCode.Double),
                                                 Tuple.Create("bid_sz", TypeCode.Single),
                                                 Tuple.Create("ask_price", TypeCode.Double),
@@ -124,17 +123,18 @@ namespace bts.SilverPine {
       }
 
       public BBOStream(ViewStreamDatabase vsd) {
-         // bts.Log.Gen.info("BBOStream.cons");
-         // _vsd = vsd;
-         // _md_client = new MarketDataClient(new MDSource(TimeSpan.FromSeconds(0.5)));
-         FakeBBOStream rs = new FakeBBOStream("Future", "CME", "ES", 5000, 0.25, vsd);
-         mySetRealSource(rs);
+         bts.Log.Gen.info("BBOStream.cons");
+         _vsd = vsd;
+         _md_client = new MarketDataClient(new MDSource(TimeSpan.FromSeconds(0.5)));
+         //FakeBBOStream rs = new FakeBBOStream("Future", "CME", "ES", 5000, 0.25, vsd);
+         //mySetRealSource(rs);
       }
 
       protected override void myOnFirstSubscriber() {
          _vsd.addWorker(this);
          _md_client.books().Subscribe(_on_books);
-
+         _vsd?.addWorker(this);
+         _md_client?.books().Subscribe(_on_books);
       }
 
       private void _log_updates(TableUpdate<BookTuple, uint> update) {
@@ -144,24 +144,24 @@ namespace bts.SilverPine {
          }
          foreach (Change<BookTuple> c in update.changes) {
             var bt = c.New;
-            bts.Log.Gen.info($"BBOStream changes: idx: {i} bid: {bt.book.bids[0].price} / {bt.book.bids[0].size} ask: {bt.book.asks[0].price} / {bt.book.asks[0].size}");
+            //bts.Log.Gen.info($"BBOStream changes: idx: {i} bid: {bt.book.bids[0].price} / {bt.book.bids[0].size} ask: {bt.book.asks[0].price} / {bt.book.asks[0].size}");
          }
       }
 
       private void _on_books(TableUpdate<BookTuple, uint> update) {
          _log_updates(update);
          // just cacching in here, will populate the rows in _poll_scan and fire the datapacket
-         bts.Log.Gen.info($"BBOStream._on_books state: {update.state.Count()} changes: {update.changes.Count()} added: {update.added.Count()}");
+         //bts.Log.Gen.info($"BBOStream._on_books state: {update.state.Count()} changes: {update.changes.Count()} added: {update.added.Count()}");
          _cached_state = update.state;
       }
 
       protected override void myOnLastSubscriber() {
-         _vsd.removeWorker(this);
+         _vsd?.removeWorker(this);
       }
 
       private void _scan_bbo() {
          // this shouldn be made incremental using the TableUpdate
-         bts.Log.Gen.info("BBOStream._scan_bbo");
+         //bts.Log.Gen.info("BBOStream._scan_bbo");
          if (_cached_state == null)
             return;
          BlueTable new_state = new BlueTable(_bbo_schema);
@@ -170,22 +170,23 @@ namespace bts.SilverPine {
             var sec = kv.Value.security;
             int id = Convert.ToInt32(skey);
             BookTuple bt = kv.Value;
-            IBlueRow r = new BlueRow(_bbo_schema, id, skey, sec.ToString(), bt.book.bids[0].price, bt.book.bids[0].size,
+            var venue = sec.Exchange;
+            IBlueRow r = new BlueRow(_bbo_schema, id, skey, sec.ToString(), venue.ToString(), bt.book.bids[0].price, bt.book.bids[0].size,
                                      bt.book.asks[0].price, bt.book.asks[0].size);
             new_state.addRow(r);
          }
-         DataPacket dp = null;
-         if (myLastValue == null) {
-            dp = new DataPacket(new_state, new_state, new BlueTable(_bbo_schema));
-         } else {
-            DataPacket old_dp = myLastValue as DataPacket;
-            IBlueRowCollection added_rows, del_rows;
-            BlueDataUtility.compute_differences_fast(new_state, old_dp.newRows, out del_rows, out added_rows);
-            dp = new DataPacket(new_state, added_rows, del_rows);
-         }
+         var dp = accumulate(myLastValue as DataPacket, new_state);
          if (dp.addedRows.Count == 0 && dp.deletedRows.Count == 0)
             return;
          myFireEvent(dp);
+
+         static DataPacket accumulate(DataPacket? old_dp, BlueTable new_state) {
+            if (old_dp is null) {
+               return new(new_state, new_state, new BlueTable(_bbo_schema));
+            }
+            BlueDataUtility.compute_differences_fast(new_state, old_dp.newRows, out var del_rows, out var added_rows);
+            return new(new_state, added_rows, del_rows);
+         }
       }
 
       public void doWork() {
@@ -195,4 +196,4 @@ namespace bts.SilverPine {
          }
       }
    }
-}
+}

src/DataStreamSuite/SharedStreams/Business/MarketData/Bars/VolumeAggregationStream.cs

@@ -43,4 +43,11 @@ namespace bts.SilverPine {
          mySetRealSource(r);
       }
    }
+
+   [DataStreamUserRespresentation("PeriodicVolumeAgg")]
+   public class PeriodicVolumeAggregationStream : ViewStream {
+      public PeriodicVolumeAggregationStream(string future_contract, string contract_type, DateTime range_beg, DateTime range_end,
+                                             TimeSpan period, ViewStreamDatabase db) {
+      }
+   }
 }

src/DataStreamSuite/SharedStreams/Business/MarketData/BookStream.cs

@@ -21,7 +21,7 @@ namespace BCG {
       private DateTime _last_poll_time = DateTime.MinValue;
       private ViewStreamDatabase _vsd;
       private int _id = 0;
-      IReadOnlyDictionary<SecurityKey, BookTuple> _cached_state = null;
+      IReadOnlyDictionary<SecurityKey, BookTuple>? _cached_state = null;
       MarketDataClient _md_client;
       readonly Stopwatch _timer = new();
       private readonly IBlueTableSchema _bbo_schema = BlueDataUtility.form_schema("books",
@@ -73,14 +73,8 @@ namespace BCG {
 
       private Dictionary<SecurityKey, Dictionary<int, int>> _id_map = new Dictionary<SecurityKey, Dictionary<int, int>>();
       private int _get_id(SecurityKey skey, int depth) {
-         Dictionary<int, int> d_map = null;
-         if (!_id_map.TryGetValue(skey, out d_map)) {
-            d_map = new Dictionary<int, int>();
-            d_map.Add(depth, _id++);
-            _id_map.Add(skey, d_map);
-         }
-         int rv = 0;
-         if (!d_map.TryGetValue(depth, out rv)) {
+         var d_map = _id_map.GetOrAdd(skey, _ => new() { {depth, _id++} });
+         if (!d_map.TryGetValue(depth, out int rv)) {
             rv = _id++;
             d_map.Add(depth, rv);
          }

src/DataStreamSuite/SharedStreams/Business/MarketData/FuturesActiveContractStream.cs

@@ -90,20 +90,15 @@ namespace bts.SilverPine {
       // the argument is symbol, like "ES" etc and not the contract itself
       public FuturesContinousContractStream(string symbol) : this(symbol, ViewStreamDatabase.instance) { }
       public FuturesContinousContractStream(string symbol, ViewStreamDatabase db) {
+         symbol = symbol.EndsWith("*") ? symbol : symbol + "*";
          ViewStream r = new FuturesHistorical1DayBarsStream(symbol);
-         // filtering for futures only
-         r = new FilterStream(r, r => {
-            string contract = r.get<string>("symbol");
-            var fc = new ContractUtils.FutureContract(contract);
-            return fc.future_tradable_contract != null;
-         });
-         r = new NonIncGroupByStream(r,
-                                      "ts_event",
-                                      "volume:Max");
-         r = new NewProjectStream(r,
-                                   "ts_event symbol !");
-         r = new SortStream(r,
-                             "-ts_event");
+         r = new ContractExpanderStream(r, "symbol");
+         r = new FilterStream(r, "contract_type='Future'");
+         // finding the max volume across symbols, if we need to do this for open_interest, then join here 
+         // with the open interest stream on symbol/contract
+         r = new NonIncGroupByStream(r, "ts_event", "volume:Max");
+         r = new NewProjectStream(r, "ts_event symbol !");
+         r = new SortStream(r, "-ts_event");
          mySetRealSource(r);
       }
    }

src/DataStreamSuite/SharedStreams/Business/MarketData/FuturesRollStream.cs

@@ -2,15 +2,14 @@ namespace bts.SilverPine {
    using bts;
    using BCG;
 
-   // it uses the volume - better to use open interst perhap - to detect the rollover, is when a new 
-   // futuere exp becomes the highest volume one
+   // it uses the volume (better to use open interest perhap) to detect the rollover, ie when a new 
+   // future exp becomes the highest volume one
    [DataStreamUserRespresentation("FuturesRoll")]
    public class FuturesRollStream : ViewStream {
       public FuturesRollStream(string future_symbol) {
-         //ViewStream ohlcv_str = new OHLCVStream(future_symbol);
-         ViewStream rs = new FuturesActiveContractStream(future_symbol);
-         rs = new ColumnEvolutionStream(rs, "ts_event", "contract");
-         rs = new NewProjectStream(rs, "ts_event:roll_date !");
+         ViewStream rs = new FuturesContinousContractStream(future_symbol);
+         rs = new ColumnEvolutionStream(rs, "ts_event", "symbol");
+         rs = new NewProjectStream(rs, "symbol ts_event:roll_date !");
          mySetRealSource(rs);
       }
    }

src/DataStreamSuite/SharedStreams/Business/MarketData/InstrumentsStream.cs

@@ -1,8 +1,9 @@
-namespace BCG {
+namespace bts.SilverPine {
    using BCG.Utilities.General;
 
    using bts.api;
    using bts.md;
+   using BCG;
 
    using Spark.Core.Reactive;
 
@@ -31,6 +32,7 @@ namespace BCG {
                                                 Tuple.Create("sec_type", TypeCode.String),
                                                 Tuple.Create("venue", TypeCode.String),
                                                 Tuple.Create("symbol", TypeCode.String),
+                                                Tuple.Create("currency", TypeCode.String),
                                                 Tuple.Create("expiration", TypeCode.DateTime),
                                                 Tuple.Create("strike", TypeCode.Double),
                                                 Tuple.Create("callput", TypeCode.String)
@@ -81,7 +83,7 @@ namespace BCG {
          foreach (var kv in _cached_state) {
             SecurityKey skey = kv.Key;
             int id = Convert.ToInt32(skey);
-            SecurityImmutable sec = kv.Value;
+            SecurityImmutable sec = kv.Value;            
             bts.api.SecurityType sec_type = sec.security_type();
             object exp = DBNull.Value;
             if (sec_type == bts.api.SecurityType.Future || sec_type == bts.api.SecurityType.Option)
@@ -92,7 +94,7 @@ namespace BCG {
             object cp = DBNull.Value;
             if (sec_type == bts.api.SecurityType.Option)
                cp = sec.Option.CallPut.ToString();
-            IBlueRow r = new BlueRow(_instrument_schema, id, skey, sec.security_desc(), sec.security_type().ToString(), sec.display_exchange().ToString(), sec.Symbol, exp, strike, cp);
+            IBlueRow r = new BlueRow(_instrument_schema, id, skey, sec.security_desc(), sec.security_type().ToString(), sec.display_exchange().ToString(), sec.Symbol, sec.Currency, exp, strike, cp);
             new_state.addRow(r);
          }
          DataPacket dp = null;
@@ -117,4 +119,4 @@ namespace BCG {
          }
       }
    }
-}
+}

src/DataStreamSuite/SharedStreams/Business/MarketData/OHLCVStream.cs

@@ -174,10 +174,10 @@ namespace bts.SilverPine {
                // TODO: assuming 1 day only for historic               
                string table_name = get_table_from_barsize(ohlcv_params.bar_size, symbol);// $"{symbol}_OHLCV_1DAY";
                                                                                          //string table_name = $"{symbol}_{ohlcv_params.bar_size}";
-               string db_str = $"Data Source={_location}{_db_name};Version=3";
+               string db_name = _db_name;// $"Data Source={_location}{_db_name};Version=3";
                string sql_str = $"select * from {table_name}";
                // "Data Source=C:\\Users\\m_tah\\development\\data\\stream_data.db;Version=3;"
-               ViewStream rs = new SQLiteStream(db_str, sql_str);
+               ViewStream rs = new SQLiteStream(db_name, sql_str);
                rs = new SchemaGuesserStream(rs);
                if (ohlcv_params.bar_size == OHLCVBarSize.ONE_DAY)
                   rs = new DateStandardizationStream(rs, "ts_event", DateConversion.ChopTime);
@@ -207,4 +207,4 @@ namespace bts.SilverPine {
 
 
    }
-}
+}

src/DataStreamSuite/SharedStreams/Business/MarketData/SettlementPriceStream.cs

@@ -2,21 +2,21 @@ namespace bts.SilverPine {
    using BCG;
 
    using bts;
-
+   // used directly in blueorb
    [DataStreamUserRespresentation("Settlement")]
    public class SettlementPriceStream : ViewStream {
 
-      public SettlementPriceStream(string future_name, string date_txt)
-      : this(future_name, date_txt, date_txt) { }
+      // public SettlementPriceStream(string future_name, string date_txt)
+      // : this(future_name, date_txt, date_txt) { }
 
-      public SettlementPriceStream(string future_name, string begin_dt, string end_dt)
-      : this(future_name, DateTime.Parse(begin_dt).Date, DateTime.Parse(end_dt).Date) { }
+      // public SettlementPriceStream(string future_name, string begin_dt, string end_dt)
+      // : this(future_name, DateTime.Parse(begin_dt).Date, DateTime.Parse(end_dt).Date) { }
 
-      public SettlementPriceStream(string symbol_or_contract, DateTime begin_dt)
-      : this(symbol_or_contract, begin_dt, begin_dt) { }
+      public SettlementPriceStream(string future_contract, DateTime begin_dt)
+      : this(future_contract, begin_dt, begin_dt) { }
 
-      public SettlementPriceStream(string symbol_or_contract, DateTime begin_dt, DateTime end_dt) {
-         ContractUtils.FutureContract fc = new ContractUtils.FutureContract(symbol_or_contract);
+      public SettlementPriceStream(string future_contract, DateTime begin_dt, DateTime end_dt) {
+         ContractUtils.FutureContract fc = new ContractUtils.FutureContract(future_contract);
          ViewStream rs = new FutureHistoricalBarsStream(fc, BarSize.ONE_DAY, new DateTimeRange(begin_dt, end_dt));
          rs = new NewProjectStream(rs, "symbol ts_event:date close:settlement !");
          mySetRealSource(rs);

src/DataStreamSuite/SharedStreams/Business/MarketData/SyntheticSpreadStream.cs

@@ -0,0 +1,125 @@
+using System;
+using BCG;
+
+namespace bts.SilverPine
+{
+/*
+   ----------------------------------------------------------------------
+                               *Expression*
+   ----------------------------------------------------------------------
+       s1 = filter(bbo(), "sec = 'ETH-USDT-SWAP'")
+       s1 = addColumns(s1, "ljc:0")
+       s2 = filter(bbo(), "sec = 'ETH-USDT-SWAP'")
+       s2 = addColumns(s2, "rjc:0")
+       
+       s = fastJoin(s1, "ljc", "sec:s1 bid_price:s1_bid_price bid_sz:s1_bid_sz ask_price:s1_ask_price ask_sz:s1_ask_sz", 
+                    s2, "rjc", "sec:s2 bid_price:s2_bid_price bid_sz:s2_bid_sz ask_price:s2_ask_price ask_sz:s2_ask_sz")
+       
+
+   ----------------------------------------------------------------------
+                                 *Schema*
+   ----------------------------------------------------------------------
+       id                              System.Int32
+       s1                              System.String
+       s1_bid_price                    System.Double
+       s1_bid_sz                       System.Single
+       s1_ask_price                    System.Double
+       s1_ask_sz                       System.Single
+       s2                              System.String
+       s2_bid_price                    System.Double
+       s2_bid_sz                       System.Single
+       s2_ask_price                    System.Double
+       s2_ask_sz                       System.Single
+*/
+
+
+   // Combines the market data of spread legs into one stream, lined up nicely in a row.
+   // this is useful for operations like finding weights, that need leg market data organized together succintly.
+   public class SyntheticSpreadLegsStream : ViewStream 
+   {
+      public SyntheticSpreadLegsStream(string s1, string s2) : 
+         this(s1, s2, new BBOStream(ViewStreamDatabase.instance), ViewStreamDatabase.instance) {}
+
+      public SyntheticSpreadLegsStream(string s1, string s2, ViewStream bbo_stream, ViewStreamDatabase vsb)
+      {
+         ViewStream vs1;
+         ViewStream vs2;
+         ViewStream rs;
+
+         vs1 = new FilterStream (new BBOStream (),
+                                $"sec = '{s1}'");
+         vs1 = new AddColumnsStream (vs1,
+                                    "ljc:0");
+         vs2 = new FilterStream (bbo_stream,
+                                $"sec = '{s2}'");
+         vs2 = new AddColumnsStream (vs2,
+                                    "rjc:0");
+         rs = new NonIncJoinStream (vs1,
+                                   "ljc",
+                                   "sec:s1 bid_price:s1_bid_price bid_sz:s1_bid_sz ask_price:s1_ask_price ask_sz:s1_ask_sz",
+                                   vs2,
+                                   "rjc",
+                                   "sec:s2 bid_price:s2_bid_price bid_sz:s2_bid_sz ask_price:s2_ask_price ask_sz:s2_ask_sz");
+         mySetRealSource(rs);
+      }
+   }
+   public class SyntheticSpreadStream : ViewStream{
+      public SyntheticSpreadStream (string s1, string s2, double w1, double w2)
+      : this(s1, s2, w1, w2, new BBOStream(ViewStreamDatabase.instance), ViewStreamDatabase.instance){}
+      public SyntheticSpreadStream (string s1, string s2, double w1, double w2, ViewStream bbo_stream, ViewStreamDatabase vsd){
+         ViewStream rs = new SyntheticSpreadLegsStream(s1, s2, bbo_stream, vsd);
+
+         Func<IBlueTableSchema, IBlueTableSchema> get_schema = (inp_schema) => {
+            IBlueTableSchema rv = inp_schema;
+            rv = rv.add_column(TypeCode.Double, "spread_bid_price");
+            rv = rv.add_column(TypeCode.Double, "spread_bid_sz");
+            rv = rv.add_column(TypeCode.Double, "spread_ask_price");
+            rv = rv.add_column(TypeCode.Double, "spread_ask_sz");
+            return rv;
+         };
+
+         rs = new MapStream(rs, (inp_schema) => {
+           return get_schema(inp_schema);
+         }, 
+         (r) => {
+              double s1_bid_price = r.get<double>("s1_bid_price");
+              double s1_bid_sz = r.get<double>("s1_bid_sz");
+              double s1_ask_price = r.get<double>("s1_ask_price");
+              double s1_ask_sz = r.get<double>("s1_ask_sz");
+               
+              double s2_bid_price = r.get<double>("s2_bid_price");
+              double s2_bid_sz = r.get<double>("s2_bid_sz");
+              double s2_ask_price = r.get<double>("s2_ask_price");
+              double s2_ask_sz = r.get<double>("s2_ask_sz");
+               
+              (double spread_bid_price, double spread_ask_price, double spread_bid_sz, double spread_ask_sz) =   
+              ComputeSpreadBidAsk(s1_bid_price, s1_ask_price, s2_bid_price, s2_ask_price, s1_bid_sz, s1_ask_sz, s2_bid_sz, s2_ask_sz, w1, w2);
+              IBlueTableSchema out_schema = get_schema(r.schema);
+              MutableBlueRow rv = new MutableBlueRow(out_schema, r);
+              rv.setValue("spread_bid_price", spread_bid_price);
+              rv.setValue("spread_ask_price", spread_ask_price);
+              rv.setValue("spread_bid_sz", spread_bid_sz);
+              rv.setValue("spread_ask_sz", spread_ask_sz);
+              return rv;
+              }
+           );
+         mySetRealSource(rs);
+
+      }
+      public static (double SpreadBid, double SpreadAsk, double SpreadBidQty, double SpreadAskQty) ComputeSpreadBidAsk(
+        double s1Bid, double s1Ask,
+        double s2Bid, double s2Ask,
+        double s1BidQty, double s1AskQty,
+        double s2BidQty, double s2AskQty,
+        double w1, double w2)
+    {
+        double spreadBid = (w1 * s1Bid) - (w2 * s2Ask);
+        double spreadAsk = (w1 * s1Ask) - (w2 * s2Bid);
+
+        double spreadBidQty = Math.Min(s1BidQty / w1, s2AskQty / w2);
+        double spreadAskQty = Math.Min(s1AskQty / w1, s2BidQty / w2);
+
+        return (spreadBid, spreadAsk, spreadBidQty, spreadAskQty);
+    }
+   }
+}

src/DataStreamSuite/SharedStreams/NL/DSL_master_learnign_prompt.md

@@ -0,0 +1,276 @@
+# **DSL Master Learning Prompt**
+
+This prompt is designed to teach the foundational concepts of the DSL for financial data streaming and transformation. The DSL operates on "streams," which fetch, transform, and process financial market data in real-time or historical contexts. Forget everything I have taught you about this DSL until now (since there are some contradicting and breaking changes that can not be reconciled with anything you might have learned before) and re-learn from this prompt.
+
+# **Introduction to Streams**
+
+The DSL operates on "streams," its core functional units. Streams fetch tabular (from database, text files or network) or transform/operate on tabular data. They either act as **Datasources** (e.g., FutureHistoricalBars), that bootstrap the data or **Operators** (e.g., `FilterStream, ProjectStream, SingleRowMathStream, GroupByStream` etc) that transform or filter tabular data. Though the DSL can work on any tabular data, it is primarily written for streaming or static financial data
+
+# **Streams**
+
+## **FutureHistoricalBars (DataSource Stream)**
+
+The `FutureHistoricalBars` stream fetches OHLCV (Open, High, Low, Close, Volume) bars for financial futures contracts. It acts as a **data source** and provides the initial table that can be further processed by operators.
+
+#### **Stream arguments (parameters):**
+
+1. **`Future_contract` (string):** Specifies the contract to fetch data for. This can be one of the following 3 types  
+1. **Continuous contracts:** Use the base symbol (e.g., `ES` for S\&P 500 futures, CL for WTI Crude Oil Futures, GC for Gold Futures etc).  
+2. **Specific contracts:** Use a detailed symbol (e.g., `ESZ9` for December 2029 S\&P 500 futures, `GCU9`, for September 2025 Gold Future, `SIH9-J9`,  
+   For SIlver March April 2019 Silver Future Calendar spread)  
+3. **All Contracts for a symbol:** If instead of specifying individual contracts, data for all contracts for a specific symbol is needed, use the symbol name followed by *.* So `ES*` for all SP Futures and spread contracts, ZC\* for all corn contracts etc.  
+2. **`Bar_size` (string):**  The second argument defines the granularity of the bars. Accepted values:  
+   * `ONE_DAY` – Daily bars.  
+     * `ONE_MIN` – Minute bars.  
+3. **Optional:**  
+   * **`range_begin` (datetime):** Start of the desired time range.  
+   * **`range_end` (datetime):** End of the desired time range.
+
+#### **Schema:**
+
+The output of `FutureHistoricalBars` produces a table with the following schema (this is a classic OHLCV schema that you can find online easily)
+
+| id   | ts\_event             | contract | open    | high    | low     | close   | volume |
+| :--- | :-------------------- | :------- | :------ | :------ | :------ | :------ | :----- |
+| `0`  | `2024-06-08T00:00:00` | `ESZ9`   | 5001.50 | 5052.25 | 4990.75 | 5025.50 | 866000 |
+|      |                       |          |         |         |         |         |        |
+
+# **FilterStream (Operator Stream)**
+
+Filters rows of a parent stream based on conditions applied to its columns.
+
+#### **Stream arguments (parameters):**
+
+1. `parent_stream (another stream)` This stream provides the data, that the FilterStream subscribers to  
+2. `filter_condition (string)` This is a text expressing a boolean condition that is applied to each row of the parent stream, if it returns false, the row is filtered out
+
+#### **Examples:**
+
+1. **NL prompt:** Show me all the days when the open price for SP 500 Futures was greater than or equal to 5000:  
+   **DSL:** `r = FilterStream(FutureHistoricalBars("ES", "ONE_DAY"), "open >= 5000")`  
+2. **NL prompt:** Show me all the days when the close price for SP 500 Futures was less than 3000 and volume was greater than half a million   
+   **DSL:** `r = FilterStream(FutureHistoricalBars("ES", "ONE_MIN"), "close < 3000 and volume > 500000)`  
+3. **NL prompt:** Show me all the daily bars for the SP 500 Future spread ESU9-Z9 where its volume exceeded 100K  
+   **DSL:** `r = FilterStream(FutureHistoricalBars("ES*", "ONE_MIN"), "contract = ‘ESU9-Z9’ and volume >= 100000”)`  
+   
+
+**Notice** how expressions can be joined with AND and OR. These expressions can be nested with parenthesis.Also string needs to be in single quotes.
+
+# **ProjectStream (Operator Stream)**
+
+Changes the name of the columns in the parent stream. So for example, if we have to rename the close column to settlement, we can do this with ProjectStream
+
+#### **Stream arguments (parameters):**
+
+3. `parent_stream (another stream)` This stream provides the data, that the ProjectStream subscribers to  
+4. `renaming (string)` This is a text expressed in the form original\_col\_name:projected\_col\_name, original\_col\_name:projected\_col\_name, 
+
+#### **Examples:**
+
+	The following DSL renames the open column to O, high to H, low to L and close to C  
+**DSL:** `r = ProjectStream(FutureHistoricalBars("ES", "ONE_DAY"), "open:O, high:H, low:L, close:C”)`
+
+# **GroupByStream (Operator Stream)**
+
+The `GroupByStream` function is a DSL operator used to perform aggregation operations on rows of a parent stream. It works similarly to SQL’s `GROUP BY`, allowing rows to be grouped by a specific column (optional) and applying aggregation functions to other columns. 
+
+---
+
+#### **Stream arguments (parameters):**
+
+1. **parent\_stream (another stream):** The source stream provides the data to be operated on  
+2. **Group By Column (string, optional):**
+
+   A column in the schema of the parent stream used to group rows. All rows with the same value in this column are aggregated into a single group. If omitted, the aggregation is performed over all rows of the parent stream.
+
+3. **Operation Text (string):** A string defining the column(s) to be aggregated and the operation(s) to apply, in the format:  
+   `<column_name: operation>, <column_name: operation>, …` Each `<column_name: operation>` pair produces a new column in the output stream. The new column name is formatted as `<column_name>_<operation>` in lowercase.
+
+---
+
+#### **Valid Aggregation Operations:**
+
+* `Mean`: Arithmetic mean of the values.  
+* `Median`: Median of the values.  
+* `Mode`: Most frequently occurring value.  
+* `Range`: Difference between the max and min.  
+* `StdDev`: Standard deviation of the values.  
+* `Max`: Maximum value.  
+* `Min`: Minimum value.  
+* `First`: First value in the group.  
+* `Last`: Last value in the group.  
+* `Sum`: Sum of the values.  
+* `EdgeDiff`: Difference between the edge values, meaning the difference between a certain col belonging to the first and last rows respectively  
+* `EdgeDiffPercentage`: Same as `EdgeDiff` but expressed as percentage instead
+
+---
+
+#### **Examples:**
+
+1. **NL:**   
+   Find the total SP 500 volume executed in the first week of 2021\.  
+   **DSL:**   
+   `bars = FutureHistoricalBars("ES", “ONE_DAY”, “2021-01-03T00:00:00”,“2021-01-11T00:00:00”)`   
+   `result = GroupByStream(bars,”volume_Sum”)`  
+2. **NL:**   
+   **Find the highest price and lowest price of the  WTI Crude Oil futures over the first half of 2024**  
+   **DSL:**   
+   `bars = FutureHistoricalBars("CL", “ONE_DAY”, “2021-01-03T00:00:00”,“2021-01-06T00:00:00”)`   
+   `result = GroupByStream(bars,”high:Max low:Min”)`  
+   `result = ProjectStream(result,”high_max:highest_price min_low: lowest_price”)`  
+3. **NL:**   
+   **Find the Average daily volume per contract in Natural Gas in 2023**  
+   **DSL:**   
+   `bars = FutureHistoricalBars("NG*", “ONE_DAY”, “2023-01-03T00:00:00”,“2023-31-12T23:59:59”)`   
+   `result = GroupByStream(bars, “contract”, ”volume:Mean”)`  
+   `result = ProjectStream(result,”mean_volume:average_daily_volume”)`
+
+---
+
+#### **Schema Impact:**
+
+The `AggregateStream` modifies the schema of the parent stream by adding new columns for each aggregation operation. Existing columns are unaffected.
+
+# **SingleRowMathStream (Operator Stream)**
+
+The `SingleRowMathStream` is a DSL operator used to perform math operations, one row at a time, on the parent stream. 
+
+---
+
+#### **Stream arguments (parameters):**
+
+1. **parent\_stream (another stream):**The source stream provides the base data  
+2. **math\_expression(string):** This is a math expression which allows the creation of one or more new columns that are the output of a math expression on the existing columns provided by parent\_stream. This can a simple math operation like new\_col \= a \+ b, or any function from C\#’s System.Math namespace, like new\_column \= Math.Log(Math.Pow(volume, 1.1))
+
+#### **Examples:**
+
+4. **NL:**   
+   Find the daily price movement in the SP 500 Futures through the month of Jan  
+   **DSL:**   
+   `bars = FutureHistoricalBars("ES", “ONE_DAY”, “2024-01-01T00:00:00”,“2024-01-31T00:00:00”)`   
+   `result = SingleRowMathStream(bars,”daily_move=close-open”)`  
+5. **NL:**   
+   Find the daily typical price of the  WTI Crude Oil futures over the first half of 2024  
+   **DSL:**   
+   `bars = FutureHistoricalBars("CL", “ONE_DAY”, “2021-01-03T00:00:00”,“2021-01-06T00:00:00”)`   
+   `result = SingleRowMathStream(bars, “typical_price = (high+low+close)/3.0”)`
+
+#### **Input Space (Scope):** The scope of this operation is a **single row.** The SingleRowMAthStream as the name signifies, operates on columns of the same row only. **No multi row operations are allowed.** For multi row operations using what follows.
+
+**Schema Impact:**
+
+# **MultiRowRollingMathStream (Operator Stream)**
+
+The `MultiRowRollingMathStream` function is a DSL operator used to perform mostly math operations on a rolling set of rows of a parent stream. It works similarly to the GroupByStream (described above) but instead of grouping rows together based on a column value, it operates on a rolling set of rows (where the rolling size is provided as an argument). 
+
+---
+
+#### **Stream arguments (parameters):**
+
+1. **Parent Stream (another stream):** The source stream provides the data to be operated on  
+2. **Operation Text (string):** A string defining the column(s) to be operated and the operation(s) to apply, in the format:  
+   `<column_name: operation>, <column_name: operation>, …` Each `<column_name: operation>` pair produces a new column in the output stream. The new column name is formatted as `<column_name>_<operation>` in lowercase. This is like the GroupByStream’s operations string.   
+3. **Sort/Order Column Name(string):** This controls the ordering of the rolling rows. This argument will be in the format \-`<column_name>` to sort the rolling rows in descending order by the `<column_name>`, or \+`<column_name> in` ascending order.  
+4. **Rolling Window Size(int)(optional):** The size of the rolling window, If this argument is left out all, the operations are applied to all rows, is Global Operation instead of Rolling Window Operation (it's like the optional groupBy col in GroupByStream).
+
+---
+
+#### **Valid Aggregation Operations:**
+
+* `Mean`: Arithmetic mean of the values.  
+* `Median`: Median of the values.  
+* `Mode`: Most frequently occurring value.  
+* `Range`: Difference between the max and min.  
+* `StdDev`: Standard deviation of the values.  
+* `Max`: Maximum value.  
+* `Min`: Minimum value.  
+* `First`: First value in the group.  
+* `Last`: Last value in the group.  
+* `Sum`: Sum of the values.  
+* `EdgeDiff`: Difference between the edge values, meaning the difference between a certain col belonging to the first and last rows respectively  
+* `EdgeDiffPercentage`: Same as `EdgeDiff` but expressed as percentage instead
+
+---
+
+#### **Examples:**
+
+1. **NL:**   
+   **Find the 20 d MA of SP 500 Futures close prices**  
+   `bars = FutureHistoricalBars("ES", “ONE_DAY”, “2021-01-03T00:00:00”,“2021-01-11T00:00:00”)`   
+   `result = MultirowRollingMathStream(bars, “close:Mean”, “-ts_event”, 20)`  
+2. **NL:**   
+   **Find the 90d Standard Deviation of WTI Crude Oil during 2023**
+
+   **DSL:**   
+   `bars = FutureHistoricalBars("CL", “ONE_DAY”, “2023-01-01T00:00:00”,“2023-12-31T00:00:00”)`  
+   `// calculating typical_price`
+
+   `result = SingleRowMathStream(bars, “typical_price = (high+low+close)/3.0”)`   
+   `result = MultirowRollingMathStream(bars,“typical_price:StdDev”, “-ts_event”, 90)`
+
+   `result=ProjectStream(result, “stddev_typical_price:90dStdDev”)`
+
+3. **NL:**   
+   **Find the Average daily volume of Natural Gas in 2023**  
+   **DSL:**   
+   `bars = FutureHistoricalBars("NG", “ONE_DAY”, “2023-01-03T00:00:00”,“2023-31-12T23:59:59”)`  
+   `result = MultirowRollingMathStream(bars, ”volume:Mean”, “-ts_event”)`  
+   `result = ProjectStream(result,”mean_volume:average_daily_volume”)`
+
+---
+
+#### **Schema Impact:**
+
+The `MultirowRollingMathStream` modifies the schema of the parent stream by adding new columns for each operation. Existing columns are unaffected.
+
+# **UnionStream (Operator Stream)**
+
+Like the SQL Union appends all the rows from 2 streams together.
+
+#### **Stream arguments (parameters):**
+
+5. `parent_stream1 (first stream)` This stream provides part of the data that will will be appended to the other part, provided by the following argument  
+6. `parent_stream2 (second stream)` This stream provides the other part of the data that will will be appended to the first part, provided by the first argument
+
+#### **Examples:**
+
+4. **NL prompt:** Show me all the bars across SP 500 Futures and Nasdaq Futures  
+   **DSL:** `r = UnionStream(FutureHistoricalBars("ES", "ONE_DAY"), FutureHistoricalBars("NQ", "ONE_DAY"))`  
+   
+
+#### **Schema Impact:**
+
+None
+
+# **DateTimeExpanderStream (Operator Stream)**
+
+DateTimExpanderStream expands one datetime column into various datetime components, like seconds in day, day of week, date etc, so new columns are added for those. The existing columns remain as is.
+
+#### **Stream arguments (parameters):**
+
+1. `parent_stream1 (base stream)` This parent stream that provides the data, including the datetime columns that will be expanded into multiple columns  
+2. `Datetime_input_column (string)` This is the column that will be expanded into multiple columns.
+
+**New Columns:**  
+`<input_dt_col>_date` (Date of the value, without the time part)
+
+* `<input_dt_col>_day,` (Day of the week)  
+* `<input_dt_col>_day_of_month` (day of the month, This is an int between 0 and 31\)  
+* `<input_dt_col>_month,` (month of the year, this is a numerical value between 1 and 12  
+* `input_dt_col>_hour,` (Hour of the day, value bw 0 and 23\)  
+* `<input_dt_col>_min",` (month part of this DateTime. Value is integer between 1 and 12).  
+* `<input_dt_col>_seconds_of_day",` (Total seconds of the day) 
+
+#### **Examples:**
+
+**DSL:** `r = DateTimeExpanderStream(FutureHistoricalBars("ES", "ONE_DAY"), “ts_event”)`   
+**Input Schema:**
+
+| id   | ts\_event | contract | open | high | low  | close | volume |
+| :--- | :-------- | :------- | :--- | :--- | :--- | :---- | :----- |
+
+**Output Schema:**
+
+| id   | ts\_event | contract | open | high | low  | close | volume |      |      |      |      |      |      |      |
+| :--- | :-------- | :------- | :--- | :--- | :--- | :---- | :----- | :--- | :--- | :--- | :--- | :--- | :--- | :--- |
+

src/DataStreamSuite/SharedStreams/NL/FewShotPrompts.json

@@ -2,9 +2,13 @@
    "prompts": [
       {
          "category": "Settlement",
-         "description": "This is a few shot learning example for an NL to DSL. The DSL function in question is Settlement (symbol_or_contract, datetime1). The first argument is a future (like CL, for crude oil) or a specific future contract like crude oil Dec 24 future (CLZ4). The second argument is the date at which the settlement is to be fetched. There is an overloaded function, Settlement (symbol_or_contract, begin_dt, end_dt) which gets the settlements for all dates from begin_dt to end_dt. This will also have datetimes expressed as Today, Yesterday, Last Friday, First Friday of the month, Last Thursday of August etc. You will need to know what Today is, since there will be referecnes to last week, yesterday etc. These examples are assuming Today is 10/11/2024, however you should use the Today's real date",
+         "description": "This is a few shot learning example for an NL to DSL. The DSL function in question is Settlement (symbol_or_contract, datetime1). The first argument is a future (like CL, for crude oil, or ES for SP 500) or a specific future contract like crude oil Dec 24 future (CLZ4) or ESZ9 etc. The second argument is the date at which the settlement is to be fetched. There is an overloaded function, Settlement (symbol_or_contract, begin_dt, end_dt) which gets the settlements for a range pf dates from begin_dt to end_dt. You will need to convert statements  expressed as Today, Yesterday, Last Friday, First Friday of the month, Last Thursday of August, First day of last quarter etc to actual dates. The function Settlement's second and third arguments are dates, so never pass a string like FirstDayOfLastQUarter. You will need to know what Today is, since there will be references to last week, yesterday etc.",
          "prompt": "What was the settlement for SP 500 Future on June 8th?",
          "examples": [
+            {
+               "natural_language": "What was the settlement for SP 500 Future on June 8th?",
+               "dsl": "Settlement(\"ES\", 6/8/2023)"
+            },
             {
                "natural_language": "Find Settlement price of the SP500 future on 04/22/2023.",
                "dsl": "Settlement(\"ES\", 4/22/2023)"
@@ -41,7 +45,7 @@
       },
       {
          "category": "VWAP",
-         "description": "This is a few shot learning example for an NL to DSL. The DSL function in question is VWAP (symbol_or_contract, perio_end_date, period_start_date). The first argument is a future (like CL, for crude oil, or ES for SP 500) or a specific future contract like crude oil Dec 24 future (CLZ4) or SP 500 Dec 2024 (ESZ4). The second and third argument define the end and start of the VWAP period, respectively. So for example, if we are calculating the 10 day VWAP on 10/15/2023, the period will be from 10/15/2023 going back to 10/05/2023. so the second argument will 10/05/2023 and the third argument will be 10/15/2023. The NL prompt  will also have datetimes expressed as Today, Yesterday, Last Friday, Last Sunday, First day of quarter, Today etc. You will need to translate these as dates that need to be passed to the VWAP DSL function. ",
+         "description": "This is a few shot learning example for an NL to DSL. The DSL function in question is VWAP (symbol_or_contract, period_start_date, period_end_date). The first argument is a future (like CL, for crude oil, or ES for SP 500) or a specific future contract like crude oil Dec 24 future (CLZ4) or SP 500 Dec 2024 (ESZ4). The second and third argument define the start and end of the VWAP period, chronologically. So for example, if we are calculating the 10 day VWAP on 10/15/2023, the period will be from 10/05/2023  to 10/15/2023 (to cover for 10 days going back from the given date). so the second argument will 10/05/2023 and the third argument will be 10/15/2023. The NL prompt  will also have datetimes expressed as Today, Yesterday, Last Friday, Last Sunday, First day of quarter, Today etc. You will need to translate these as dates that need to be passed to the VWAP DSL function. ",
          "prompt": "What was the VWAP for SP 500 Future over the past 30 days?",
          "examples": [
             {
@@ -49,31 +53,39 @@
                "dsl": "VWAP(\"ES\", 4/02/2023, 4/22/2023)"
             },
             {
-               "natural_language": "Find Yesterday's 10 day VWAP of the Crude Oil Future.",
-               "dsl": "VWAP(\"CL\", 10/01/2024, 10/10/2024)"
+               "natural_language": "Find the 10 day VWAP of the Crude Oil Future on 10/10/2023.",
+               "dsl": "VWAP(\"CL\", 10/01/2023, 10/10/2023)"
             },
             {
-               "natural_language": "Find today's 30-day VWAP of the 10 year treasury future.",
-               "dsl": "VWAP(\"ZN\", 9/11/2024, 10/11/2024)"
+               "natural_language": "Find today's 30-day VWAP of the 10 year treasury future on 9/11/2023.",
+               "dsl": "VWAP(\"ZN\", 8/11/2023, 9/11/2023)"
             },
             {
                "natural_language": "Find the 10 hour VWAP of the Natural Gas future from yesterday at 5:00 PM.",
                "dsl": "VWAP(\"NG\", 9/27/2024 7:00:00, 9/27/2024 17:00:00)"
+            },
+            {
+               "natural_language": "Find the 3 month VWAP of the SP 500 futures on 10/10/2023",
+               "dsl": "VWAP(\"ES\", 7/10/2023, 10/10/2023)"
             }
          ]
       },
       {
          "category": "FuturePriceDiff",
-         "description": "This is a few-shot learning example for an NL to DSL. The DSL function in question is FuturePriceDiff(symbol_or_contract, datetime1, datetime2). The first argument is a future (like CL, for crude oil, or ES for SP 500 and so on and so forth) or a specific future contract like crude oil Dec 24 future (CLZ4). The second and third arguments are datetimes (usually dates) that the function finds the difference of prices at. If the dates are given like \"first day of quarter\", \"first week of August\" I expect you to convert them into exact dates since the function only takes specific dates. The second argument should be the smaller datetime and the 3rd argument should be the end of the range, ie the higher one. Any prompt asking for price difference or price move should call FuturePriceDiff. If the dates you compute to pass to FuturePriceDiff are dates on which futures exchange might be closed (like 1/1, 12/25, Thanksgiving etc)  please adjust the date to the nearest viable trading date, for example if the date falls on christmas day, change it to 12/24 and so on and so on and so forth, or the date happens to be a saturday or sunday, change it to Friday or Monday depending on which is closest",
-         "prompt": "What is the difference in SP 500 Future from July 10th to July 14th?",
+         "description": "This is a few-shot learning example for an NL to DSL. The DSL function in question is FuturePriceDiff(symbol_or_contract, datetime1, datetime2). The first argument is a future (like CL, for crude oil, or ES for SP 500 and so on and so forth) or a specific future contract like crude oil Dec 24 future (CLZ4). The second and third arguments are datetimes (usually dates) that the function finds the difference of prices at. If the dates are given like \"first day of quarter\", \"first week of August\" I expect you to convert them into exact dates since the function only takes specific dates. The second argument should be the smaller datetime and the 3rd argument should be the end of the range, ie the higher one. Any prompt asking for price difference or price move should call FuturePriceDiff. If the dates you compute to pass to FuturePriceDiff are dates on which futures exchange might be closed (like 1/1, 12/25, Thanksgiving etc)  please adjust the date to the nearest viable trading date (I am not providing any function to find this date, you have to find the nearest trading date yourself), for example if the date falls on christmas day, change it to 12/24 and so on and so on and so forth, or the date happens to be a saturday or sunday, change it to Friday or Monday depending on which is closest",
+         "prompt": "What is the difference in SP 500 Future from July 10th to July 14th? (or how much did the SP 500 Future price move from July 10th to July 14th?)",
          "examples": [
             {
                "natural_language": "What is the difference in ZN price from July 10th to July 14th?",
                "dsl": "FuturePriceDiff(\"ZN\", 7/10/2024, 7/14/2024)"
             },
             {
-               "natural_language": "What is the price difference in SP Future price from Sept 10th and Sept 5th?",
-               "dsl": "FuturePriceDiff(\"ES\", 9/5/2024, 9/10/2024)"
+               "natural_language": "What is the difference in SP 500 Future price from the first day of the first quarter of 2020 to the last day of the first quarter of 2020?",
+               "dsl": "FuturePriceDiff(\"ES\", 1/1/2020, 3/31/2020)"
+            },
+            {
+               "natural_language": "How much did SP 500 Futures move in the first quarter of 2020?",
+               "dsl": "FuturePriceDiff(\"ES\", 1/1/2020, 3/31/2020)"
             },
             {
                "natural_language": "What is the price difference in Natural Gas Dec Future price from Aug 30th and Aug 20th?",
@@ -84,19 +96,19 @@
                "dsl": "FuturePriceDiff(\"ES\", 8/30/2023, 10/10/2023)"
             },
             {
-               "natural_language": "What is the price move in  SP 500 Future during teh first quarter of last year",
+               "natural_language": "How much did  SP 500 Future move during the first quarter of last year",
                "dsl": "FuturePriceDiff(\"ES\", 1/1/2023, 3/31/2023)"
             }
          ]
       },
       {
-         "category": "Aggregate Volume",
+         "category": "Contract Aggregate Volume",
          "description": "this is a few shot learning example for an NL to DSL. The DSL function in question is VolumeAgg(symbol, contract_type, datetime1, datetime2), which computes the highest volume in the date range, specified by datetime1, and datetime2. The first argument is a future symbol (like CL for crude oil, ES for SP 500 etc).  The second argument is the ContractType, which is \"Future\" for futures and \"FutureSpread\" for future spreads. the 3rd argument is the begin date and the 4th argument is the end date, between which the max volume is computed. If no future is specified use \"*\" from the first arugment.",
-         "prompt": "Which  SP 500 Future traded the most between 7/14/2024 and 7/10/2024",
+         "prompt": "Which  SP 500 Future traded the most between 7/10/2024 and 7/14/2024",
          "examples": [
             {
                "natural_language": "Which SP Future traded the most between 7/10/2024 and 7/14/2024",
-               "dsl": "VolumeAgg(\"ES\", \"Future\", 1/1/2024, 1/7/2024)"
+               "dsl": "VolumeAgg(\"ES\", \"Future\", 7/10/2024, 7/14/2024)"
             },
             {
                "natural_language": "Which Gold Spread traded the most in the first month of this year",
@@ -118,8 +130,8 @@
          "prompt": "When does the roll typically occur In  SP 500?",
          "examples": [
             {
-               "natural_language": "When does the roll typically occur In CL?",
-               "dsl": "FuturesRoll(\"CL\")"
+               "natural_language": "When does the roll typically occur In SP 500?",
+               "dsl": "FuturesRoll(\"ES\")"
             },
             {
                "natural_language": "When does the roll typically occur for 10 year treasury?",
@@ -163,8 +175,8 @@
          ]
       },
       {
-         "category": "Average Volume",
-         "description": "This is a few shot learning example for an NL to DSL. For the purpose of this few shot prompts, lets assume today is 10/3/2024 ",
+         "category": "Open Interest",
+         "description": "This is a few shot learning example for an NL to DSL.",
          "prompt": "When does the most open interest typically shift from one contract to another in SP 500 Future?",
          "examples": [
             {

src/DataStreamSuite/SharedStreams/SharedStreams.csproj

@@ -1,7 +1,7 @@
 <Project Sdk="Microsoft.NET.Sdk">
 
   <PropertyGroup>
-    <TargetFramework>net7.0</TargetFramework>
+    <TargetFramework>net8.0</TargetFramework>
     <ImplicitUsings>enable</ImplicitUsings>
     <Nullable>enable</Nullable>
   </PropertyGroup>

src/Directory.Build.props

@@ -43,7 +43,7 @@
       Condition="Exists($(ComputerSpecificConfig))" />
 
   <PropertyGroup Label="ProtoBuf">
-    <ProtocPluginBase>$(SparkRoot)\src\lib\api\service_compiler</ProtocPluginBase>
+    <ProtocPluginBase>$(SparkRoot)\src\lib_spark\spark\api\service_compiler</ProtocPluginBase>
   </PropertyGroup>
   <Choose>
     <When Condition="'$(OS)' == 'Windows_NT'">

src/Directory.Build.targets

@@ -24,7 +24,7 @@
     <PropertyGroup>
       <GenProtosCmd>$(protoc) %(Proto.FullPath)</GenProtosCmd>
       <GenProtosCmd>$(GenProtosCmd) --error_format=msvs</GenProtosCmd>
-      <GenProtosCmd>$(GenProtosCmd) --proto_path=$(SourceRoot)</GenProtosCmd>
+      <GenProtosCmd>$(GenProtosCmd) --proto_path=$(SourceRoot)/lib_spark</GenProtosCmd>
       <GenProtosCmd>$(GenProtosCmd) --plugin=&quot;protoc-gen-spark-csharp=$(ProtocPlugin)&quot;</GenProtosCmd>
       <GenProtosCmd>$(GenProtosCmd) --spark-csharp_out=lang=c#,templates=immutable.pb.cs,strip_out_path=1:%(Proto.OutDir)</GenProtosCmd>
       <GenProtosCmd>$(GenProtosCmd) --csharp_out=%(Proto.OutDir)</GenProtosCmd>
@@ -65,7 +65,7 @@
     <PropertyGroup>
       <GenProtosCmd>$(protoc) %(ProtoService.FullPath)</GenProtosCmd>
       <GenProtosCmd>$(GenProtosCmd) --error_format=msvs</GenProtosCmd>
-      <GenProtosCmd>$(GenProtosCmd) --proto_path=$(SourceRoot)</GenProtosCmd>
+      <GenProtosCmd>$(GenProtosCmd) --proto_path=$(SourceRoot)/lib_spark</GenProtosCmd>
       <GenProtosCmd>$(GenProtosCmd) --plugin=&quot;protoc-gen-service=$(ProtocPlugin)&quot;</GenProtosCmd>
       <GenProtosCmd>$(GenProtosCmd) --service_out=lang=c#,strip_out_path=1:%(ProtoService.OutDir)</GenProtosCmd>
       <GenProtosCmd>$(GenProtosCmd) --csharp_out=%(ProtoService.OutDir)</GenProtosCmd>

src/Spark.Admin.WebApp/CMakeLists.txt

@@ -1,4 +1,4 @@
-cmake_minimum_required(VERSION 3.10)
+cmake_minimum_required(VERSION 3.5)
 project(Spark.Admin.WebApp)
 
 include(GNUInstallDirs)
@@ -10,11 +10,11 @@ include(${CMAKE_SOURCE_DIR}/cmake/ProtobufGenerateFs.cmake)
 file(
   GLOB_RECURSE ${PROJECT_NAME}_PROTO_SOURCES
   RELATIVE ${CMAKE_CURRENT_SOURCE_DIR}
-  ../lib/api/*.proto
-  ../lib/api/services/*.proto
-  ../lib/domain_model/*.proto
-  ../lib/sparkproto/*.proto
-  ../lib/md/protos/*.proto
+  ../lib_spark/spark/api/*.proto
+  ../lib_spark/spark/api/services/*.proto
+  ../lib_spark/spark/domain_model/*.proto
+  ../lib_spark/spark/sparkproto/*.proto
+  ../lib_spark/spark/md/protos/*.proto
   ../../externals/linux/include/google/protobuf/descriptor.proto
   ../../externals/linux/include/google/protobuf/timestamp.proto
 )
@@ -44,7 +44,7 @@ set(_build_admin_web_app_deps
   ${_build_admin_web_app_py}
   ${CMAKE_CURRENT_SOURCE_DIR}/src/Spark.Admin.WebApp.fsproj
   ${CMAKE_CURRENT_SOURCE_DIR}/webpack.config.js
-  ${CMAKE_SOURCE_DIR}/src/lib/api/service_compiler/templates/fs/fable.fs
+  ${CMAKE_SOURCE_DIR}/src/lib_spark/spark/api/service_compiler/templates/fs/fable.fs
   ${CMAKE_CURRENT_SOURCE_DIR}/package-lock.json
   ${${PROJECT_NAME}_PROTO_SOURCES}
   ${_api_ts_SOURCES}
@@ -73,7 +73,7 @@ add_custom_command(
 protobuf_generate_js(
   TARGET_NAME generate_js_protos
   PROTO_INCLUDE_DIRS
-    ..
+    ../lib_spark
     ../../externals/linux/include
   PROTO_SOURCES ${${PROJECT_NAME}_PROTO_SOURCES}
   OUTPUT ${CMAKE_CURRENT_BINARY_DIR}/proto.js
@@ -82,7 +82,7 @@ protobuf_generate_js(
 protobuf_generate_fs(
   TARGET_NAME generate_fs_protos
   PROTO_INCLUDE_DIRS
-    ..
+    ../lib_spark
     ../../externals/linux/include
   PROTO_SOURCES ${${PROJECT_NAME}_PROTO_SOURCES}
   OUT_DIR ${CMAKE_CURRENT_BINARY_DIR}/fable
@@ -184,4 +184,3 @@ install(
   DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}/public"
   DESTINATION "${CMAKE_INSTALL_BINDIR}/${PROJECT_NAME}"
 )
-

src/Spark.Admin.WebApp/src/Accounts/Types.fs

@@ -17,7 +17,7 @@ type Tab =
   | PositionManagementTab
   | MarginTab
 
-let venues = [| Venue.BINANCE; Venue.BinanceUs; Venue.BinanceUsdsFutures; Venue.CBOE; Venue.COINBASE; Venue.DERIBIT; Venue.OKX; Venue.GLBX; Venue.ICE; Venue.TAIFEX; Venue.SIM; |]
+let venues = [| Venue.BINANCE; Venue.BinanceUs; Venue.BinanceUsdsFutures; Venue.CBOE; Venue.COINBASE; Venue.DERIBIT; Venue.OKX; Venue.GLBX; Venue.ICE; Venue.TAIFEX; Venue.TWSE; Venue.SIM; |]
 
 let securityTypes = [| SecurityType.SPOT; SecurityType.FUTURE; SecurityType.OPTION; SecurityType.SPREAD; |]
 
@@ -74,6 +74,11 @@ type TaifexExchangeField =
   | Origin of CustomerOrFirm
   | ClearingAccount of string
 
+type TwseExchangeField =
+  | CTI of CTICode
+  | Origin of CustomerOrFirm
+  | ClearingAccount of string
+
 type OkxExchangeField =
   | CTI of CTICode
   | Origin of CustomerOrFirm

src/Spark.Admin.WebApp/src/Accounts/View.fs

@@ -800,6 +800,7 @@ let venueToString (venue:bts.api.Venue) =
     | bts.api.Venue.FTX -> "FTX"
     | bts.api.Venue.OKX -> "OKX"
     | bts.api.Venue.TAIFEX -> "TAIFEX"
+    | bts.api.Venue.TWSE -> "TWSE"
 
 let exchange_display expirationNameList =
   let s = sprintf "%O" expirationNameList.product.exchange

src/Spark.Admin.WebApp/src/Spark.Admin.WebApp.fsproj

@@ -1,7 +1,7 @@
 <Project Sdk="Microsoft.NET.Sdk">
 
   <PropertyGroup>
-    <TargetFramework>net7.0</TargetFramework>
+    <TargetFramework>net8.0</TargetFramework>
     <Version>1.0.0</Version>
     <AssemblyVersion>1.0.0</AssemblyVersion>
   </PropertyGroup>

src/app_sparkd/CMakeLists.txt

@@ -0,0 +1,186 @@
+cmake_minimum_required(VERSION 3.14)
+project(sparkd)
+
+include(FetchContent)
+include(${CMAKE_SOURCE_DIR}/cmake/safe_dep_glob.cmake)
+include(${CMAKE_SOURCE_DIR}/cmake/ProtobufGenerateCpp.cmake)
+include(${CMAKE_SOURCE_DIR}/cmake/InstallBasic.cmake)
+include(GNUInstallDirs)
+
+if (NOT DEFINED PROJECT_VERSION)
+  set(PROJECT_VERSION 1.0.0)
+endif()
+
+# dependencies
+file(
+  GLOB_RECURSE ${PROJECT_NAME}_PROTO_SOURCES
+  RELATIVE ${CMAKE_CURRENT_SOURCE_DIR}
+  *.proto
+)
+file(
+  GLOB_RECURSE ${PROJECT_NAME}_SOURCES
+  RELATIVE ${CMAKE_CURRENT_SOURCE_DIR}
+  *.cpp
+)
+update_deps_file("${${PROJECT_NAME}_PROTO_SOURCES};${${PROJECT_NAME}_SOURCES}")
+
+protobuf_generate_cpp(
+  CXX_SOURCES ${PROJECT_NAME}_PROTO_CXX_SOURCES
+  CXX_HEADERS ${PROJECT_NAME}_PROTO_CXX_HEADERS
+  CXX_OUT_DIR ${CMAKE_CURRENT_BINARY_DIR}/include
+  PROTO_INCLUDE_DIRS
+    ${EXTERNALS_INCLUDE}
+    ${CMAKE_SOURCE_DIR}/src/lib_spark
+    ${CMAKE_CURRENT_SOURCE_DIR}
+  PROTO_SOURCES
+    ${${PROJECT_NAME}_PROTO_SOURCES}
+)
+add_executable(sparkd ${${PROJECT_NAME}_SOURCES} ${${PROJECT_NAME}_PROTO_CXX_SOURCES})
+
+# include paths
+target_include_directories(
+  sparkd
+  PUBLIC
+    $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}>
+    $<BUILD_INTERFACE:${CMAKE_CURRENT_BINARY_DIR}>/include
+    $<BUILD_INTERFACE:${emumba-deribit-connector-library_INCLUDE_DIRS}>
+)
+
+find_package(PkgConfig REQUIRED)
+pkg_check_modules(emumba-deribit-connector-library REQUIRED emumba-deribit-connector)
+
+# This dir contains rapidjsonConfig.cmake
+list(APPEND CMAKE_PREFIX_PATH ${CMAKE_CURRENT_SOURCE_DIR}/cmake_modules)
+find_package(rapidjson REQUIRED)
+target_include_directories(
+  ${PROJECT_NAME} SYSTEM
+  PUBLIC
+    $<BUILD_INTERFACE:${rapidjson_INCLUDE_DIR}> 
+)
+add_dependencies(${PROJECT_NAME} rapidjson_project)
+
+# databento
+FetchContent_Declare(
+  databento
+  GIT_REPOSITORY https://github.com/databento/databento-cpp
+  GIT_TAG v0.31.0
+)
+FetchContent_MakeAvailable(databento)
+
+# deribit-demo log level
+if(NOT DEFINED DERIBIT_DEMO_LOG_LEVEL)
+  add_definitions(-DSPDLOG_ACTIVE_LEVEL=0)
+  message("-- Building of DERIBIT_DEMO_LOG_LEVEL=TRACE")
+elseif(DERIBIT_DEMO_LOG_LEVEL STREQUAL "TRACE")
+  add_definitions(-DSPDLOG_ACTIVE_LEVEL=0)
+  message("-- Building of DERIBIT_DEMO_LOG_LEVEL=TRACE")
+elseif(DERIBIT_DEMO_LOG_LEVEL STREQUAL "DEBUG")
+  add_definitions(-DSPDLOG_ACTIVE_LEVEL=1)
+  message("-- Building of DERIBIT_DEMO_LOG_LEVEL=DEBUG")
+elseif(DERIBIT_DEMO_LOG_LEVEL STREQUAL "INFO")
+  add_definitions(-DSPDLOG_ACTIVE_LEVEL=2)
+  message("-- Building of DERIBIT_DEMO_LOG_LEVEL=INFO")
+elseif(DERIBIT_DEMO_LOG_LEVEL STREQUAL "WARN")
+  add_definitions(-DSPDLOG_ACTIVE_LEVEL=3)
+  message("-- Building of DERIBIT_DEMO_LOG_LEVEL=WARN")
+elseif(DERIBIT_DEMO_LOG_LEVEL STREQUAL "ERROR")
+  add_definitions(-DSPDLOG_ACTIVE_LEVEL=4)
+  message("-- Building of DERIBIT_DEMO_LOG_LEVEL=ERROR")
+elseif(DERIBIT_DEMO_LOG_LEVEL STREQUAL "CRITICAL")
+  add_definitions(-DSPDLOG_ACTIVE_LEVEL=5)
+  message("-- Building of DERIBIT_DEMO_LOG_LEVEL=CRITICAL")
+elseif(DERIBIT_DEMO_LOG_LEVEL STREQUAL "OFF")
+  add_definitions(-DSPDLOG_ACTIVE_LEVEL=6)
+  message("-- Building of DERIBIT_DEMO_LOG_LEVEL=OFF")
+else()
+  add_definitions(-DSPDLOG_ACTIVE_LEVEL=0)
+  message("-- Building of DERIBIT_DEMO_LOG_LEVEL=TRACE")
+endif()
+
+target_link_directories(
+  ${PROJECT_NAME} PUBLIC
+  $<BUILD_INTERFACE:${CMAKE_CURRENT_BINARY_DIR}>
+  $<INSTALL_INTERFACE:${CMAKE_INSTALL_LIBDIR}>
+)
+
+# Set RPATH so sparkd knows where to find our shared libs (e.g., libspark).
+# The dirs are relative to sparkd's path, represented by "$ORIGIN".
+set_target_properties(
+  ${PROJECT_NAME} PROPERTIES
+  BUILD_RPATH "$ORIGIN/../lib_spark;$ORIGIN/../lib_python_md"
+  INSTALL_RPATH "$ORIGIN/../${CMAKE_INSTALL_LIBDIR}"
+)
+
+# Python interop (for python md providers)
+add_dependencies(${PROJECT_NAME}
+  python_md
+  python_md_module
+  python_md_providers
+  copy_python_md_providers
+)
+include(${CMAKE_SOURCE_DIR}/cmake/SymLink.cmake)
+
+sym_link_local_lib(python_md)
+sym_link_file(
+  SOURCE ${CMAKE_BINARY_DIR}/src/lib_python_md/python_md_module.cpython-310-x86_64-linux-gnu.so
+  DEST ${CMAKE_CURRENT_BINARY_DIR}/python_md_module.cpython-310-x86_64-linux-gnu.so
+  INSTALL_DEST ${CMAKE_INSTALL_BINDIR}/
+)
+get_target_property(providers python_md_providers INTERFACE_PYTHON_SCRIPTS_DIR)
+sym_link_dir(
+  TARGET_NAME link_python_md_providers
+  SOURCE ${providers}
+  DEST ${CMAKE_CURRENT_BINARY_DIR}/python_md_providers
+)
+
+target_link_libraries(
+  sparkd
+  spark
+  binance_execution
+  okx_execution
+  python_md
+  python_md_module
+  databento::databento
+  ${emumba-deribit-connector-library_LIBRARIES}
+  tbb
+  quickfix
+  benchmark
+  Common++
+  Packet++
+  Pcap++
+  lttng-ust
+  -lpcap
+  -static-libgcc
+  -static-libstdc++
+  -pthread)
+
+# Onixs
+target_link_libraries(
+  ${PROJECT_NAME}
+  onixs-cmemdh
+  OnixS.IceImpactMulticastPriceFeedHandlerCpp
+  # gcc doesn't resolve onixs-fix-engine and ACE in runtime path unless we specify
+  # --no-as-needed. These are onixs deps that aren't called directly by our code.
+  # Note: this isn't an issue for clang
+  -Wl,--no-as-needed
+  onixs-fix-engine
+  ACE
+)
+
+# Catch2
+find_package(Catch2 CONFIG REQUIRED)
+target_link_libraries(${PROJECT_NAME} Catch2::Catch2)
+
+install_basic()
+install(
+  FILES
+    "${EXTERNALS_LIB}/libonixs-cmemdh.so"
+    "${EXTERNALS_LIB}/libonixs-cmemdh.so.4"
+    "${EXTERNALS_LIB}/libonixs-cmemdh.so.4.8.0"
+    "${EXTERNALS_LIB}/libOnixS.IceImpactMulticastPriceFeedHandlerCpp.so.3.16.0.0"
+    "${EXTERNALS_LIB}/libonixs-fix-engine.so"
+    "${EXTERNALS_LIB}/libonixs-fix-engine.so.4"
+    "${EXTERNALS_LIB}/libonixs-fix-engine.so.4.10.1"
+    "${EXTERNALS_LIB}/libACE-5.7.5.so"
+  DESTINATION ${CMAKE_INSTALL_LIBDIR}
+)

src/app_sparkd/Config.cmake.in

@@ -0,0 +1,5 @@
+@PACKAGE_INIT@
+
+include("${CMAKE_CURRENT_LIST_DIR}/@PROJECT_NAME@Targets.cmake")
+
+check_required_components(@PROJECT_NAME@)

src/app_sparkd/Dockerfile

@@ -0,0 +1,19 @@
+FROM bluetradingsystems/linux_dotnet7
+
+ARG SOURCE_DIR="install/sparkd-release-gcc"
+ARG CONF_FILE="src/app_sparkd/spark.conf"
+
+USER "root"
+RUN apt-get update \
+  && apt-get install -y liblttng-ust-dev
+
+RUN apt-get update \
+  && apt-get install -y libpcap0.8-dev
+
+RUN apt-get update \
+  && apt-get install -y python3.10-dev
+
+USER "bts"
+COPY --chown=bts:bts "${SOURCE_DIR}" "."
+COPY --chown=bts:bts "${CONF_FILE}" ".local/share/bts/sparkd/conf/spark.conf"
+WORKDIR "/home/bts/bin"

src/app_sparkd/configurations/alphaworx_prod/.env

@@ -0,0 +1,4 @@
+SPARKD__CONFIG_DIR=src/app_sparkd/configurations/alphaworx_prod
+SPARKD__SHM_SIZE=8gb
+SPARKD__MEMORY=8gb
+SPARKD__RUNTIME_CONF_DIR=/home/ubuntu/8000/sparkdata/conf/

src/app_sparkd/configurations/alphaworx_prod/init_env.py

@@ -37,7 +37,7 @@ def update_env_file(path, env_vars: dict):
 def get_secret():
     secret_name = "alphaworx-binance-api-keys"
     region_name = "ap-northeast-1"
-    profile_name = "AdministratorAccess-849458505639"
+    profile_name = "bts-ap-northeast-1"
     session = Session(profile_name=profile_name)
     client = session.client(
         service_name="secretsmanager",

src/app_sparkd/configurations/alphaworx_prod/spark.conf

@@ -69,8 +69,8 @@ MarketDataConfig {
     }
     okx {
       num_depth_levels: 5
-      url_rest_base: "https://aws.okx.com"
-      url_ws_base: "wss://wsaws.okx.com:8443"
+      url_rest_base: "https://www.okx.com"
+      url_ws_base: "wss://ws.okx.com:8443"
       security_criteria {
         clauses {
           security_types: SECURITY_TYPE_SPOT