cleanup: Remove unnecessary includes

feat: Add binance gw for cross margin orders
Spots only
2025-03-04 05:02:35 +00:00 · 2025-03-04 05:02:35 +00:00 · 2025-03-03 19:32:16 +00:00 · 2025-03-03 19:32:16 +00:00 · 2025-03-03 19:32:16 +00:00 · 2025-03-03 19:32:16 +00:00
2403 changed files with 15547 additions and 45236 deletions
@@ -65,7 +65,7 @@ RUN \
   && apt-get install -y cmake

 # Dotnet
-RUN apt-get update && apt-get install -y dotnet8
+RUN apt-get update && apt-get install -y dotnet7

 # Node
 ARG node_major=18
@@ -90,12 +90,6 @@ RUN apt-get update && apt-get install -y libpcap0.8-dev
 # Databento (deps)
 RUN apt-get update && apt-get -y install libzstd-dev

-# Docker in Docker
-RUN apt-get update && apt-get install -y \
-    docker.io \
-    docker-compose \
-    docker-compose-v2
-
 # Set up default executables (update-alternatives)
 RUN \
   update-alternatives --install /usr/bin/clang clang /usr/bin/clang-$clang_major 100; \
@@ -12,10 +12,17 @@
  // example, if 8000-8003 is taken, go with 9000-9003, then 10000-10003, etc.
  "runArgs": [
    "--shm-size",
-    "8gb"
-  ],
-  "mounts": [
-    "source=/var/run/docker.sock,target=/var/run/docker.sock,type=bind"
+    "8gb",
+    "-p",
+    "8000:8000",
+    "-p",
+    "8001:8001",
+    "-p",
+    "8002:8002",
+    "-p",
+    "8003:8003",
+    "-p",
+    "12345:12345"
  ],
  "customizations": {
    "vscode": {
@@ -1,49 +0,0 @@
-# This file defines the global default configuration for docker deployments.
-# It is tracked in source control. For temporary modifications, please create
-# a separate file called ".env.local" (it's in .gitignore, so won't be tracked).
-# If a variable is defined in both files, the value in .env.local will be used.
-#
-# About the deployment procedure:
-#   We use Docker Compose to build images and launch containers for various services
-#   that make up our software suite:
-#     - sparkd
-#     - TODO: pubsub
-#     - TODO: md-publisher
-#     - TODO: webapp (DSQA on the web)
-#     - TODO: api-server (websocket based)
-#   From inside a devcontainer, you can launch all services in separate containers
-#   on your machine by running:
-#
-#     tools/deploy.sh
-#
-#   This script builds release versions for each service, packaged (generally) in
-#   self-contained directories. Next, it runs a few "docker compose" commands to
-#   build docker images loaded with the self-contained binaries from the previous
-#   step, and start containers with those images to run each service. To view the
-#   commands that deploy.sh runs internally (without actually running them), run:
-#
-#     tools/deploy.sh --dry-run
-#
-#   For additional information/help, run:
-#
-#     tools/deploy.sh --help
-#
-
-### Sparkd ###
-# The internal image for sparkd to use. For more info, view:
-#   src/server/sparkd/Dockerfile
-SPARKD__IMAGE=sparkd
-# The config dir for this service to use
-SPARKD__CONFIG_DIR=src/app_sparkd/configurations/dev
-# Mount a specific volume or dir to "~/sparkdata/data" inside the docker.
-# Defaults to a volume named "sparkd_<config_port>" (e.g., sparkd_8000).
-# SPARKD__DATA_DIR=/home/bts/sparkdata/data
-SPARKD__SOURCE_DIR=install/sparkd-release-gcc
-SPARKD__RUNTIME_CONF_DIR=/home/ubuntu/8000/sparkdata/conf/
-SPARKD__CONFIG_PORT=8000
-SPARKD__QUICKBUS_PORT=8001
-SPARKD__SIM_FIX_PORT=8002
-SPARKD__SHM_SIZE=4gb
-SPARKD__PYTHON_REQUIREMENTS=requirements.txt
-
-# TODO: add config for other services (pubsub, md-publisher, webapp, api-server, etc.)
@@ -59,5 +59,3 @@ trace_data/
 .devcontainer/
 *.Generated.cs
 *.lst
-.env.local
-.secrets
@@ -1,6 +1,6 @@
 [submodule "externals"]
 	path = externals
-	url = https://bitbucket.org/bluetradesys/externals.git
+	url = git@bitbucket.org:bluetradesys/externals.git
 [submodule "vcpkg"]
 	path = vcpkg
 	url = https://github.com/microsoft/vcpkg.git
@@ -22,7 +22,7 @@
        "--config",
        "${workspaceFolder}/src/DataStreamSuite/DataStreamPublisher/Configurations/${input:dataStreamPublisherConfig}.json"
      ],
-      "cwd": "build/Debug/bin/net8.0",
+      "cwd": "build/Debug/bin/net7.0",
      "stopAtEntry": false,
      "console": "integratedTerminal"
    },
@@ -30,12 +30,12 @@
      "name": "(gdb) Launch",
      "type": "cppdbg",
      "request": "launch",
-      "program": "${workspaceFolder}/build/sparkd-debug-clang/src/app_sparkd/sparkd",
+      "program": "/home/bts/spark/build/sparkd-debug-clang/server/sparkd",
      "args": [
        "run"
      ],
      "stopAtEntry": false,
-      "cwd": "${workspaceFolder}/build/sparkd-debug-clang/src/app_sparkd/",
+      "cwd": "/home/bts/spark/build/sparkd-debug-clang/server",
      "environment": [],
      "externalConsole": false,
      "MIMode": "gdb",
@@ -82,7 +82,7 @@
          "--config",
          "${workspaceFolder}/src/DataStreamSuite/DataStreamQueryAnalyzer/Configurations/DataStreamQueryAnalyzer.json"
        ],
-        "cwd": "build/Debug/bin/net8.0-windows",
+        "cwd": "build/Debug/bin/net7.0-windows",
        "env": {
          "PYTHON_EXE": "${workspaceFolder}/.venv/Scripts/python.exe"
        },
@@ -106,7 +106,7 @@
        "--config",
        "${workspaceFolder}/src/DataStreamSuite/PubSubServer/Configurations/${input:pubSubServerConfig}.json"
      ],
-      "cwd": "build/Debug/bin/net8.0",
+      "cwd": "build/Debug/bin/net7.0",
      "stopAtEntry": false,
      "console": "integratedTerminal"
    },
@@ -118,7 +118,7 @@
      "windows": {
        "preLaunchTask": "Build Spark (Debug)",
        "program": "Spark.exe",
-        "cwd": "build/Debug/bin/net8.0-windows",
+        "cwd": "build/Debug/bin/net7.0-windows",
      },
      "stopAtEntry": false,
      "console": "integratedTerminal"
@@ -130,7 +130,7 @@
      "program": "",
      "windows": {
        "program": "Spark.exe",
-        "cwd": "build/Debug/bin/net8.0-windows",
+        "cwd": "build/Debug/bin/net7.0-windows",
      },
      "stopAtEntry": false,
      "console": "integratedTerminal"
@@ -147,7 +147,7 @@
          "--sparkd",
          "${input:sparkConfigServer}"
        ],
-        "cwd": "build/Debug/bin/net8.0-windows",
+        "cwd": "build/Debug/bin/net7.0-windows",
      },
      "stopAtEntry": false,
      "console": "integratedTerminal"
@@ -160,7 +160,7 @@
      "windows": {
        "preLaunchTask": "Build Spark (Debug)",
        "program": "Spark.Console.exe",
-        "cwd": "build/Debug/bin/net8.0",
+        "cwd": "build/Debug/bin/net7.0",
        "args": [
          "test",
          "${input:sparkTest}"
@@ -176,7 +176,7 @@
      "program": "",
      "windows": {
        "program": "Spark.Console.exe",
-        "cwd": "build/Debug/bin/net8.0",
+        "cwd": "build/Debug/bin/net7.0",
        "args": [
          "test",
          "${input:sparkTest}"
@@ -191,27 +191,24 @@
      "request": "launch",
      "program": "",
      "linux": {
-        "program": "${workspaceFolder}/build/sparkd-debug-clang/src/app_sparkd/sparkd",
-      },
-      "args": [
-        "run"
-      ],
-      "cwd": "${workspaceFolder}/build/sparkd-debug-clang/src/app_sparkd",
-      "stopAtEntry": false,
-      "externalConsole": false,
-      "MIMode": "gdb",
-      "setupCommands": [
-        {
-          "description": "Enable pretty-printing for gdb",
-          "text": "-enable-pretty-printing",
-          "ignoreFailures": true
-        },
-        {
-          "description": "Set Disassembly Flavor to Intel",
-          "text": "-gdb-set disassembly-flavor intel",
-          "ignoreFailures": true
-        }
-      ]
+        "preLaunchTask": "Build Sparkd (Debug)",
+        "program": "${workspaceFolder}/build/sparkd-debug-clang/server/sparkd",
+        "args": [
+          "run"
+        ],
+        "stopAtEntry": false,
+        "cwd": "${workspaceFolder}/build/sparkd-debug-clang/server",
+        "environment": [],
+        "externalConsole": false,
+        "MIMode": "gdb",
+        "setupCommands": [
+          {
+            "description": "Enable pretty-printing for gdb",
+            "text": "-enable-pretty-printing",
+            "ignoreFailures": true
+          }
+        ]
+      }
    },
    {
      "name": "Sparkd (Skip Build)",
@@ -219,27 +216,74 @@
      "request": "launch",
      "program": "",
      "linux": {
-        "program": "${workspaceFolder}/build/sparkd-debug-clang/src/app_sparkd/sparkd",
-      },
-      "args": [
-        "run"
-      ],
-      "cwd": "${workspaceFolder}/build/sparkd-debug-clang/src/app_sparkd",
-      "stopAtEntry": false,
-      "externalConsole": false,
-      "MIMode": "gdb",
-      "setupCommands": [
-        {
-          "description": "Enable pretty-printing for gdb",
-          "text": "-enable-pretty-printing",
-          "ignoreFailures": true
-        },
-        {
-          "description": "Set Disassembly Flavor to Intel",
-          "text": "-gdb-set disassembly-flavor intel",
-          "ignoreFailures": true
-        }
-      ]
+        "program": "${workspaceFolder}/build/sparkd-debug-clang/server/sparkd",
+        "args": [
+          "run"
+        ],
+        "stopAtEntry": false,
+        "cwd": "${workspaceFolder}/build/sparkd-debug-clang/server",
+        "environment": [],
+        "externalConsole": false,
+        "MIMode": "gdb",
+        "setupCommands": [
+          {
+            "description": "Enable pretty-printing for gdb",
+            "text": "-enable-pretty-printing",
+            "ignoreFailures": true
+          }
+        ]
+      }
+    },
+    {
+      "name": "Sparkd Test",
+      "type": "cppdbg",
+      "request": "launch",
+      "program": "",
+      "linux": {
+        "preLaunchTask": "Build Sparkd (Debug)",
+        "program": "${workspaceFolder}/build/sparkd-debug-clang/server/sparkd",
+        "args": [
+          "test",
+          "${input:sparkdTest}",
+        ],
+        "stopAtEntry": false,
+        "cwd": "${workspaceFolder}/build/sparkd-debug-clang/server",
+        "environment": [],
+        "externalConsole": false,
+        "MIMode": "gdb",
+        "setupCommands": [
+          {
+            "description": "Enable pretty-printing for gdb",
+            "text": "-enable-pretty-printing",
+            "ignoreFailures": true
+          }
+        ]
+      }
+    },
+    {
+      "name": "Sparkd Test (Skip Build)",
+      "type": "cppdbg",
+      "request": "launch",
+      "program": "",
+      "linux": {
+        "program": "${workspaceFolder}/build/sparkd-debug-clang/server/sparkd",
+        "args": [
+          "test",
+          "${input:sparkdTest}",
+        ],
+        "stopAtEntry": false,
+        "cwd": "${workspaceFolder}/build/sparkd-debug-clang/server",
+        "environment": [],
+        "externalConsole": false,
+        "MIMode": "gdb",
+        "setupCommands": [
+          {
+            "description": "Enable pretty-printing for gdb",
+            "text": "-enable-pretty-printing",
+            "ignoreFailures": true
+          }
+        ]
+      }
    },
  ],
  "inputs": [
@@ -363,7 +363,7 @@
          "${workspaceFolder}/src/DataStreamSuite/DataStreamQueryAnalyzer/Configurations/DataStreamQueryAnalyzer.json",
        ],
        "options": {
-          "cwd": "${workspaceFolder}/build/${input:buildConfiguration}/bin/net8.0-windows"
+          "cwd": "${workspaceFolder}/build/${input:buildConfiguration}/bin/net7.0-windows"
        },
      },
      "icon": {
@@ -383,7 +383,7 @@
          "${workspaceFolder}/src/DataStreamSuite/DataStreamQueryAnalyzer/Configurations/DataStreamQueryAnalyzer.json",
        ],
        "options": {
-          "cwd": "${workspaceFolder}/build/Debug/bin/net8.0-windows"
+          "cwd": "${workspaceFolder}/build/Debug/bin/net7.0-windows"
        },
      },
      "icon": {
@@ -403,7 +403,7 @@
          "${workspaceFolder}/src/DataStreamSuite/DataStreamQueryAnalyzer/Configurations/DataStreamQueryAnalyzer.json",
        ],
        "options": {
-          "cwd": "${workspaceFolder}/build/Release/bin/net8.0-windows"
+          "cwd": "${workspaceFolder}/build/Release/bin/net7.0-windows"
        },
      },
      "icon": {
@@ -427,7 +427,7 @@
        "${workspaceFolder}/src/DataStreamSuite/DataStreamPublisher/Configurations/${input:dataStreamPublisherConfig}.json",
      ],
      "options": {
-        "cwd": "${workspaceFolder}/build/${input:buildConfiguration}/bin/net8.0"
+        "cwd": "${workspaceFolder}/build/${input:buildConfiguration}/bin/net7.0"
      },
      "icon": {
        "id": "run",
@@ -450,7 +450,7 @@
        "${workspaceFolder}/src/DataStreamSuite/DataStreamPublisher/Configurations/${input:dataStreamPublisherConfig}.json",
      ],
      "options": {
-        "cwd": "${workspaceFolder}/build/Debug/bin/net8.0"
+        "cwd": "${workspaceFolder}/build/Debug/bin/net7.0"
      },
      "icon": {
        "id": "run",
@@ -473,7 +473,7 @@
        "${workspaceFolder}/src/DataStreamSuite/DataStreamPublisher/Configurations/${input:dataStreamPublisherConfig}.json",
      ],
      "options": {
-        "cwd": "${workspaceFolder}/build/Release/bin/net8.0"
+        "cwd": "${workspaceFolder}/build/Release/bin/net7.0"
      },
      "icon": {
        "id": "run",
@@ -496,7 +496,7 @@
        "${workspaceFolder}/src/DataStreamSuite/PubSubServer/Configurations/${input:pubSubServerConfig}.json",
      ],
      "options": {
-        "cwd": "${workspaceFolder}/build/${input:buildConfiguration}/bin/net8.0"
+        "cwd": "${workspaceFolder}/build/${input:buildConfiguration}/bin/net7.0"
      },
      "icon": {
        "id": "run",
@@ -519,7 +519,7 @@
        "${workspaceFolder}/src/DataStreamSuite/PubSubServer/Configurations/${input:pubSubServerConfig}.json",
      ],
      "options": {
-        "cwd": "${workspaceFolder}/build/Debug/bin/net8.0"
+        "cwd": "${workspaceFolder}/build/Debug/bin/net7.0"
      },
      "icon": {
        "id": "run",
@@ -541,7 +541,7 @@
        "${workspaceFolder}/src/DataStreamSuite/PubSubServer/Configurations/${input:pubSubServerConfig}.json",
      ],
      "options": {
-        "cwd": "${workspaceFolder}/build/Debug/bin/net8.0"
+        "cwd": "${workspaceFolder}/build/Debug/bin/net7.0"
      },
      "icon": {
        "id": "run",
@@ -564,7 +564,7 @@
        "${workspaceFolder}/src/DataStreamSuite/PubSubServer/Configurations/${input:pubSubServerConfig}.json",
      ],
      "options": {
-        "cwd": "${workspaceFolder}/build/Release/bin/net8.0"
+        "cwd": "${workspaceFolder}/build/Release/bin/net7.0"
      },
      "icon": {
        "id": "run",
@@ -586,7 +586,7 @@
        "${workspaceFolder}/src/DataStreamSuite/PubSubServer/Configurations/${input:pubSubServerConfig}.json",
      ],
      "options": {
-        "cwd": "${workspaceFolder}/build/Release/bin/net8.0"
+        "cwd": "${workspaceFolder}/build/Release/bin/net7.0"
      },
      "icon": {
        "id": "run",
@@ -601,7 +601,7 @@
      "windows": {
        "command": "Spark.exe",
        "options": {
-          "cwd": "${workspaceFolder}/build/${input:buildConfiguration}/bin/net8.0-windows"
+          "cwd": "${workspaceFolder}/build/${input:buildConfiguration}/bin/net7.0-windows"
        },
      },
      "icon": {
@@ -617,7 +617,7 @@
      "windows": {
        "command": "Spark.exe",
        "options": {
-          "cwd": "${workspaceFolder}/build/Debug/bin/net8.0-windows"
+          "cwd": "${workspaceFolder}/build/Debug/bin/net7.0-windows"
        },
      },
      "icon": {
@@ -633,7 +633,7 @@
      "windows": {
        "command": "Spark.exe",
        "options": {
-          "cwd": "${workspaceFolder}/build/Release/bin/net8.0-windows"
+          "cwd": "${workspaceFolder}/build/Release/bin/net7.0-windows"
        },
      },
      "icon": {
@@ -648,7 +648,7 @@
      "windows": {
        "command": "Spark.exe",
        "options": {
-          "cwd": "${workspaceFolder}/build/${input:buildConfiguration}/bin/net8.0-windows"
+          "cwd": "${workspaceFolder}/build/${input:buildConfiguration}/bin/net7.0-windows"
        },
      },
      "icon": {
@@ -673,7 +673,7 @@
        "${input:sparkConfigQuery}",
      ],
      "options": {
-        "cwd": "${workspaceFolder}/build/${input:buildConfiguration}/bin/net8.0"
+        "cwd": "${workspaceFolder}/build/${input:buildConfiguration}/bin/net7.0"
      },
      "icon": {
        "id": "run",
@@ -754,7 +754,7 @@
          "${input:sparkTest}"
        ],
        "options": {
-          "cwd": "${workspaceFolder}/build/Debug/bin/net8.0",
+          "cwd": "${workspaceFolder}/build/Debug/bin/net7.0",
        }
      },
    },
@@ -1,10 +1,8 @@
 cmake_minimum_required(VERSION 3.12)
 project(spark)
 include(GNUInstallDirs)
-include(${CMAKE_SOURCE_DIR}/cmake/GitVersion.cmake)
 set(CMAKE_CXX_STANDARD 20)

-get_version_from_git()
 if(WIN32)
  set(platform "windows")
 else()
@@ -22,9 +20,9 @@ set(
 # can run simultaneously without fighting over the same shm files (e.g., sparkd,
 # api server, etc.).
 set(SHARED_MEMORY_PREFIX "bts" CACHE STRING "Prefix to use when creating shared memory files")
-set(EXTERNALS_BIN "${EXTERNALS_ROOT}/bin")
-set(EXTERNALS_INCLUDE "${EXTERNALS_ROOT}/include")
-set(EXTERNALS_LIB "${EXTERNALS_ROOT}/lib/$<CONFIG>")
+set(SPARK_BIN "${EXTERNALS_ROOT}/bin")
+set(SPARK_INCLUDE "${EXTERNALS_ROOT}/include")
+set(SPARK_LIB "${EXTERNALS_ROOT}/lib/$<CONFIG>")
 set(BUILD_DOCS OFF CACHE BOOL "Build documentation")
 set(BUILD_SDK OFF CACHE BOOL "Build the sdk")
 set(EXPORT_MD_CONSUMER OFF CACHE BOOL "Export (extern) C functions for MD consumer interop")
@@ -36,8 +34,8 @@ else()
  set(BUILD_ADMIN_WEB_APP ON CACHE BOOL "Build the Admin WebApp")
 endif()

-include_directories(BEFORE SYSTEM ${EXTERNALS_INCLUDE} ${EXTERNALS_ROOT}/include/zlib)
-link_directories(${EXTERNALS_LIB})
+include_directories(BEFORE SYSTEM ${SPARK_INCLUDE} ${EXTERNALS_ROOT}/include/zlib)
+link_directories(${SPARK_LIB})

 add_compile_definitions(SHARED_MEMORY_PREFIX="${SHARED_MEMORY_PREFIX}")
 add_compile_definitions(APP_DATA_DIR="${APP_DATA_DIR}")
@@ -47,8 +45,12 @@ endif()
 add_compile_definitions(EXPORT_MD_CONSUMER="${EXPORT_MD_CONSUMER}")
 if(WIN32)
  # subdirs
-  add_subdirectory(src/lib_spark)
+  add_subdirectory(src/lib)
 else()
+  # make sparkd look for libspark.so in its directory
+  set(CMAKE_BUILD_WITH_INSTALL_RPATH TRUE)
+  set(CMAKE_INSTALL_RPATH ".")
+
  if(BUILD_SPARKD)
    # Note: All macros that begin w/ 'CCAPI' are ccapi's macros (defined and/or
    # read by their code). We override some below.
@@ -113,15 +115,12 @@ else()
  endif()

  # subdirs
-  add_subdirectory(src/lib_spark)
  if (BUILD_SPARKD)
-    add_subdirectory(src/lib_python_interop)
-    add_subdirectory(src/lib_onixs_fix)
    add_subdirectory(src/lib_binance_execution)
-    add_subdirectory(src/lib_okx_execution)
-    add_subdirectory(src/app_sparkd)
+    add_subdirectory(src/server/sparkd)
    add_subdirectory(src/data_files)
  endif()
+  add_subdirectory(src/lib)
  if (BUILD_ADMIN_WEB_APP)
    add_subdirectory(src/Spark.Admin.WebApp)
  endif()
@@ -131,4 +130,37 @@ else()
  if(BUILD_SDK)
    add_subdirectory(src/sdk)
  endif()
+
+  if(BUILD_SPARKD)
+    add_custom_command(
+      OUTPUT ${CMAKE_BINARY_DIR}/server
+      COMMAND mkdir -p ${CMAKE_BINARY_DIR}/server
+      COMMAND ln ARGS -sf ${CMAKE_BINARY_DIR}/src/server/sparkd/sparkd ${CMAKE_BINARY_DIR}/server
+      COMMAND ln ARGS -sf ${CMAKE_BINARY_DIR}/data_files ${CMAKE_BINARY_DIR}/server
+      COMMAND ln ARGS -sf ${CMAKE_BINARY_DIR}/src/lib/libspark.so ${CMAKE_BINARY_DIR}/server
+      COMMAND ln ARGS -sf ${EXTERNALS_ROOT}/lib/${CMAKE_BUILD_TYPE}/libonixs-fix-engine.so.4 ${CMAKE_BINARY_DIR}/server
+      
+      COMMAND
+        ln ARGS -sf
+        ${EXTERNALS_ROOT}/lib/Release/libOnixS.IceImpactMulticastPriceFeedHandlerCpp.so.3.16.0.0
+        ${CMAKE_BINARY_DIR}/server
+      COMMAND ln ARGS -sf ${EXTERNALS_ROOT}/lib/Release/libACE-5.7.5.so ${CMAKE_BINARY_DIR}/server
+      COMMAND mkdir ARGS -p ${CMAKE_BINARY_DIR}/server/plugins
+      COMMENT "Creating server runtime layout"
+      DEPENDS sparkd data_files_dir)
+
+    add_custom_target(server_layout ALL DEPENDS ${CMAKE_BINARY_DIR}/server)
+
+    install(
+      DIRECTORY "${CMAKE_BINARY_DIR}/server/plugins"
+      DESTINATION "${CMAKE_INSTALL_BINDIR}")
+    install(
+      FILES
+      "${SPARK_LIB}/libACE-5.7.5.so"
+      "${SPARK_LIB}/libOnixS.IceImpactMulticastPriceFeedHandlerCpp.so.3.16.0.0"
+      "${SPARK_LIB}/libonixs-fix-engine.so.4.10.1"
+      "${SPARK_LIB}/libonixs-fix-engine.so.4"
+      "${SPARK_LIB}/libonixs-fix-engine.so"
+      DESTINATION "${CMAKE_INSTALL_BINDIR}")
+  endif()
 endif()
@@ -31,8 +31,9 @@
        "CMAKE_SHARED_LINKER_FLAGS_RELEASE": "/OPT:REF /OPT:ICF /LTCG",
        "CMAKE_TOOLCHAIN_FILE": "${sourceDir}/vcpkg/scripts/buildsystems/vcpkg.cmake",
        "EXTERNALS_ROOT": "${sourceDir}/externals/windows",
-        "CONFIG_DEFAULTS_PATH": "spark/config/profiles/default/*.conf",
-        "DOTNET_BINARY_DIR": "${sourceDir}/build/$<CONFIG>/bin/net8.0;${sourceDir}/build/$<CONFIG>/bin/net8.0-windows",
+        "CONFIG_DEFAULTS_PATH": "config/profiles/default/*.conf",
+        "DOTNET_BINARY_DIR": "${sourceDir}/build/$<CONFIG>/bin/net7.0",
+        "DOTNET_WINDOWS_BINARY_DIR": "${sourceDir}/build/$<CONFIG>/bin/net7.0-windows",
        "EXPORT_MD_CONSUMER": {
          "type": "BOOL",
          "value": "ON"
@@ -57,18 +58,12 @@
        "APP_DATA_DIR_OVERRIDE": "${dollar}{SPARKD_DATA_DIR}",
        "CMAKE_BUILD_WITH_INSTALL_RPATH": {
          "type": "BOOL",
-          "value": "OFF"
+          "value": "ON"
        },
        "CMAKE_CXX_STANDARD": "20",
        "CMAKE_INSTALL_RPATH": ".",
        "CMAKE_INSTALL_PREFIX": "${sourceDir}/install/${presetName}",
-        "CMAKE_PREFIX_PATH": "${sourceDir}/cmake/;${sourceDir}/.venv/lib/python3.10/site-packages/pybind11/share/cmake/pybind11",
        "CMAKE_EXE_LINKER_FLAGS": "-Wno-undef",
-        "CMAKE_TOOLCHAIN_FILE": "${sourceDir}/vcpkg/scripts/buildsystems/vcpkg.cmake",
-        "DATABENTO_USE_EXTERNAL_JSON": {
-          "type": "BOOL",
-          "value": "ON"
-        },
        "EXTERNALS_ROOT": "${sourceDir}/externals/linux",
        "BUILD_SDK": {
          "type": "BOOL",
@@ -76,7 +71,7 @@
        },
        "BUILD_ADMIN_WEB_APP": {
          "type": "BOOL",
-          "value": "OFF"
+          "value": "ON"
        },
        "SHARED_MEMORY_PREFIX": "sparkd"
      }
@@ -93,7 +88,7 @@
          "value": "ON"
        },
        "CCAPI_LOG_LEVEL": "Info",
-        "CONFIG_DEFAULTS_PATH": "spark/config/profiles/md_dev/*.conf;spark/config/profiles/default/*.conf"
+        "CONFIG_DEFAULTS_PATH": "config/profiles/md_dev/*.conf;config/profiles/default/*.conf"
      }
    },

@@ -104,7 +99,7 @@
      "cacheVariables": {
        "CMAKE_BUILD_TYPE": "Release",
        "CCAPI_LOG_LEVEL": "Warn",
-        "CONFIG_DEFAULTS_PATH": "spark/config/profiles/md_prod/*.conf;spark/config/profiles/default/*.conf"
+        "CONFIG_DEFAULTS_PATH": "config/profiles/md_prod/*.conf;config/profiles/default/*.conf"
      }
    },

@@ -186,12 +181,8 @@
        "CMAKE_CXX_FLAGS_DEBUG": "-g -fPIC -Wall -pedantic -Wextra -Werror -Wno-missing-braces -Wno-ignored-qualifiers --std=c++20 -fdiagnostics-color -DDEBUG",
        "CMAKE_CXX_FLAGS_RELEASE": "-fPIC -Wall -pedantic -Wextra -Werror -Wno-missing-braces -Wno-ignored-qualifiers --std=c++20 -fdiagnostics-color -fno-omit-frame-pointer -DNDEBUG -O3",
        "CMAKE_INSTALL_PREFIX": "${sourceDir}/install/${presetName}",
-        "CMAKE_EXPORT_COMPILE_COMMANDS": {
-          "type": "BOOL",
-          "value": "ON"
-        },
-        "CONFIG_DEFAULTS_PATH": "spark/config/profiles/default/*.conf",
-        "DOTNET_BINARY_DIR": "${sourceDir}/build/$<CONFIG>/bin/net8.0",
+        "CONFIG_DEFAULTS_PATH": "config/profiles/default/*.conf",
+        "DOTNET_BINARY_DIR": "${sourceDir}/build/$<CONFIG>/bin/net7.0",
        "EXPORT_MD_CONSUMER": {
          "type": "BOOL",
          "value": "ON"
@@ -231,7 +222,7 @@
      "cacheVariables": {
        "APP_DATA_DIR": "${dollar}{APPDATA}/bts/DataStream",
        "APP_DATA_DIR_OVERRIDE": "${dollar}{DATA_STREAM_DATA_DIR}",
-        "CONFIG_DEFAULTS_PATH": "spark/config/profiles/default/*.conf;spark/config/profiles/data_stream/*.conf",
+        "CONFIG_DEFAULTS_PATH": "config/profiles/default/*.conf;config/profiles/data_stream/*.conf",
        "SHARED_MEMORY_PREFIX": "data_stream"
      }
    },
@@ -266,7 +257,8 @@
        "CMAKE_CXX_COMPILER": "clang++",
        "CMAKE_CXX_FLAGS_DEBUG": "-g -fPIC -Wall -pedantic -Wextra -Werror -Wno-missing-braces -Wno-ignored-qualifiers --std=c++20 -fdiagnostics-color -DDEBUG",
        "CMAKE_CXX_FLAGS_RELEASE": "-fPIC -Wall -pedantic -Wextra -Werror -Wno-missing-braces -Wno-ignored-qualifiers --std=c++20 -fdiagnostics-color -fno-omit-frame-pointer -DNDEBUG -O3",
-        "CONFIG_DEFAULTS_PATH": "spark/config/profiles/default/*.conf;spark/config/profiles/data_stream/*.conf",
+        "CMAKE_INSTALL_PREFIX": "${sourceDir}/install/${presetName}",
+        "CONFIG_DEFAULTS_PATH": "config/profiles/default/*.conf;config/profiles/data_stream/*.conf",
        "DOTNET_BINARY_DIR": "${sourceDir}/build/$<CONFIG>/bin/net7.0",
        "EXPORT_MD_CONSUMER": {
          "type": "BOOL",
@@ -1,7 +0,0 @@
-include_guard(GLOBAL)
-
-macro(export_to_parent)
-  foreach(var IN ITEMS ${ARGN})
-    set(${var} ${${var}} PARENT_SCOPE)
-  endforeach()
-endmacro()
@@ -1,48 +0,0 @@
-include_guard(GLOBAL)
-
-function(get_version_from_git)
-  find_package(Git QUIET)
-  if(NOT Git_FOUND)
-    message(WARNING "Git not found")
-    return()
-  endif()
-
-  execute_process(
-    COMMAND ${GIT_EXECUTABLE} describe --tags --always
-    WORKING_DIRECTORY ${CMAKE_SOURCE_DIR}
-    OUTPUT_VARIABLE GIT_TAG
-    OUTPUT_STRIP_TRAILING_WHITESPACE
-    RESULT_VARIABLE GIT_RESULT
-  )
-
-  if(NOT GIT_RESULT EQUAL 0)
-    message(WARNING "Failed to get git tag")
-    return()
-  endif()
-
-  execute_process(
-    COMMAND ${GIT_EXECUTABLE} rev-parse --short=7 HEAD
-    WORKING_DIRECTORY ${CMAKE_SOURCE_DIR}
-    OUTPUT_VARIABLE GIT_COMMIT_SHORT_HASH
-    OUTPUT_STRIP_TRAILING_WHITESPACE
-  )
-  set(PROJECT_VERSION_BUILD_METADATA ${GIT_COMMIT_SHORT_HASH})
-  set(PROJECT_VERSION_BUILD_METADATA ${GIT_COMMIT_SHORT_HASH} PARENT_SCOPE)
-
-  string(REGEX REPLACE "^v" "" CLEAN_TAG "${GIT_TAG}")
-  if(CLEAN_TAG MATCHES "^([0-9]+)\\.([0-9]+)\\.([0-9]+)(-.*)?$")
-    set(PROJECT_VERSION_MAJOR ${CMAKE_MATCH_1})
-    set(PROJECT_VERSION_MAJOR ${CMAKE_MATCH_1} PARENT_SCOPE)
-    set(PROJECT_VERSION_MINOR ${CMAKE_MATCH_2})
-    set(PROJECT_VERSION_MINOR ${CMAKE_MATCH_2} PARENT_SCOPE)
-    set(PROJECT_VERSION_PATCH ${CMAKE_MATCH_3})
-    set(PROJECT_VERSION_PATCH ${CMAKE_MATCH_3} PARENT_SCOPE)
-
-    set(PROJECT_VERSION "${CMAKE_MATCH_1}.${CMAKE_MATCH_2}.${CMAKE_MATCH_3}") 
-    set(PROJECT_VERSION "${CMAKE_MATCH_1}.${CMAKE_MATCH_2}.${CMAKE_MATCH_3}" PARENT_SCOPE)
-    set(PROJECT_VERSION_FULL "${CMAKE_MATCH_1}.${CMAKE_MATCH_2}.${CMAKE_MATCH_3}+${GIT_COMMIT_SHORT_HASH}")
-    set(PROJECT_VERSION_FULL "${CMAKE_MATCH_1}.${CMAKE_MATCH_2}.${CMAKE_MATCH_3}+${GIT_COMMIT_SHORT_HASH}" PARENT_SCOPE)
-  else()
-    message(WARNING "Tag '${CLEAN_TAG}' does not match semver format")
-  endif()
-endfunction()
@@ -1,142 +0,0 @@
-include_guard(DIRECTORY)
-include(${CMAKE_SOURCE_DIR}/cmake/ExportToParent.cmake)
-include(${CMAKE_SOURCE_DIR}/cmake/InstallBasic.cmake)
-include(${CMAKE_SOURCE_DIR}/cmake/ProtobufGenerateCpp.cmake)
-include(${CMAKE_SOURCE_DIR}/cmake/SetFirstDefined.cmake)
-include(${CMAKE_SOURCE_DIR}/cmake/safe_dep_glob.cmake)
-
-function(init_library)
-  set(_options SHARED STATIC)
-  set(_one_value_args
-    NAME
-    SOURCE_DIR
-    INCLUDE_DIR
-  )
-  set(_multi_value_args)
-  cmake_parse_arguments(_ARGS "${_options}" "${_one_value_args}" "${_multi_value_args}" ${ARGN})
-  if (_ARGS_SHARED AND _ARGS_STATIC)
-    message(FATAL_ERROR "Invalid args passed to init_library: cannot specify STATIC and SHARED")
-  endif()
-  set_first_defined(_name _ARGS_NAME PROJECT_NAME)
-  set_first_defined(_source_dir _ARGS_SOURCE_DIR _name)
-  set_first_defined(_include_dir _ARGS_INCLUDE_DIR _source_dir)
-
-  # Cpp sources
-  file(
-    GLOB_RECURSE ${_name}_SOURCES
-    RELATIVE ${CMAKE_CURRENT_SOURCE_DIR}
-    ${_source_dir}/*.cpp
-  )
-  # Note: cpp files inside 'tests' dir(s) are considered tests.
-  # Remove tests from main sources, and add them to test sources
-  foreach(item ${${_name}_SOURCES})
-    if(${item} MATCHES "tests/.*\.cpp$")
-      list(APPEND ${_name}_TEST_SOURCES ${item})
-    endif()
-  endforeach()
-  foreach(item ${${_name}_TEST_SOURCES})
-    list(REMOVE_ITEM ${_name}_SOURCES ${item})
-  endforeach()
-
-  # Proto sources 
-  file(
-    GLOB_RECURSE ${_name}_PROTO_SOURCES
-    RELATIVE ${CMAKE_CURRENT_SOURCE_DIR}
-    ${_source_dir}/protobufs/*.proto
-  )
-  update_deps_file("${${_name}_PROTO_SOURCES};")
-  protobuf_generate_cpp(
-    CXX_SOURCES ${_name}_CXX_PROTO_SOURCES
-    CXX_HEADERS ${_name}_CXX_PROTO_HEADERS
-    CXX_OUT_DIR ${CMAKE_CURRENT_BINARY_DIR}
-    PROTO_INCLUDE_DIRS
-      ${EXTERNALS_INCLUDE}
-      ${CMAKE_CURRENT_SOURCE_DIR}
-    PROTO_SOURCES
-      ${${_name}_PROTO_SOURCES}
-  )
-
-  export_to_parent(
-    ${_name}_SOURCES
-    ${_name}_TEST_SOURCES
-    ${_name}_PROTO_SOURCES
-    ${_name}_CXX_PROTO_SOURCES
-    ${_name}_CXX_PROTO_HEADERS
-  )
-
-  # Add main lib
-  if (_ARGS_STATIC)
-    add_library(${_name} STATIC ${${_name}_SOURCES} ${${_name}_CXX_PROTO_SOURCES})
-  elseif(_ARGS_SHARED)
-    add_library(${_name} SHARED ${${_name}_SOURCES} ${${_name}_CXX_PROTO_SOURCES})
-  else()
-    add_library(${_name} ${${_name}_SOURCES} ${${_name}_CXX_PROTO_SOURCES})
-  endif()
-  target_include_directories(
-    ${_name}
-    SYSTEM
-    BEFORE
-    PUBLIC
-      $<BUILD_INTERFACE:${EXTERNALS_ROOT}/include>
-      $<BUILD_INTERFACE:${EXTERNALS_ROOT}/include/zlib>
-  )
-  target_include_directories(
-    ${_name}
-    PUBLIC
-      $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}>
-      $<INSTALL_INTERFACE:${CMAKE_INSTALL_INCLUDEDIR}>
-  )
-  # Include protos (if any)
-  if (${_name}_PROTO_SOURCES)
-    target_include_directories(
-      ${_name}
-      PUBLIC
-        $<BUILD_INTERFACE:${CMAKE_CURRENT_BINARY_DIR}>
-    )
-  endif()
-  target_link_directories(
-    ${_name}
-    PUBLIC
-      $<BUILD_INTERFACE:${CMAKE_CURRENT_BINARY_DIR}>
-      $<BUILD_INTERFACE:${EXTERNALS_ROOT}/lib/$<CONFIG>>
-      $<INSTALL_INTERFACE:${CMAKE_INSTALL_LIBDIR}>
-  )
-  # Install main lib
-  foreach(_hdr ${${_name}_CXX_PROTO_HEADERS})
-    file(RELATIVE_PATH _relative_path ${CMAKE_CURRENT_BINARY_DIR} ${_hdr})
-    install(FILES ${_hdr} DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}/${_relative_path})
-  endforeach()
-  install_basic(
-    NAME ${_name}
-    INCLUDE_DIR ${_include_dir}
-  )
-
-  # Add tests executable (if there are any)
-  if (${_name}_TEST_SOURCES AND NOT WIN32)
-    add_executable(${_name}-tests ${${_name}_TEST_SOURCES})
-
-    get_target_property(_lib_type ${_name} TYPE)
-    if (_lib_type STREQUAL "SHARED_LIBRARY")
-      # Set RPATH so the tests exe knows where to find the shared lib.
-      set_target_properties(
-        ${_name}-tests
-        PROPERTIES
-          BUILD_RPATH "$ORIGIN"
-          INSTALL_RPATH "$ORIGIN/../${CMAKE_INSTALL_LIBDIR}"
-      )
-    endif()
-
-    # Link main lib
-    target_link_directories(
-      ${_name}-tests
-      PRIVATE
-        $<BUILD_INTERFACE:${CMAKE_CURRENT_BINARY_DIR}>
-        $<INSTALL_INTERFACE:${CMAKE_INSTALL_LIBDIR}>
-    )
-    target_link_libraries(${_name}-tests PRIVATE ${_name})
-
-    # Catch2
-    find_package(Catch2 CONFIG REQUIRED COMPONENTS Catch2WithMain)
-    target_link_libraries(${_name}-tests PRIVATE Catch2::Catch2WithMain)
-  endif()
-endfunction()
@@ -1,71 +0,0 @@
-include_guard(DIRECTORY)
-include(GNUInstallDirs)
-include(${CMAKE_SOURCE_DIR}/cmake/ExportToParent.cmake)
-include(${CMAKE_SOURCE_DIR}/cmake/SetFirstDefined.cmake)
-
-function(init_python_module)
-  set(_options)
-  set(_one_value_args
-    NAME
-  )
-  set(_multi_value_args
-    SOURCES
-  )
-  cmake_parse_arguments(_ARGS "${_options}" "${_one_value_args}" "${_multi_value_args}" ${ARGN})
-  set_first_defined(_name _ARGS_NAME PROJECT_NAME)
-  set_first_defined(_sources _ARGS_SOURCES ${_name}_SOURCES)
-
-  set(module_proj ${_name}_module)
-  set(${module_proj}_SOURCES ${_sources})
-  pybind11_add_module(${module_proj} SHARED ${_sources})
-  target_link_libraries(
-    ${module_proj}
-    PUBLIC
-      pybind11::embed
-      spark
-      ${_name}
-  )
-  target_include_directories(
-    ${module_proj}
-    PRIVATE
-      $<BUILD_INTERFACE:${pybind11_INCLUDE_DIRS}>
-  )
-  target_include_directories(
-    ${module_proj}
-    PUBLIC
-      ${Python3_INCLUDE_DIRS}
-      $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}>
-      $<INSTALL_INTERFACE:${CMAKE_INSTALL_INCLUDEDIR}>
-  )
-  add_custom_command(
-    OUTPUT ${CMAKE_CURRENT_BINARY_DIR}/stubs/${_name}.pyi
-    DEPENDS ${_name} ${module_proj}
-    COMMAND PYTHONPATH=${CMAKE_CURRENT_BINARY_DIR} pybind11-stubgen --numpy-array-use-type-var --ignore-invalid-expressions '.*' ${_name}_module
-    COMMENT "Generating py stubs"
-  )
-  add_custom_target(
-    "${_name}_py_stubs"
-    ALL
-    DEPENDS
-      ${CMAKE_CURRENT_BINARY_DIR}/stubs/${_name}.pyi
-  )
-  install(
-    TARGETS ${module_proj}
-    EXPORT ${module_proj}Targets
-    RUNTIME
-      DESTINATION ${CMAKE_INSTALL_BINDIR}
-    LIBRARY
-      DESTINATION ${CMAKE_INSTALL_LIBDIR}
-    ARCHIVE
-      DESTINATION ${CMAKE_INSTALL_LIBDIR}
-  )
-  foreach(dir ${pybind11_INCLUDE_DIRS})
-    install(DIRECTORY ${dir} DESTINATION ${CMAKE_INSTALL_INCLUDEDIR})
-  endforeach()
-  install(
-    EXPORT ${module_proj}Targets
-    FILE ${module_proj}Targets.cmake
-    NAMESPACE ${_name}::
-    DESTINATION ${CMAKE_INSTALL_LIBDIR}/cmake/${_name}
-  )
-endfunction()
@@ -1,65 +0,0 @@
-include_guard(DIRECTORY)
-include(GNUInstallDirs)
-
-function(init_python_plugin)
-  set(_options)
-  set(_one_value_args NAME PLUGIN_NAME)
-  set(_multi_value_args SOURCES)
-  cmake_parse_arguments(_ARGS "${_options}" "${_one_value_args}" "${_multi_value_args}" ${ARGN})
-  set_first_defined(_name _ARGS_NAME PROJECT_NAME)
-  set(_plugin_name ${_ARGS_PLUGIN_NAME})
-  if (NOT DEFINED _ARGS_PLUGIN_NAME)
-    set(_plugin_name ${_name}_plugins)
-  endif()
-  set(${_plugin_name}_SOURCES ${_ARGS_SOURCES})
-  if (NOT DEFINED _ARGS_SOURCES)
-    file(
-      GLOB_RECURSE ${_plugin_name}_SOURCES
-      RELATIVE ${CMAKE_CURRENT_SOURCE_DIR}
-      ${_plugin_name}/*.py)
-  endif()
-  set(src_outs)
-  foreach(_src ${${_plugin_name}_SOURCES})
-    set(_src_in ${CMAKE_CURRENT_SOURCE_DIR}/${_src})
-    set(_src_out ${CMAKE_CURRENT_BINARY_DIR}/${_src})
-    add_custom_command(
-      OUTPUT ${_src_out}
-      COMMAND ${CMAKE_COMMAND} -E copy ${_src_in} ${_src_out}
-      DEPENDS ${_src_in}
-      COMMENT "Copying ${_plugin_name} plugin ${_src} to ${_src_out}"
-    )
-    list(APPEND src_outs ${_src_out})
-  endforeach()
-  
-  add_custom_target(copy_${_plugin_name} ALL DEPENDS ${src_outs})
-  
-  add_library(${_plugin_name} INTERFACE)
-  add_library(${_name}::${_plugin_name} ALIAS ${_plugin_name})
-  
-  set_target_properties(
-    ${_plugin_name}
-    PROPERTIES
-      INTERFACE_PYTHON_SCRIPTS_DIR
-        $<BUILD_INTERFACE:${CMAKE_CURRENT_BINARY_DIR}/${_plugin_name}>$<INSTALL_INTERFACE:${CMAKE_INSTALL_BINDIR}/${_plugin_name}>
-  )
-  
-  set_target_properties(
-    ${_plugin_name}
-    PROPERTIES
-    INTERFACE_PYTHON_SCRIPTS $<BUILD_INTERFACE:${CMAKE_CURRENT_BINARY_DIR}/${${_plugin_name}_SOURCES}>$<INSTALL_INTERFACE:${CMAKE_INSTALL_BINDIR}/${${_plugin_name}_SOURCES}>
-  )
-  
-  install(
-    TARGETS ${_plugin_name}
-    EXPORT ${_plugin_name}Targets
-    DESTINATION ${CMAKE_INSTALL_BINDIR}
-  )
-  
-  install(
-    EXPORT ${_plugin_name}Targets
-    FILE ${_plugin_name}Targets.cmake
-    NAMESPACE ${_name}::
-    DESTINATION ${CMAKE_INSTALL_LIBDIR}/cmake/${_name}
-  )
-  install(DIRECTORY ${_plugin_name} DESTINATION ${CMAKE_INSTALL_BINDIR})
-endfunction()
@@ -1,39 +0,0 @@
-include_guard(DIRECTORY)
-include(GNUInstallDirs)
-include(${CMAKE_SOURCE_DIR}/cmake/PackBasic.cmake)
-include(${CMAKE_SOURCE_DIR}/cmake/SetFirstDefined.cmake)
-
-function(install_basic)
-  set(_options)
-  set(_one_value_args NAME INCLUDE_DIR)
-  set(_multi_value_args)
-  cmake_parse_arguments(_ARGS "${_options}" "${_one_value_args}" "${_multi_value_args}" ${ARGN})
-  set_first_defined(_name _ARGS_NAME PROJECT_NAME)
-  set_first_defined(_include_dir _ARGS_INCLUDE_DIR _name)
-
-  file(
-    GLOB_RECURSE ${_name}_HEADERS
-    RELATIVE ${CMAKE_CURRENT_SOURCE_DIR}
-    ${_include_dir}/*.h)
-
-  target_sources(${_name}
-    PUBLIC
-      FILE_SET HEADERS FILES ${${_name}_HEADERS})
-
-  install(
-    TARGETS ${_name}
-    EXPORT ${_name}Targets
-    RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR}
-    LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR}
-    ARCHIVE DESTINATION ${CMAKE_INSTALL_LIBDIR}
-    FILE_SET HEADERS DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}
-  )
-
-  install(
-    EXPORT ${_name}Targets
-    FILE ${_name}Targets.cmake
-    NAMESPACE ${_name}::
-    DESTINATION ${CMAKE_INSTALL_LIBDIR}/cmake/${_name})
-
-  pack_basic()
-endfunction()
@@ -1,44 +0,0 @@
-include_guard(DIRECTORY)
-include(CMakePackageConfigHelpers)
-include(${CMAKE_SOURCE_DIR}/cmake/SetFirstDefined.cmake)
-
-function(pack_basic)
-  set(_options)
-  set(_one_value_args NAME)
-  set(_multi_value_args)
-  cmake_parse_arguments(_ARGS "${_options}" "${_one_value_args}" "${_multi_value_args}" ${ARGN})
-  set_first_defined(_name _ARGS_NAME PROJECT_NAME)
-
-  find_package(Git QUIET)
-
-  if(GIT_FOUND)
-    execute_process(
-      COMMAND ${GIT_EXECUTABLE} describe --tags --always --dirty
-      WORKING_DIRECTORY ${CMAKE_SOURCE_DIR}
-      OUTPUT_VARIABLE PROJECT_VERSION
-      OUTPUT_STRIP_TRAILING_WHITESPACE
-    )
-  endif()
-
-  if(NOT GIT_FOUND OR PROJECT_VERSION STREQUAL "")
-    set(PROJECT_VERSION "0.0.0")
-  endif()
-
-  write_basic_package_version_file(
-    "${CMAKE_CURRENT_BINARY_DIR}/${_name}ConfigVersion.cmake"
-    VERSION ${PROJECT_VERSION}
-    COMPATIBILITY AnyNewerVersion
-  )
-
-  configure_package_config_file(
-    "${CMAKE_CURRENT_SOURCE_DIR}/Config.cmake.in"
-    "${CMAKE_CURRENT_BINARY_DIR}/${_name}Config.cmake"
-    INSTALL_DESTINATION ${CMAKE_INSTALL_LIBDIR}/cmake/${_name}
-  )
-
-  install(
-    FILES
-      "${CMAKE_CURRENT_BINARY_DIR}/${_name}Config.cmake"
-      "${CMAKE_CURRENT_BINARY_DIR}/${_name}ConfigVersion.cmake"
-    DESTINATION ${CMAKE_INSTALL_LIBDIR}/cmake/${_name})
-endfunction()
@@ -1,23 +1,12 @@
-set(Protobuf_PROTOC_EXECUTABLE "${EXTERNALS_BIN}/protoc")
-set(PROTOBUF_INCLUDE_DIRS "${EXTERNALS_INCLUDE}")
+set(Protobuf_PROTOC_EXECUTABLE "${SPARK_BIN}/protoc")
+set(PROTOBUF_INCLUDE_DIRS "${SPARK_INCLUDE}")
 set(PROTOBUF_STATIC_LIB "libprotobuf")
-set(_protobuf_src_dir ${CMAKE_SOURCE_DIR}/src/lib_spark)
-set(_protobuf_bin_dir ${CMAKE_BINARY_DIR}/src/lib_spark)

-function(Protobuf_Generate_Cpp)
-  set(_options)
-  set(_one_value_args CXX_SOURCES CXX_HEADERS CXX_OUT_DIR)
-  set(_multi_value_args PROTO_INCLUDE_DIRS PROTO_SOURCES)
-  cmake_parse_arguments(_ARGS "${_options}" "${_one_value_args}" "${_multi_value_args}" ${ARGN})
-  set(${_ARGS_CXX_SOURCES})
-  set(${_ARGS_CXX_HEADERS})
+function(Protobuf_Generate_Cpp SRCS HDRS)
+  set(${SRCS})
+  set(${HDRS})

-  set(_proto_include_cli_args)
-  foreach(_dir ${_ARGS_PROTO_INCLUDE_DIRS})
-    list(APPEND _proto_include_cli_args -I ${_dir})
-  endforeach()
-
-  foreach(FIL ${_ARGS_PROTO_SOURCES})
+  foreach(FIL ${ARGN})
    get_filename_component(ABS_FIL ${FIL} ABSOLUTE)
    get_filename_component(FIL_WE ${FIL} NAME_WE)
    get_filename_component(FIL_DIR ${FIL} DIRECTORY)
@@ -25,56 +14,48 @@ function(Protobuf_Generate_Cpp)
      set(FIL_WE "${FIL_DIR}/${FIL_WE}")
    endif()

-    list(APPEND ${_ARGS_CXX_SOURCES} "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.pb.cc")
-    list(APPEND ${_ARGS_CXX_HEADERS} "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.pb.h")
+    list(APPEND ${SRCS} "${CMAKE_CURRENT_BINARY_DIR}/${FIL_WE}.pb.cc")
+    list(APPEND ${HDRS} "${CMAKE_CURRENT_BINARY_DIR}/${FIL_WE}.pb.h")

    add_custom_command(
-      OUTPUT "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.pb.cc"
-             "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.pb.h"
+      OUTPUT "${CMAKE_CURRENT_BINARY_DIR}/${FIL_WE}.pb.cc"
+             "${CMAKE_CURRENT_BINARY_DIR}/${FIL_WE}.pb.h"
      COMMAND
-        ${Protobuf_PROTOC_EXECUTABLE} ARGS --cpp_out=${_ARGS_CXX_OUT_DIR} ${_proto_include_cli_args} ${ABS_FIL}
+        ${Protobuf_PROTOC_EXECUTABLE} ARGS --cpp_out=${CMAKE_BINARY_DIR}/src -I
+        ${PROTOBUF_INCLUDE_DIRS} -I ${CMAKE_SOURCE_DIR}/src ${ABS_FIL}
      DEPENDS ${ABS_FIL}
      COMMENT "Running C++ protocol buffer compiler on ${FIL}"
      VERBATIM)
  endforeach()

-  set_source_files_properties(${${_ARGS_CXX_SOURCES}} ${${_ARGS_CXX_HEADERS}} PROPERTIES GENERATED TRUE)
+  set_source_files_properties(${${SRCS}} ${${HDRS}} PROPERTIES GENERATED TRUE)

  # remove some warnings from protobuf generated files
  if(WIN32)
-    set_source_files_properties(${${_ARGS_CXX_SOURCES}} PROPERTIES COMPILE_FLAGS
+    set_source_files_properties(${${SRCS}} PROPERTIES COMPILE_FLAGS
                                                      "/wd4309 /wd4838")
  else()
    set_source_files_properties(
-      ${${_ARGS_CXX_SOURCES}}
+      ${${SRCS}}
      PROPERTIES
        COMPILE_FLAGS
        "-Wno-unused-parameter -Wno-effc++ -Wno-pedantic -Wno-sign-compare -Wno-uninitialized"
    )
  endif()

-  set(${_ARGS_CXX_SOURCES}
-      ${${_ARGS_CXX_SOURCES}}
+  set(${SRCS}
+      ${${SRCS}}
      PARENT_SCOPE)
-  set(${_ARGS_CXX_HEADERS}
-      ${${_ARGS_CXX_HEADERS}}
+  set(${HDRS}
+      ${${HDRS}}
      PARENT_SCOPE)
 endfunction()

-function(Service_Compiler_Generate_Cpp)
-  set(_options)
-  set(_one_value_args CXX_SOURCES CXX_HEADERS CXX_OUT_DIR)
-  set(_multi_value_args PROTO_INCLUDE_DIRS PROTO_SOURCES)
-  cmake_parse_arguments(_ARGS "${_options}" "${_one_value_args}" "${_multi_value_args}" ${ARGN})
-  set(${_ARGS_CXX_SOURCES})
-  set(${_ARGS_CXX_HEADERS})
+function(Service_Compiler_Generate_Cpp SRCS HDRS)
+  set(${SRCS})
+  set(${HDRS})

-  set(_proto_include_cli_args)
-  foreach(_dir ${_ARGS_PROTO_INCLUDE_DIRS})
-    list(APPEND _proto_include_cli_args -I ${_dir})
-  endforeach()
-
-  foreach(FIL ${_ARGS_PROTO_SOURCES})
+  foreach(FIL ${ARGN})
    get_filename_component(ABS_FIL ${FIL} ABSOLUTE)
    get_filename_component(FIL_WE ${FIL} NAME_WE)
    get_filename_component(FIL_DIR ${FIL} DIRECTORY)
@@ -82,58 +63,58 @@ function(Service_Compiler_Generate_Cpp)
      set(FIL_WE "${FIL_DIR}/${FIL_WE}")
    endif()

-    list(APPEND ${_ARGS_CXX_SOURCES} "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.pb.cc")
-    list(APPEND ${_ARGS_CXX_HEADERS} "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.pb.h")
+    list(APPEND ${SRCS} "${CMAKE_CURRENT_BINARY_DIR}/${FIL_WE}.pb.cc")
+    list(APPEND ${HDRS} "${CMAKE_CURRENT_BINARY_DIR}/${FIL_WE}.pb.h")

-    list(APPEND ${_ARGS_CXX_SOURCES} "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.client.pb.cc")
-    list(APPEND ${_ARGS_CXX_HEADERS} "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.client.pb.h")
+    list(APPEND ${SRCS} "${CMAKE_CURRENT_BINARY_DIR}/${FIL_WE}.client.pb.cc")
+    list(APPEND ${HDRS} "${CMAKE_CURRENT_BINARY_DIR}/${FIL_WE}.client.pb.h")

-    list(APPEND ${_ARGS_CXX_SOURCES} "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.server.pb.cc")
-    list(APPEND ${_ARGS_CXX_HEADERS} "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.server.pb.h")
+    list(APPEND ${SRCS} "${CMAKE_CURRENT_BINARY_DIR}/${FIL_WE}.server.pb.cc")
+    list(APPEND ${HDRS} "${CMAKE_CURRENT_BINARY_DIR}/${FIL_WE}.server.pb.h")

    set(TEMPLATES
-        ${CMAKE_SOURCE_DIR}/src/lib_spark/spark/api/service_compiler/templates/cpp/client.pb.cc
-        ${CMAKE_SOURCE_DIR}/src/lib_spark/spark/api/service_compiler/templates/cpp/client.pb.h
-        ${CMAKE_SOURCE_DIR}/src/lib_spark/spark/api/service_compiler/templates/cpp/server.pb.cc
-        ${CMAKE_SOURCE_DIR}/src/lib_spark/spark/api/service_compiler/templates/cpp/server.pb.h
+        ${CMAKE_CURRENT_SOURCE_DIR}/api/service_compiler/templates/cpp/client.pb.cc
+        ${CMAKE_CURRENT_SOURCE_DIR}/api/service_compiler/templates/cpp/client.pb.h
+        ${CMAKE_CURRENT_SOURCE_DIR}/api/service_compiler/templates/cpp/server.pb.cc
+        ${CMAKE_CURRENT_SOURCE_DIR}/api/service_compiler/templates/cpp/server.pb.h
    )

-    set(_plugin_py ${CMAKE_SOURCE_DIR}/src/lib_spark/spark/api/service_compiler/plugin.py)
    if(WIN32)
-      set(_plugin_cmd ${CMAKE_SOURCE_DIR}/src/lib_spark/spark/api/service_compiler/plugin.cmd)
      add_custom_command(
-        OUTPUT "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.pb.cc"
-               "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.pb.h"
-               "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.client.pb.cc"
-               "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.client.pb.h"
-               "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.server.pb.cc"
-               "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.server.pb.h"
+        OUTPUT "${CMAKE_CURRENT_BINARY_DIR}/${FIL_WE}.pb.cc"
+               "${CMAKE_CURRENT_BINARY_DIR}/${FIL_WE}.pb.h"
+               "${CMAKE_CURRENT_BINARY_DIR}/${FIL_WE}.client.pb.cc"
+               "${CMAKE_CURRENT_BINARY_DIR}/${FIL_WE}.client.pb.h"
+               "${CMAKE_CURRENT_BINARY_DIR}/${FIL_WE}.server.pb.cc"
+               "${CMAKE_CURRENT_BINARY_DIR}/${FIL_WE}.server.pb.h"
        COMMAND SET PYTHONPATH=${SPARK_PYTHON}
        COMMAND ${CMAKE_SOURCE_DIR}/.venv/Scripts/activate.bat
        COMMAND
-          ${Protobuf_PROTOC_EXECUTABLE} ARGS --cpp_out=${_ARGS_CXX_OUT_DIR}
-          --plugin=protoc-gen-service=${_plugin_cmd}
-          --service_out=lang=cpp:${_ARGS_CXX_OUT_DIR} -I
-          ${PROTOBUF_INCLUDE_DIRS} -I ${_protobuf_src_dir} ${ABS_FIL}
-        DEPENDS ${ABS_FIL} ${TEMPLATES} ${_plugin_cmd} ${_plugin_py}
+          ${Protobuf_PROTOC_EXECUTABLE} ARGS --cpp_out=${CMAKE_BINARY_DIR}/src
+          --plugin=protoc-gen-service=${CMAKE_SOURCE_DIR}/src/lib/api/service_compiler/plugin.cmd
+          --service_out=lang=cpp:${CMAKE_BINARY_DIR}/src -I
+          ${PROTOBUF_INCLUDE_DIRS} -I ${CMAKE_SOURCE_DIR}/src ${ABS_FIL}
+        DEPENDS ${ABS_FIL} ${TEMPLATES}
+                ${CMAKE_CURRENT_SOURCE_DIR}/api/service_compiler/plugin.py
        COMMENT "Running C++ service compiler on ${FIL}"
        VERBATIM)
    else()
      add_custom_command(
-        OUTPUT "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.pb.cc"
-               "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.pb.h"
-               "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.client.pb.cc"
-               "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.client.pb.h"
-               "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.server.pb.cc"
-               "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.server.pb.h"
+        OUTPUT "${CMAKE_CURRENT_BINARY_DIR}/${FIL_WE}.pb.cc"
+               "${CMAKE_CURRENT_BINARY_DIR}/${FIL_WE}.pb.h"
+               "${CMAKE_CURRENT_BINARY_DIR}/${FIL_WE}.client.pb.cc"
+               "${CMAKE_CURRENT_BINARY_DIR}/${FIL_WE}.client.pb.h"
+               "${CMAKE_CURRENT_BINARY_DIR}/${FIL_WE}.server.pb.cc"
+               "${CMAKE_CURRENT_BINARY_DIR}/${FIL_WE}.server.pb.h"
        COMMAND ${CMAKE_SOURCE_DIR}/tools/venv_exec.sh
        ARGS
          ${Protobuf_PROTOC_EXECUTABLE}
-          --cpp_out=${_ARGS_CXX_OUT_DIR}
-          --plugin=protoc-gen-service=${_plugin_py}
-          --service_out=lang=cpp,make_file_name=${ABS_FIL}:${_ARGS_CXX_OUT_DIR}
-          ${_proto_include_cli_args} ${ABS_FIL}
-        DEPENDS ${ABS_FIL} ${TEMPLATES} ${_plugin_py}
+          --cpp_out=${CMAKE_BINARY_DIR}/src
+          --plugin=protoc-gen-service=${CMAKE_SOURCE_DIR}/src/lib/api/service_compiler/plugin.py
+          --service_out=lang=cpp,make_file_name=${ABS_FIL}:${CMAKE_BINARY_DIR}/src
+          -I ${PROTOBUF_INCLUDE_DIRS} -I ${CMAKE_SOURCE_DIR}/src ${ABS_FIL}
+        DEPENDS ${ABS_FIL} ${TEMPLATES}
+                ${CMAKE_CURRENT_SOURCE_DIR}/api/service_compiler/plugin.py
        COMMENT "Running C++ service compiler on ${FIL}"
        VERBATIM)
    endif()
@@ -142,22 +123,22 @@ function(Service_Compiler_Generate_Cpp)

  # remove some warnings from protobuf generated files
  if(WIN32)
-    set_source_files_properties(${${_ARGS_CXX_SOURCES}} PROPERTIES COMPILE_FLAGS
+    set_source_files_properties(${${SRCS}} PROPERTIES COMPILE_FLAGS
                                                      "/wd4309 /wd4838")
  else()
    set_source_files_properties(
-      ${${_ARGS_CXX_SOURCES}}
+      ${${SRCS}}
      PROPERTIES
        COMPILE_FLAGS
        "-Wno-unused-parameter -Wno-effc++ -Wno-pedantic -Wno-sign-compare -Wno-uninitialized"
    )
  endif()

-  set(${_ARGS_CXX_SOURCES}
-      ${${_ARGS_CXX_SOURCES}}
+  set(${SRCS}
+      ${${SRCS}}
      PARENT_SCOPE)
-  set(${_ARGS_CXX_HEADERS}
-      ${${_ARGS_CXX_HEADERS}}
+  set(${HDRS}
+      ${${HDRS}}
      PARENT_SCOPE)

 endfunction()
@@ -1,83 +0,0 @@
-include_guard(GLOBAL)
-
-function(protobuf_generate_cpp)
-  set(_options)
-  set(_one_value_args
-    CXX_SOURCES
-    CXX_HEADERS
-    CXX_OUT_DIR
-    PROTOC
-  )
-  set(_multi_value_args
-    PROTO_INCLUDE_DIRS
-    PROTO_SOURCES
-  )
-  cmake_parse_arguments(_ARGS "${_options}" "${_one_value_args}" "${_multi_value_args}" ${ARGN})
-
-  if (NOT DEFINED _ARGS_PROTOC)
-    if (WIN32)
-      set(_ARGS_PROTOC "${EXTERNALS_BIN}/protoc.exe")
-    else()
-      set(_ARGS_PROTOC "${EXTERNALS_BIN}/protoc")
-    endif()
-  endif()
-  set(${_ARGS_CXX_SOURCES})
-  set(${_ARGS_CXX_HEADERS})
-
-  set(_proto_include_cli_args)
-  foreach(_dir ${_ARGS_PROTO_INCLUDE_DIRS})
-    list(APPEND _proto_include_cli_args -I ${_dir})
-  endforeach()
-
-  foreach(_fil ${_ARGS_PROTO_SOURCES})
-    get_filename_component(_abs_fil ${_fil} ABSOLUTE)
-    get_filename_component(_fil_we ${_fil} NAME_WE)
-    get_filename_component(_fil_dir ${_fil} DIRECTORY)
-    if(_fil_dir)
-      set(_fil_we "${_fil_dir}/${_fil_we}")
-    endif()
-    list(APPEND ${_ARGS_CXX_SOURCES} "${_ARGS_CXX_OUT_DIR}/${_fil_we}.pb.cc")
-    list(APPEND ${_ARGS_CXX_HEADERS} "${_ARGS_CXX_OUT_DIR}/${_fil_we}.pb.h")
-    message(STATUS "add_custom_command for ${_fil} -> ${_ARGS_CXX_OUT_DIR}/${_fil_we}.pb.h")
-    # TODO: Pass proto sources by relative path (so this fn can be called from
-    # arbitrary subdirs).
-    add_custom_command(
-      OUTPUT
-        "${_ARGS_CXX_OUT_DIR}/${_fil_we}.pb.cc"
-        "${_ARGS_CXX_OUT_DIR}/${_fil_we}.pb.h"
-      COMMAND
-        ${_ARGS_PROTOC}
-      ARGS
-        --cpp_out=${_ARGS_CXX_OUT_DIR} ${_proto_include_cli_args} ${_abs_fil}
-      DEPENDS ${_abs_fil}
-      COMMENT "Running C++ protocol buffer compiler on ${_fil}"
-      VERBATIM
-    )
-  endforeach()
-
-  set_source_files_properties(
-    ${${_ARGS_CXX_SOURCES}}
-    ${${_ARGS_CXX_HEADERS}}
-    PROPERTIES
-      GENERATED TRUE
-  )
-
-  # remove some warnings from protobuf generated files
-  if(WIN32)
-    set_source_files_properties(
-      ${${_ARGS_CXX_SOURCES}}
-      PROPERTIES
-        COMPILE_FLAGS "/wd4309 /wd4838"
-    )
-  else()
-    set_source_files_properties(
-      ${${_ARGS_CXX_SOURCES}}
-      PROPERTIES
-        COMPILE_FLAGS "-Wno-unused-parameter -Wno-effc++ -Wno-pedantic -Wno-sign-compare -Wno-uninitialized"
-    )
-  endif()
-
-  set(${_ARGS_CXX_SOURCES} ${${_ARGS_CXX_SOURCES}} PARENT_SCOPE)
-  set(${_ARGS_CXX_HEADERS} ${${_ARGS_CXX_HEADERS}} PARENT_SCOPE)
-endfunction()
-
@@ -1,61 +0,0 @@
-include_guard(GLOBAL)
-
-function(protobuf_generate_cpp_config)
-  set(_options)
-  set(_one_value_args
-    PROTO_SOURCES_HEADER
-    PROTO_DEFAULTS_HEADER
-  )
-  set(_multi_value_args
-    PROTO_SOURCES
-    PROTO_DEFAULTS
-  )
-  cmake_parse_arguments(_ARGS "${_options}" "${_one_value_args}" "${_multi_value_args}" ${ARGN})
-
-  # Configure the 'config_proto_files' header. It provides an array of strings populated with the
-  # files in PROTO_SOURCES. At runtime, we use this array to generate a special 'config' proto
-  # message with fields for each message defined in said files. A global instance of this 'config'
-  # proto is then constructed, and populated with [compile-time] defaults, and [runtime] overrides
-  # (i.e., in spark.conf). Calls to `bts::config::get<T>()` retrieve the value from the field
-  # associated with type `T` in this global instance.
-  message(STATUS "PROTO_SOURCES: ${_ARGS_PROTO_SOURCES}")
-  message(STATUS "PROTO_SOURCES_HEADER: ${_ARGS_PROTO_SOURCES_HEADER}")
-  message(STATUS "PROTO_DEFAULTS_HEADER: ${_ARGS_PROTO_DEFAULTS_HEADER}")
-  if (_ARGS_PROTO_SOURCES_HEADER)
-    set(PROTO_SOURCES)
-    set(PROTO_SOURCES_CXX_ARRAY_ITEMS "")
-    foreach(_proto_path ${_ARGS_PROTO_SOURCES})
-      list(APPEND PROTO_SOURCES ${_proto_path})
-      string(APPEND PROTO_SOURCES_CXX_ARRAY_ITEMS "\"${_proto_path}\",\n")
-    endforeach()
-    list(LENGTH PROTO_SOURCES PROTO_SOURCES_LENGTH)
-    message(STATUS "CONFIGURING PROTO_SOURCES_HEADER): ${_ARGS_PROTO_SOURCES_HEADER}")
-    configure_file(${CMAKE_SOURCE_DIR}/cmake/config_proto_files.h.in ${_ARGS_PROTO_SOURCES_HEADER})
-  endif()
-
-  # Generate static defaults. Ideally, this would be a single string, but we use an array because
-  # the compiler complains if the string is longer than 65k.
-  set(_cfg_defaults_str "inline const char * config_static_defaults[] = { ")
-
-  foreach(_cfg_file_path ${_ARGS_PROTO_DEFAULTS})
-    file(READ ${_cfg_file_path} _cfg_file_content)
-    string(REPLACE "\"" "\\\"" _cfg_file_content_escaped ${_cfg_file_content})
-    string(REPLACE "\n" ";" _cfg_lines ${_cfg_file_content_escaped})
-
-    foreach(_line ${_cfg_lines})
-      string(APPEND _cfg_defaults_str "\n  \"${_line}\",")
-    endforeach(_line)
-  endforeach(_cfg_file_path)
-
-  string(APPEND _cfg_defaults_str "};")
-  set(_old_cfg_defaults_str "")
-
-  if(EXISTS ${_ARGS_PROTO_DEFAULTS_HEADER})
-    file(READ ${_ARGS_PROTO_DEFAULTS_HEADER} _old_cfg_defaults_str)
-  endif()
-
-  if(NOT _cfg_defaults_str STREQUAL _old_cfg_defaults_str)
-    file(WRITE "${_ARGS_PROTO_DEFAULTS_HEADER}" "${_cfg_defaults_str}")
-  endif()
-endfunction()
-
@@ -10,7 +10,7 @@ function(protobuf_generate_fs)
    set(_ARGS_TARGET_NAME "${PROJECT_NAME}_generate_fs_protos")
  endif()
  if (NOT DEFINED _ARGS_PROTOC)
-    set(_ARGS_PROTOC "${EXTERNALS_BIN}/protoc")
+    set(_ARGS_PROTOC "${SPARK_BIN}/protoc")
  endif()

  set(_proto_include_cli_args)
@@ -27,8 +27,8 @@ function(protobuf_generate_fs)
  endforeach()

  set(_venv_exec ${CMAKE_SOURCE_DIR}/tools/venv_exec.sh)
-  set(_plugin_py ${CMAKE_SOURCE_DIR}/src/lib_spark/spark/api/service_compiler/plugin.py)
-  set(_template_fs ${CMAKE_SOURCE_DIR}/src/lib_spark/spark/api/service_compiler/templates/fs/fable.fs)
+  set(_plugin_py ${CMAKE_SOURCE_DIR}/src/lib/api/service_compiler/plugin.py)
+  set(_template_fs ${CMAKE_SOURCE_DIR}/src/lib/api/service_compiler/templates/fs/fable.fs)
  add_custom_command(
    OUTPUT ${_outputs}
    WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}
@@ -1,10 +0,0 @@
-include_guard(GLOBAL)
-
-macro(set_first_defined RESULT)
-  foreach(_arg ${ARGN})
-    if (DEFINED ${_arg})
-      set(${RESULT} ${${_arg}})
-      break()
-    endif()
-  endforeach()
-endmacro()
@@ -1,39 +0,0 @@
-include_guard(DIRECTORY)
-include(${CMAKE_SOURCE_DIR}/cmake/SymLink.cmake)
-
-function(sym_link_python_module)
-  set(_options)
-  set(_one_value_args LIB_NAME MODULE_NAME PLUGIN_NAME)
-  set(_multi_value_args)
-  cmake_parse_arguments(_ARGS "${_options}" "${_one_value_args}"
-                        "${_multi_value_args}" ${ARGN})
-  if(NOT DEFINED _ARGS_LIB_NAME)
-    message(
-      FATAL_ERROR "LIB_NAME must be specified for sym_link_python_module()")
-  endif()
-  set(_lib_name ${_ARGS_LIB_NAME})
-  set(_module_name ${_ARGS_MODULE_NAME})
-  if(NOT DEFINED _ARGS_MODULE_NAME)
-    set(_module_name ${_lib_name}_module)
-  endif()
-  set(_plugin_name ${_ARGS_PLUGIN_NAME})
-  set(_deps ${_lib_name} ${_module_name})
-  if(DEFINED _ARGS_PLUGIN_NAME)
-    list(APPEND _deps ${_plugin_name} copy_${_plugin_name})
-  endif()
-  add_dependencies(${PROJECT_NAME} ${_deps})
-  sym_link_local_lib(${_lib_name})
-  set(_module_suffix cpython-310-x86_64-linux-gnu.so)
-  sym_link_file(
-    SOURCE
-    ${CMAKE_BINARY_DIR}/src/lib_${_lib_name}/${_module_name}.${_module_suffix}
-    DEST
-    ${CMAKE_CURRENT_BINARY_DIR}/${_module_name}.${_module_suffix}
-    INSTALL_DEST
-    ${CMAKE_INSTALL_BINDIR}/)
-  if(DEFINED _ARGS_PLUGIN_NAME)
-    get_target_property(plugins ${_plugin_name} INTERFACE_PYTHON_SCRIPTS_DIR)
-    sym_link_dir(TARGET_NAME ${PROJECT_NAME}_link_${_plugin_name} SOURCE
-                 ${plugins} DEST ${CMAKE_CURRENT_BINARY_DIR}/${_plugin_name})
-  endif()
-endfunction()
@@ -1,11 +0,0 @@
-#pragma once
-
-#include <array>
-
-namespace bts::@PROJECT_NAME@::protobufs {
-
-inline std::array<const char *, @PROTO_SOURCES_LENGTH@> config_proto_files = {
-@PROTO_SOURCES_CXX_ARRAY_ITEMS@
-};
-
-} // namespace bts::@PROJECT_NAME@::protobufs
@@ -1,11 +0,0 @@
-#ifndef BTS_SPARK_VERSION_H
-#define BTS_SPARK_VERSION_H
-
-#define BTS_SPARK_VERSION_MAJOR @PROJECT_VERSION_MAJOR@
-#define BTS_SPARK_VERSION_MINOR @PROJECT_VERSION_MINOR@
-#define BTS_SPARK_VERSION_PATCH @PROJECT_VERSION_PATCH@
-#define BTS_SPARK_VERSION_BUILD_METADATA @PROJECT_VERSION_BUILD_METADATA@
-#define BTS_SPARK_VERSION "@PROJECT_VERSION@"
-#define BTS_SPARK_VERSION_FULL "@PROJECT_VERSION_FULL@"
-
-#endif
@@ -0,0 +1,24 @@
+cmake_minimum_required(VERSION 3.5)
+
+# Explicitly set policy to convert relative paths to absolute (although this is a default policy,
+# setting it explicitly prevents a compile-warning)
+cmake_policy(SET CMP0076 NEW)
+
+# add protos to the library to facilitate source_group
+target_sources(libspark PUBLIC ${Config_Protos} ${NonConfig_Protos} ${Service_Protos})
+
+function(assign_source_group)
+  foreach(_source IN ITEMS ${ARGN})
+    if(IS_ABSOLUTE "${_source}")
+      file(RELATIVE_PATH _source_rel "${CMAKE_CURRENT_SOURCE_DIR}" "${_source}")
+    else()
+      set(_source_rel "${_source}")
+    endif()
+    get_filename_component(_source_path "${_source_rel}" PATH)
+    string(REPLACE "/" "\\" _source_path_msvc "${_source_path}")
+    source_group("${_source_path_msvc}" FILES "${_source}")
+    set(SOURCE_FILES "${_source}")
+  endforeach()
+endfunction(assign_source_group)
+
+assign_source_group(${SparkLib_Sources} ${Config_Protos} ${NonConfig_Protos} ${Service_Protos})
@@ -1,106 +0,0 @@
-services:
-  sparkd:
-    volumes:
-      - ${SPARKD__DATA_DIR:-sparkd_data}:/home/bts/sparkdata/data
-    build:
-      dockerfile: src/app_sparkd/Dockerfile
-      args:
-        - IMAGE=${SPARKD__IMAGE}
-        - SOURCE_DIR=${SPARKD__SOURCE_DIR}
-        - CONFIG_DIR=${SPARKD__CONFIG_DIR}
-        - PYTHON_REQUIREMENTS=${SPARKD__PYTHON_REQUIREMENTS}
-    shm_size: "${SPARKD__SHM_SIZE:-512mb}"
-    command: >
-      sh -c "
-        echo bts | sudo -S chown -R bts:bts /home/bts/sparkdata/data
-        if [ -e "/home/bts/sparkdata/data/logging.sparkd/log" ]; then
-          mv "/home/bts/sparkdata/data/logging.sparkd/log" "/home/bts/sparkdata/data/logging.sparkd/log_$(date '+%Y%m%d-%H%M')"
-        fi
-        if [ -e "/home/bts/sparkdata/conf/.env.local" ]; then
-          echo 'Sourcing /home/bts/sparkdata/conf/.env.local'
-          set -a
-          . /home/bts/sparkdata/conf/.env.local
-          set +a
-        fi
-        ./sparkd run"
-    env_file:
-      - .env
-      - ${SPARKD__CONFIG_DIR}/.env
-    ports:
-      - "${SPARKD__CONFIG_PORT}:${SPARKD__CONFIG_PORT}"
-      - "${SPARKD__QUICKBUS_PORT}:${SPARKD__QUICKBUS_PORT}"
-      - "${SPARKD__SIM_FIX_PORT}:${SPARKD__SIM_FIX_PORT}"
-    deploy:
-      resources:
-        limits:
-          memory: "${SPARKD__MEMORY:-4gb}"
-
-  sparkd-symbols:
-    volumes:
-      - ${SPARKD__DATA_DIR:-sparkd_data}:/home/bts/sparkdata/data
-      - ${SPARKD__RUNTIME_CONF_DIR:-sparkd_conf}:/home/bts/sparkdata/conf
-    build:
-      dockerfile: src/app_sparkd/Dockerfile
-      args:
-        - IMAGE=${SPARKD__IMAGE}
-        - SOURCE_DIR=${SPARKD__SOURCE_DIR}
-        - CONFIG_DIR=${SPARKD__CONFIG_DIR}
-        - PYTHON_REQUIREMENTS=${SPARKD__PYTHON_REQUIREMENTS}
-    shm_size: "${SPARKD__SHM_SIZE:-512mb}"
-    command: >
-      sh -c "
-        echo bts | sudo -S chown -R bts:bts /home/bts/sparkdata
-        if [ -e "/home/bts/sparkdata/data/logging.sparkd/log" ]; then
-          mv "/home/bts/sparkdata/data/logging.sparkd/log" "/home/bts/sparkdata/data/logging.sparkd/log_$(date '+%Y%m%d-%H%M')"
-        fi
-        ls -alh /home/bts/sparkdata/conf/
-        cat /home/bts/sparkdata/conf/.env.local
-        if [ -e "/home/bts/sparkdata/conf/.env.local" ]; then
-          echo 'Sourcing /home/bts/sparkdata/conf/.env.local'
-          set -a
-          . /home/bts/sparkdata/conf/.env.local
-          set +a
-        fi
-        ./sparkd run"
-    env_file:
-      - .env
-      - ${SPARKD__CONFIG_DIR}/.env
-    ports:
-      - "${SPARKD__CONFIG_PORT}:${SPARKD__CONFIG_PORT}"
-      - "${SPARKD__QUICKBUS_PORT}:${SPARKD__QUICKBUS_PORT}"
-      - "${SPARKD__SIM_FIX_PORT}:${SPARKD__SIM_FIX_PORT}"
-    deploy:
-      restart_policy:
-        condition: any
-        delay: 3s
-      resources:
-        limits:
-          memory: "${SPARKD__MEMORY:-4gb}"
-
-  sparkd-tail:
-    image: bluetradingsystems/linux_dotnet7
-    volumes:
-      - type: volume
-        read_only: true
-        source: sparkd_data
-        target: /home/bts/sparkdata/data/logging.sparkd
-        volume:
-          subpath: logging.sparkd
-    command: ["tail", "-f", "/home/bts/sparkdata/data/logging.sparkd/log"]
-
-  sparkd-ping-test:
-    depends_on:
-      - sparkd
-    profiles:
-      - ping-test
-    build:
-      context: .
-      dockerfile_inline: |
-        FROM bluetradingsystems/linux_dotnet7
-        USER "root"
-        RUN apt-get update && apt-get -y install iputils-ping
-    command: [ "ping", "sparkd" ]
-
-volumes:
-  sparkd_data:
-    name: "sparkd_${SPARKD__CONFIG_PORT:-8000}"
@@ -124,8 +124,8 @@ service OrderService {
 ```
 ### C++ Example
 ```cpp
-#include "spark/api/public/ServiceContext.h"
-#include "spark/api/services/OrderService.client.pb.h"
+#include "lib/api/public/ServiceContext.h"
+#include "lib/api/services/OrderService.client.pb.h"
 using namespace bts::api;
 ServiceContext ctx;
 auto orders = ctx.locate<orderman::OrderService>();
@@ -29,9 +29,9 @@ namespace sample {
 ```

 ```cpp
-#include "spark/api/public/ServiceContext.h"
-#include "spark/domain_model/Security.h"
-#include "spark/api/services/MarketDataService.client.pb.h"
+#include "lib/api/public/ServiceContext.h"
+#include "spark/lib/domain_model/Security.h"
+#include "lib/api/services/MarketDataService.client.pb.h"
 #include <thread>

 using namespace bts::api;
@@ -1,38 +0,0 @@
-FROM ubuntu:22.04
-
-ARG DEBIAN_FRONTEND=noninteractive
-
-RUN apt-get update && apt-get install -y pkg-config
-RUN apt-get update && apt-get install -y locales
-RUN apt-get update && apt-get install -y ca-certificates
-RUN apt-get update && apt-get install -y libssh2-1-dev
-RUN apt-get update && apt-get install -y software-properties-common
-RUN apt-get update && apt-get install -y sudo
-
-# GCC
-ARG gcc_major=13
-RUN add-apt-repository ppa:ubuntu-toolchain-r/test \
-  && apt-get update \
-  && apt-get install -y g++-$gcc_major
-
-RUN apt-get update \
-  && apt-get install -y dotnet7
-
-RUN locale-gen en_US.UTF-8
-
-ARG docker_user="bts"
-ARG docker_uid="1000"
-ARG docker_pass="bts"
-
-RUN \
-  groupadd -g "$docker_uid" "$docker_user" \
-  && useradd \
-  -m \
-  -u "$docker_uid" \
-  -g "$docker_uid" \
-  -G sudo \
-  --password "$(openssl passwd -1 "$docker_pass")" \
-  "$docker_user"
-
-USER bts
-WORKDIR /home/bts
@@ -1,42 +1,22 @@
 astroid==2.7.3
 autopep8==1.5.7
-blinker==1.9.0
-boto3==1.37.9
-botocore==1.37.9
 certifi==2023.5.7
 charset-normalizer==3.1.0
-click==8.1.8
 colorama==0.4.4
-Flask==3.1.0
 idna==3.4
 isort==5.9.3
-itsdangerous==2.2.0
-Jinja2==3.1.6
-jmespath==1.0.1
 lazy-object-proxy==1.6.0
-MarkupSafe==3.0.2
 mccabe==0.6.1
-numpy==2.2.6
 platformdirs==2.3.0
 protobuf==4.21.12
 pybars3==0.9.7
-pybind11==2.13.6
-pybind11-stubgen==2.5.3
-pybit==5.11.0
 pycodestyle==2.7.0
 pycparser==2.20
-pycryptodome==3.23.0
 pylint==2.10.2
 PyMeta3==0.5.1
-python-dateutil==2.9.0.post0
-python-dotenv==1.0.1
 requests==2.31.0
-s3transfer==0.11.4
-six==1.17.0
 termcolor==1.1.0
 toml==0.10.2
 urllib3==2.0.3
-websocket-client==1.8.0
-Werkzeug==3.1.3
 wrapt==1.12.1
 yapf==0.31.0
@@ -1,7 +1,7 @@
 <Project Sdk="Microsoft.NET.Sdk">

  <PropertyGroup>
-    <TargetFramework>net8.0</TargetFramework>
+    <TargetFramework>net7.0</TargetFramework>
    <ImplicitUsings>enable</ImplicitUsings>
    <Nullable>enable</Nullable>
  </PropertyGroup>
@@ -1,7 +1,7 @@
 <Project Sdk="Microsoft.NET.Sdk">

  <PropertyGroup>
-    <TargetFramework>net8.0</TargetFramework>
+    <TargetFramework>net7.0</TargetFramework>
    <ImplicitUsings>enable</ImplicitUsings>
    <Nullable>enable</Nullable>
  </PropertyGroup>
@@ -1,7 +1,7 @@
 <Project Sdk="Microsoft.NET.Sdk">

  <PropertyGroup>
-    <TargetFramework>net8.0</TargetFramework>
+    <TargetFramework>net7.0</TargetFramework>
    <ImplicitUsings>enable</ImplicitUsings>
    <Nullable>enable</Nullable>
  </PropertyGroup>
@@ -1,14 +1,14 @@
 <Project Sdk="Microsoft.NET.Sdk">

  <PropertyGroup>
-    <TargetFramework>net8.0</TargetFramework>
+    <TargetFramework>net7.0</TargetFramework>
    <ImplicitUsings>enable</ImplicitUsings>
    <Nullable>enable</Nullable>
  </PropertyGroup>

  <ItemGroup>
    <PackageReference Include="Google.Protobuf" Version="3.24.3" />
-    <PackageReference Include="YaccLexTools" Version="1.2.3" />
+    <PackageReference Include="YaccLexTools" Version="1.2.0" />
  </ItemGroup>

  <ItemGroup>
@@ -24,6 +24,8 @@ namespace QUT.GplexBuffers
        public BufferException(string message) : base(message) { }
        public BufferException(string message, Exception innerException)
            : base(message, innerException) { }
+        protected BufferException(SerializationInfo info, StreamingContext context)
+            : base(info, context) { }
    }

    internal abstract class ScanBuff
@@ -58,10 +60,12 @@ namespace QUT.GplexBuffers
            return new BuildBuffer(source);
        }

+#if (!BYTEMODE)
        public static ScanBuff GetBuffer(Stream source, int fallbackCodePage)
        {
            return new BuildBuffer(source, fallbackCodePage);
        }
+#endif // !BYTEMODE
 #endif // !NOFILES
    }

@@ -349,15 +353,17 @@ namespace QUT.GplexBuffers
        {
            FileStream fStrm = (stream as FileStream);
            if (fStrm != null) FileName = fStrm.Name;
-            NextBlk = BlockReaderFactory.Get(stream);
+            NextBlk = BlockReaderFactory.Raw(stream);
        }

+#if (!BYTEMODE)
        public BuildBuffer(Stream stream, int fallbackCodePage)
        {
            FileStream fStrm = (stream as FileStream);
            if (fStrm != null) FileName = fStrm.Name;
            NextBlk = BlockReaderFactory.Get(stream, fallbackCodePage);
        }
+#endif

        /// <summary>
        /// Marks a conservative lower bound for the buffer,
@@ -428,19 +434,43 @@ namespace QUT.GplexBuffers
    //
    internal static class BlockReaderFactory
    {
+        public static BlockReader Raw(Stream stream)
+        {
+            return delegate(char[] block, int index, int number)
+            {
+                byte[] b = new byte[number];
+                int count = stream.Read(b, 0, number);
+                int i = 0;
+                int j = index;
+                for (; i < count; i++, j++)
+                    block[j] = (char)b[i];
+                return count;
+            };
+        }

-        public static BlockReader Get(Stream stream, int fallbackCodePage = -2)
+#if (!BYTEMODE)
+        public static BlockReader Get(Stream stream, int fallbackCodePage)
        {
            Encoding encoding;
            int preamble = Preamble(stream);

            if (preamble != 0)  // There is a valid BOM here!
                encoding = Encoding.GetEncoding(preamble);
-            else if (fallbackCodePage >= 0) // Anything but "guess"
+            else if (fallbackCodePage == -1) // Fallback is "raw" bytes
+                return Raw(stream);
+            else if (fallbackCodePage != -2) // Anything but "guess"
                encoding = Encoding.GetEncoding(fallbackCodePage);
            else // This is the "guess" option
-                encoding = GuessEncoding(stream);
-
+            {
+                int guess = new Guesser(stream).GuessCodePage();
+                stream.Seek(0, SeekOrigin.Begin);
+                if (guess == -1) // ==> this is a 7-bit file
+                    encoding = Encoding.ASCII;
+                else if (guess == 65001)
+                    encoding = Encoding.UTF8;
+                else             // ==> use the machine default
+                    encoding = Encoding.Default;
+            }
            StreamReader reader = new StreamReader(stream, encoding);
            return reader.Read;
        }
@@ -458,14 +488,6 @@ namespace QUT.GplexBuffers
            int b2 = stream.ReadByte();
            if (b0 == 0xef && b1 == 0xbb && b2 == 0xbf)
                return 65001; // UTF8
-
-            int b3 = stream.ReadByte();
-            // Check for UTF-32 BOM (Byte Order Mark)
-            if (b0 == 0x00 && b1 == 0x00 && b2 == 0xFE && b3 == 0xFF)
-                return 12001; // UTF-32BE
-            if (b0 == 0xFF && b1 == 0xFE && b2 == 0x00 && b3 == 0x00)
-                return 12000; // UTF-32LE
-
            //
            // There is no unicode preamble, so we
            // return denoter for the machine default.
@@ -473,150 +495,7 @@ namespace QUT.GplexBuffers
            stream.Seek(0, SeekOrigin.Begin);
            return 0;
        }
-
-        private static Encoding GuessEncoding(Stream stream)
-        {
-            stream.Seek(0, SeekOrigin.Begin);
-            if (IsUtf16(stream, out Encoding encoding))
-            {
-                stream.Seek(0, SeekOrigin.Begin);
-                return encoding;
-            }
-
-            stream.Seek(0, SeekOrigin.Begin);
-            if (IsUtf8(stream))
-            {
-                stream.Seek(0, SeekOrigin.Begin);
-                return Encoding.UTF8;
-            }
-
-            stream.Seek(0, SeekOrigin.Begin);
-            if (IsAscii(stream))
-            {
-                stream.Seek(0, SeekOrigin.Begin);
-                return Encoding.ASCII;
-            }
-
-#if !NET462_OR_GREATER
-            stream.Seek(0, SeekOrigin.Begin);
-            if (IsLatin1(stream))
-            {
-                stream.Seek(0, SeekOrigin.Begin);
-                return Encoding.Latin1;
-            }
-#endif
-            stream.Seek(0, SeekOrigin.Begin);
-            return Encoding.Default;
-        }
-
-        private static bool IsUtf8(Stream stream)
-        {
-            int b;
-            while ((b = stream.ReadByte()) != -1)
-            {
-                if ((b & 0x80) == 0) // 0xxxxxxx
-                {
-                    continue;
-                }
-                else if ((b & 0xE0) == 0xC0) // 110xxxxx 10xxxxxx
-                {
-                    if (!IsValidUtf8ContinuationByte(stream)) return false;
-                }
-                else if ((b & 0xF0) == 0xE0) // 1110xxxx 10xxxxxx 10xxxxxx
-                {
-                    if (!IsValidUtf8ContinuationByte(stream) || !IsValidUtf8ContinuationByte(stream)) return false;
-                }
-                else if ((b & 0xF8) == 0xF0) // 11110xxx 10xxxxxx 10xxxxxx 10xxxxxx
-                {
-                    if (!IsValidUtf8ContinuationByte(stream) || !IsValidUtf8ContinuationByte(stream) || !IsValidUtf8ContinuationByte(stream)) return false;
-                }
-                else
-                {
-                    return false;
-                }
-            }
-            return true;
-        }
-
-        private static bool IsValidUtf8ContinuationByte(Stream stream)
-        {
-            int b = stream.ReadByte();
-            return b != -1 && (b & 0xC0) == 0x80; // 10xxxxxx
-        }
-
-        private static bool IsUtf16(Stream stream, out Encoding encoding)
-        {
-            encoding = Encoding.Default;
-
-            if (stream.Length >= 2)
-            {
-                // Read the first two bytes
-                int b0 = stream.ReadByte();
-                int b1 = stream.ReadByte();
-
-                // Check for UTF-16 BOM (Byte Order Mark)
-                if (b0 == 0xFF && b1 == 0xFE)
-                {
-                    encoding = Encoding.Unicode;
-                    return true; // UTF-16LE
-                }
-                if (b0 == 0xFE && b1 == 0xFF)
-                {
-                    encoding = Encoding.BigEndianUnicode;
-                    return true; // UTF-16BE
-                }
-
-                // Reset the stream position
-                stream.Seek(0, SeekOrigin.Begin);
-
-                // Check for common UTF-16 patterns
-                byte[] buffer = new byte[2];
-                while (stream.Read(buffer, 0, 2) == 2)
-                {
-                    if (buffer[0] == 0x00 && buffer[1] != 0x00)
-                    {
-                        encoding = Encoding.BigEndianUnicode;
-                        return true; // Likely UTF-16BE
-                    }
-                    if (buffer[0] != 0x00 && buffer[1] == 0x00)
-                    {
-                        encoding = Encoding.Unicode;
-                        return true; // Likely UTF-16LE
-                    }
-                }
-            }
-
-            return false;
-        }
-
-        private static bool IsAscii(Stream stream)
-        {
-            int b;
-            while ((b = stream.ReadByte()) != -1)
-            {
-                // Latin1 includes byte values from 0x00 to 0xFF
-                if (b > 0x7F)
-                {
-                    return false;
-                }
-            }
-            return true;
-        }
-
-        private static bool IsLatin1(Stream stream)
-        {
-            int b;
-            while ((b = stream.ReadByte()) != -1)
-            {
-                // Latin1 includes byte values from 0x00 to 0xFF
-                if (b > 0xFF)
-                {
-                    return false;
-                }
-            }
-            return true;
-        }
-
+#endif // !BYTEMODE
    }
 #endif // !NOFILES
    #endregion Buffer classes
@@ -661,6 +540,180 @@ namespace QUT.GplexBuffers
            return 0;
        }
    }
+#region guesser
+#if (!BYTEMODE)
+    // ==============================================================
+    // ============          Encoding Guesser           =============
+    // ==============================================================
+
+    /// <summary>
+    /// This class provides a simple finite state automaton that
+    /// scans the file looking for (1) valid UTF-8 byte patterns,
+    /// (2) bytes >= 0x80 which are not part of a UTF-8 sequence.
+    /// The method then guesses whether it is UTF-8 or maybe some 
+    /// local machine default encoding.  This works well for the
+    /// various Latin encodings.
+    /// </summary>
+    internal class Guesser
+    {
+        ScanBuff buffer;
+
+        public int GuessCodePage() { return Scan(); }
+
+        const int maxAccept = 10;
+        const int initial = 0;
+        const int eofNum = 0;
+        const int goStart = -1;
+        const int INITIAL = 0;
+        const int EndToken = 0;
+
+        #region user code
+        /* 
+         *  Reads the bytes of a file to determine if it is 
+         *  UTF-8 or a single-byte code page file.
+         */
+        public long utfX;
+        public long uppr;
+        #endregion user code
+
+        int state;
+        int currentStart = startState[0];
+        int code;
+
+        #region ScannerTables
+        static int[] startState = new int[] { 11, 0 };
+
+        #region CharacterMap
+        static sbyte[] map = new sbyte[256] {
+/*     '\0' */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 
+/*   '\x10' */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 
+/*   '\x20' */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 
+/*      '0' */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 
+/*      '@' */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 
+/*      'P' */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 
+/*      '`' */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 
+/*      'p' */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 
+/*   '\x80' */ 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 
+/*   '\x90' */ 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 
+/*   '\xA0' */ 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 
+/*   '\xB0' */ 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 
+/*   '\xC0' */ 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 
+/*   '\xD0' */ 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 
+/*   '\xE0' */ 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 
+/*   '\xF0' */ 4, 4, 4, 4, 4, 4, 4, 4, 5, 5, 5, 5, 5, 5, 5, 5 };
+        #endregion
+
+        static sbyte[][] nextState = new sbyte[][] {
+            new sbyte[] {0, 0, 0, 0, 0, 0},
+            new sbyte[] {-1, -1, 10, -1, -1, -1},
+            new sbyte[] {-1, -1, -1, -1, -1, -1},
+            new sbyte[] {-1, -1, 8, -1, -1, -1},
+            new sbyte[] {-1, -1, 5, -1, -1, -1},
+            new sbyte[] {-1, -1, 6, -1, -1, -1},
+            new sbyte[] {-1, -1, 7, -1, -1, -1},
+            null,
+            new sbyte[] {-1, -1, 9, -1, -1, -1},
+            null,
+            null,
+            new sbyte[] {-1, 1, 2, 3, 4, 2}
+        };
+
+
+        [SuppressMessage("Microsoft.Performance", "CA1810:InitializeReferenceTypeStaticFieldsInline")]
+        // Reason for suppression: cannot have self-reference in array initializer.
+        static Guesser()
+        {
+            nextState[7] = nextState[2];
+            nextState[9] = nextState[2];
+            nextState[10] = nextState[2];
+        }
+
+        int NextState()
+        {
+            if (code == ScanBuff.EndOfFile)
+                return eofNum;
+            else
+                return nextState[state][map[code]];
+        }
+        #endregion
+
+        public Guesser(System.IO.Stream file) { SetSource(file); }
+
+        public void SetSource(System.IO.Stream source)
+        {
+            this.buffer = new BuildBuffer(source);
+            code = buffer.Read();
+        }
+
+        int Scan()
+        {
+            for (; ; )
+            {
+                int next;
+                state = currentStart;
+                while ((next = NextState()) == goStart)
+                    code = buffer.Read();
+
+                state = next;
+                code = buffer.Read();
+
+                while ((next = NextState()) > eofNum)
+                {
+                    state = next;
+                    code = buffer.Read();
+                }
+                if (state <= maxAccept)
+                {
+                    #region ActionSwitch
+#pragma warning disable 162
+                    switch (state)
+                    {
+                        case eofNum:
+                            switch (currentStart)
+                            {
+                                case 11:
+                                    if (utfX == 0 && uppr == 0) return -1; /* raw ascii */
+                                    else if (uppr * 10 > utfX) return 0;   /* default code page */
+                                    else return 65001;                     /* UTF-8 encoding */
+                                    break;
+                            }
+                            return EndToken;
+                        case 1: // Recognized '{Upper128}',	Shortest string "\xC0"
+                        case 2: // Recognized '{Upper128}',	Shortest string "\x80"
+                        case 3: // Recognized '{Upper128}',	Shortest string "\xE0"
+                        case 4: // Recognized '{Upper128}',	Shortest string "\xF0"
+                            uppr++;
+                            break;
+                        case 5: // Recognized '{Utf8pfx4}{Utf8cont}',	Shortest string "\xF0\x80"
+                            uppr += 2;
+                            break;
+                        case 6: // Recognized '{Utf8pfx4}{Utf8cont}{2}',	Shortest string "\xF0\x80\x80"
+                            uppr += 3;
+                            break;
+                        case 7: // Recognized '{Utf8pfx4}{Utf8cont}{3}',	Shortest string "\xF0\x80\x80\x80"
+                            utfX += 3;
+                            break;
+                        case 8: // Recognized '{Utf8pfx3}{Utf8cont}',	Shortest string "\xE0\x80"
+                            uppr += 2;
+                            break;
+                        case 9: // Recognized '{Utf8pfx3}{Utf8cont}{2}',	Shortest string "\xE0\x80\x80"
+                            utfX += 2;
+                            break;
+                        case 10: // Recognized '{Utf8pfx2}{Utf8cont}',	Shortest string "\xC0\x80"
+                            utfX++;
+                            break;
+                        default:
+                            break;
+                    }
+#pragma warning restore 162
+                    #endregion
+                }
+            }
+        }
+    } // end class Guesser
+    
+#endif // !BYTEMODE
+#endregion
 #endif // !NOFILES

 // End of code copied from embedded resource
@@ -1,7 +1,7 @@
 <Project Sdk="Microsoft.NET.Sdk">

  <PropertyGroup>
-    <TargetFramework>net8.0</TargetFramework>
+    <TargetFramework>net7.0</TargetFramework>
    <ImplicitUsings>enable</ImplicitUsings>
    <Nullable>enable</Nullable>
  </PropertyGroup>
@@ -1,7 +1,7 @@
 <Project Sdk="Microsoft.NET.Sdk">

  <PropertyGroup>
-    <TargetFramework>net8.0</TargetFramework>
+    <TargetFramework>net7.0</TargetFramework>
    <ImplicitUsings>enable</ImplicitUsings>
    <Nullable>enable</Nullable>
  </PropertyGroup>
@@ -1,7 +1,7 @@
 <Project Sdk="Microsoft.NET.Sdk">

  <PropertyGroup>
-    <TargetFramework>net8.0</TargetFramework>
+    <TargetFramework>net7.0</TargetFramework>
    <ImplicitUsings>enable</ImplicitUsings>
    <Nullable>enable</Nullable>
  </PropertyGroup>
@@ -1,49 +0,0 @@
-using System;
-
-namespace BCG {
-   [DataStreamUserRespresentation("count")]
-   public class CountStream : ViewStream {
-      private DataPacket myLastFiredDataPacket;
-      private IBlueTableSchema mySchema = new BlueTableSchema("count",
-         new ColumnInfo("id", 0, typeof(int)),
-         new ColumnInfo("count", 1, typeof(int)));
-      private int myId = 0;
-      private ViewStream myVS;
-
-      public CountStream(ViewStream vs) {
-         myVS = vs;
-      }
-
-      public CountStream(string tableName)
-         : this(new TableStream(tableName)) { }
-
-      protected override void myOnFirstSubscriber() {
-         myVS.onChange += new ChangeDelegate(myHandleChanges);
-      }
-      protected override void myOnLastSubscriber() {
-         myVS.onChange -= new ChangeDelegate(myHandleChanges);
-      }
-
-
-      private void myHandleChanges(object newValue) {
-         myFireEvent(
-            myLastFiredDataPacket =
-               myLastFiredDataPacket == null ?
-                  new DataPacket(myGetRows(newValue, ++myId), myGetRows(newValue, myId),
-                  new BlueTable(mySchema))
-                  :
-                  new DataPacket(myGetRows(newValue, ++myId), myGetRows(newValue, myId),
-                  myLastFiredDataPacket.newRows)
-            );
-      }
-
-      private IBlueRowCollection myGetRows(object dp, int id) {
-         BlueTable retVal = new BlueTable(mySchema);
-         retVal.addRow
-            (new BlueRow(mySchema, id, (dp as DataPacket).newRows.Count));
-         return retVal;
-      }
-
-
-   }
-}
@@ -1,38 +0,0 @@
-namespace bts.BlueOrb {
-   using bts;
-
-   using BCG;
-
-   using System;
-   [DataStreamUserRespresentation("DateTimeExpanderStream")]
-   public class DateTimeExpanstionStream : ViewStream {
-      public DateTimeExpanstionStream(ViewStream parent_stream, string inp_col) {
-         Func<IBlueTableSchema, IBlueTableSchema> get_schema = (s) => {
-            IBlueTableSchema rv = s.add_column(System.TypeCode.DateTime, $"{inp_col}_date");
-            rv = rv.add_column(System.TypeCode.String, $"{inp_col}_day_of_week");
-            rv = rv.add_column(System.TypeCode.Int32, $"{inp_col}_day_of_month");
-            rv = rv.add_column(System.TypeCode.Int32, $"{inp_col}_month");
-            rv = rv.add_column(System.TypeCode.Int32, $"{inp_col}_hour");
-            rv = rv.add_column(System.TypeCode.Int32, $"{inp_col}_minute");
-            rv = rv.add_column(System.TypeCode.Int32, $"{inp_col}_seconds_of_day");
-            return rv;
-         };
-         ViewStream rs = new MapStream(parent_stream,
-         s => get_schema(s),
-         r => {
-            DateTime dt = r.get_datetime(inp_col);
-            IBlueTableSchema s = get_schema(r.schema);
-            MutableBlueRow rv = new MutableBlueRow(s, r);
-            rv.setValue($"{inp_col}_date", dt.Date);
-            rv.setValue($"{inp_col}_day_of_week", dt.DayOfWeek.ToString());
-            rv.setValue($"{inp_col}_day_of_month", dt.Day);
-            rv.setValue($"{inp_col}_month", dt.Date.Month);
-            rv.setValue($"{inp_col}_hour", dt.Hour);
-            rv.setValue($"{inp_col}_minute", dt.Minute);
-            rv.setValue($"{inp_col}_seconds_of_day", (dt - dt.Date).TotalSeconds);
-            return rv;
-         });
-         mySetRealSource(rs);
-      }
-   }
-}
@@ -1,158 +0,0 @@
-namespace bts.BlueOrb {
-   using BCG;
-   using System;
-   using System.Text.RegularExpressions;
-
-
-   public class RegExpFilter : IStreamFilter {
-      private Regex[] myRegExps;
-      private string[] myCols;
-
-      public RegExpFilter(string col, string regExp)
-         : this(new string[] { col }, new string[] { regExp }) { }
-
-      public RegExpFilter(string[] cols, string[] regExps) {
-         myCols = cols;
-         myRegExps = new Regex[regExps.Length];
-         for (int i = 0; i < regExps.Length; i++)
-            myRegExps[i] = new Regex(regExps[i]);
-      }
-
-      #region IStreamFilter Members
-      public bool filter(IBlueRow dr) {
-         bool rv = true;
-         for (int i = 0; i < myCols.Length; i++) {
-            rv &= myRegExps[i].IsMatch(dr[myCols[i]] is DBNull ? "" : dr[myCols[i]].ToString());
-            if (!rv)
-               return rv;
-         }
-         return rv;
-      }
-      #endregion
-   }
-
-   //[DataStreamUserReprsentation("FilterStream")]
-   public class FilterStream : ViewStream {
-      private BlueTable myLastFiredRows;
-      private IBlueRowCollection myLastReceivedRows;
-      private IStreamFilter myFilter;
-      private ViewStream myParentStream;
-      private string myFilterString;
-      private bool myUseNewFilter = false;// Configuration.get<DataStreamConfig>().UseNewFilter;
-
-      #region constructors
-      public delegate bool OnFilter(IBlueRow r);
-      private class OnFilterWrapper : IStreamFilter {
-         public OnFilterWrapper(OnFilter f) { this.f = f; }
-
-         public bool filter(IBlueRow dr) { return f(dr); }
-
-         private OnFilter f;
-      }
-      public FilterStream(ViewStream vs, OnFilter f)
-         : this(vs, new OnFilterWrapper(f)) { }
-
-
-      public FilterStream(ViewStream vs, IStreamFilter filter) {
-         {
-            myParentStream = vs;
-            myFilter = filter;
-            myFilterString = filter.ToString();
-            if (filter is IDynamicStreamFilter)
-               (filter as IDynamicStreamFilter).onFilterChange += new FilterChangeDelegate(myHandleFilterChange);
-         }
-      }
-
-      public FilterStream(ViewStream vs, string filterExpression)
-         : this(vs, new BooleanExpressionFilter(filterExpression)) {
-         myFilterString = filterExpression;
-      }
-
-      public FilterStream(ViewStream vs, string col, string regExp)
-         : this(vs, new RegExpFilter(col, regExp)) {
-         myFilterString = col + " , " + regExp;
-      }
-
-      public FilterStream(ViewStream vs)
-         : this(vs, new BinaryFilter(true)) {
-         myFilterString = true.ToString();
-      }
-      #endregion
-
-      #region overridden methods
-      protected override void myOnFirstSubscriber() { myParentStream.onChange += new ChangeDelegate(myHandleCallbacks); }
-
-      protected override void myOnLastSubscriber() {
-         myLastFiredRows = null;
-         myParentStream.onChange -= new ChangeDelegate(myHandleCallbacks);
-      }
-
-      protected override void myFireEvent(object o) {
-         DataPacket dp = o as DataPacket;
-         if (myLastValue == null || dp.addedRows.Count + dp.deletedRows.Count > 0)
-            base.myFireEvent(o);
-      }
-      #endregion
-
-      private void myHandleCallbacks(object newValue) {
-         DataPacket retVal;
-         DataPacket arrivedPacket = newValue as DataPacket;
-         myLastReceivedRows = arrivedPacket.newRows;
-         if (myLastFiredRows != null) {
-            retVal = new DataPacket(myLastFiredRows, new BlueTable(myLastFiredRows.schema),
-               new BlueTable(myLastFiredRows.schema));
-            foreach (IBlueRow r in arrivedPacket.deletedRows)
-               if (myLastFiredRows.containsID((int)r["id"])) {
-                  myLastFiredRows.deleteRow(r);
-                  (retVal.deletedRows as BlueTable).addRow(r);
-               }
-            foreach (IBlueRow r in arrivedPacket.addedRows)
-               if (myFilter.filter(r)) {
-                  myLastFiredRows.addRow(r);
-                  (retVal.addedRows as BlueTable).addRow(r);
-               }
-         } else {
-            myLastFiredRows = new BlueTable(arrivedPacket.schema);
-            retVal = new DataPacket(myLastFiredRows, myLastFiredRows, new BlueTable(myLastFiredRows.schema));
-            foreach (IBlueRow r in arrivedPacket.newRows)
-               if (myFilter.filter(r))
-                  myLastFiredRows.addRow(r);
-         }
-         myFireEvent(retVal);
-      }
-
-      public IStreamFilter filter {
-         set {
-            {
-               myFilter = value;
-               if (numberOfSubscribers > 0) {
-                  if (myLastReceivedRows != null) {
-                     BlueTable nRows = new BlueTable(myLastReceivedRows.schema);
-                     foreach (IBlueRow r in myLastReceivedRows)
-                        if (myFilter.filter(r))
-                           nRows.addRow(r);
-                     IBlueRowCollection dRows = new BlueTable(nRows.schema),
-                        aRows = new BlueTable(nRows.schema);
-                     BlueDataUtility.computeDifferences(nRows, myLastFiredRows, out dRows, out aRows);
-                     myLastFiredRows = nRows;
-                     myFireEvent(new DataPacket(nRows, aRows, dRows));
-                  }
-               }
-            }
-         }
-         get {            
-               return myFilter;
-         }
-      }
-
-      //public string filterString { set { filter = new BooleanExpressionFilter(value); } }
-
-      public override string ToString() {
-         return GetType().ToString() + " " + myFilterString;
-      }
-
-      private void myHandleFilterChange() {
-         filter = myFilter;
-      }
-   }
-}
@@ -1,121 +0,0 @@
-using System;
-using System.Collections.Generic;
-using System.Diagnostics;
-using System.Linq;
-using System.Text.Json;
-
-using bts;
-//using bts.SilverPine;
-using BCG;
-
-namespace bts.BlueOrb {
-   [DataStreamUserRespresentation("GroupByStream")]
-   public class GroupByStream : ViewStream {
-
-      public interface INonIncGroupByEvaluator {
-         // grouped_rows
-         IBlueRowCollection evaluate(IDictionary<int, List<IBlueRow>> grouped_rows);
-         string[] get_group_by_cols();
-      }
-
-      private class DelNonIncGroupByEvaluator : INonIncGroupByEvaluator {
-         Func<string[]> _groupby_cols_func;
-         Func<List<IBlueRow>, List<IBlueRow>> _evaluate_func;
-         public DelNonIncGroupByEvaluator(Func<string[]> groupby_cols_func,
-                                          Func<List<IBlueRow>, List<IBlueRow>> evaluate_func) {
-            _groupby_cols_func = groupby_cols_func;
-            _evaluate_func = evaluate_func;
-         }
-         public IBlueRowCollection evaluate(IDictionary<int, List<IBlueRow>> grouped_rows) {
-            BlueTable rv = null;
-            foreach (var kv in grouped_rows) {
-               List<IBlueRow> g_rows = kv.Value;
-               List<IBlueRow> out_rows = _evaluate_func.Invoke(g_rows);
-               if (out_rows != null && out_rows.Count > 0) {
-                  rv = rv ?? new BlueTable(out_rows.ElementAt(0).schema);
-               }
-               rv.addRows(out_rows);
-            }
-            return rv;
-         }
-         public string[] get_group_by_cols() {
-            return _groupby_cols_func();
-         }
-
-      }
-      private ViewStream _main_stream;
-      private INonIncGroupByEvaluator _groupby_func;
-      private BlueRowHasher _row_hasher;
-      private string[] _groupby_cols;
-      private Stopwatch _timer = new();
-      private int _n_total_fires = 0, _n_expensive_fires = 0;
-
-      public GroupByStream(ViewStream main_stream, INonIncGroupByEvaluator groupby_func) {
-         _main_stream = main_stream;
-         _groupby_func = groupby_func;
-         _groupby_cols = _groupby_func.get_group_by_cols();
-         _row_hasher = new BlueRowHasher(_groupby_cols == null ? 0 : _groupby_cols.Length);
-      }
-
-      public GroupByStream(ViewStream main_stream, Func<string[]> groupby_cols_func,
-                                 Func<List<IBlueRow>, List<IBlueRow>> evaluate_func)
-                                 : this(main_stream, new DelNonIncGroupByEvaluator(groupby_cols_func, evaluate_func)) { }
-
-      public GroupByStream(ViewStream parent_str, string groupby_col, string standard_group_by_text)
-      : this(parent_str, groupby_col, standard_group_by_text, String.Empty) { }
-
-      // this is the main constructor that does everything
-      public GroupByStream(ViewStream parent_str, string groupby_col, string standard_group_by_ops, string optional_sort_col) {
-         MultiRowOpHelper std_group_by = MultiRowOpHelper.parse_ops(standard_group_by_ops);
-         Func<string[]> groupby_cols_func = () => {
-            return String.IsNullOrEmpty(groupby_col) ? null : new string[] { groupby_col };
-         };
-         Func<List<IBlueRow>, List<IBlueRow>> evaluate_func = (group_rows) => {
-            if (!String.IsNullOrEmpty(optional_sort_col))
-               group_rows.Sort((r1, r2) => BlueDataUtility.compare_rows(r2, r1, optional_sort_col));
-            // this si where the group by aggregation happens
-            IBlueRow rv = MultiRowOpHelper.group_by_compute(group_rows, std_group_by);
-            return new IBlueRow[] { rv }.ToList();
-         };
-         ViewStream rs = new GroupByStream(parent_str, groupby_cols_func, evaluate_func);
-         mySetRealSource(rs);
-      }
-      public GroupByStream(ViewStream parent_str, string standard_group_by_text) :
-      this(parent_str, String.Empty, standard_group_by_text) {
-      }
-
-
-      protected override void myOnFirstSubscriber() { _main_stream.onChange += _main_stream_onchange; }
-
-      protected override void myOnLastSubscriber() { _main_stream.onChange -= _main_stream_onchange; }
-
-      private void _main_stream_onchange(object nv) {
-         DataPacket dp = nv as DataPacket;
-         ///grouping together
-         _timer.Restart();
-         IDictionary<int, List<IBlueRow>> grouped_rows = (_groupby_cols == null || _groupby_cols.Length == 0) ?
-         new Dictionary<int, List<IBlueRow>>() { { 0, dp.newRows.to_list() } } :
-         BlueDataUtility.group_together(dp.newRows, _groupby_cols, ref _row_hasher);
-         _timer.Stop();
-         var groupElapsedMs = _timer.ElapsedMilliseconds;
-         if (groupElapsedMs >= 50) {
-            Log.DataFlow.info($"Time to group {dp.newRows.Count} rows into {grouped_rows.Count} groups: {_timer.ElapsedMilliseconds}ms");
-         }
-         // Log.log(bts.Log.Level.Info, bts.log.Subsystem.gen, String.Format("Time to group {0} rows into {1} groups = {2}", dp.newRows.Count, grouped_rows.Count, _timer.lastIntervalTime));
-         /// reducing
-         _timer.Restart();
-         IBlueRowCollection total_state = _groupby_func.evaluate(grouped_rows);
-         _timer.Stop();
-         var applyElapsedMs = _timer.ElapsedMilliseconds;
-         _n_total_fires++;
-         if (applyElapsedMs >= 50) {
-            _n_expensive_fires++;
-            Log.DataFlow.info(
-               $"Time to evaluate {grouped_rows.Count} groups {_timer.ElapsedMilliseconds}ms, ratio {_n_expensive_fires}/{_n_total_fires}");
-         }
-         // TODO,
-         if (total_state != null)
-            myFireEvent(new DataPacket(total_state, total_state, myLastValue == null ? new BlueTable(total_state.schema) : (myLastValue as DataPacket).newRows));
-      }
-   }
-}
@@ -1,17 +0,0 @@
-using System;
-using System.Collections.Generic;
-using System.Linq;
-using System.Text;
-using System.Threading.Tasks;
-using BCG;
-
-namespace bts.BlueOrb {   
-   public interface INonIncJoinInlineMapper
-   {      
-      IBlueTableSchema get_output_schema(IBlueTableSchema schema);
-
-      IBlueRowCollection map_rows(IBlueRowCollection rows);
-      
-      event MapperChangeDelegate onMapperChange;
-   }
-}
@@ -1,152 +0,0 @@
-using bts;
-using bts.collections;
-
-using mbts;
-
-using System;
-using System.Collections.Generic;
-using System.Diagnostics;
-using System.Linq;
-using System.Security.Cryptography;
-using BCG;
-
-namespace bts.BlueOrb {   
-   internal class NonIncJoinBookkeeper : IDisposable {
-      public Action on_new_data;
-      private readonly NonIncJoinSpec _join_spec;
-      private readonly int _join_position;
-      private readonly int _start_index_in_output;
-      private readonly BlueRowHasher _row_hasher;
-      private readonly bool _is_left_most;
-      private readonly bool _is_tag_along;
-      private int _n_total_fires = 0, _n_expensive_fires = 0;
-
-      private IBlueTableSchema _schema = null;
-      private IDictionary<int, IBlueRow> _row_map = new Dictionary<int, IBlueRow>();
-      private List<Pair<int, IBlueRow>> _row_list = new List<Pair<int, IBlueRow>>();
-      private IBlueRow _solitary_row = null;
-
-      private int[] _join_col_indices = null,
-                     _data_col_indices = null;
-
-      public NonIncJoinBookkeeper(NonIncJoinSpec the_join_spec, int join_position, int start_index_in_output, BlueRowHasher row_hasher) {
-         _join_spec = the_join_spec;
-         _join_position = join_position;
-         _start_index_in_output = start_index_in_output;
-         _row_hasher = row_hasher;
-         _is_left_most = _join_position == 0;
-         _is_tag_along = _join_spec.join_col_names.Length == 0;
-      }
-
-      private bool _filled_missing_data = false;
-      private void _fill_missing_data(DataPacket dp) {
-         if (_filled_missing_data)
-            return;
-         Func<string[], bool> null_or_empty = (str_arr) => {
-            if (str_arr == null || str_arr.Length == 0)
-               return true;
-            foreach (string s in str_arr) {
-               if (!String.IsNullOrEmpty(s)) {
-                  return false;
-               }
-            }
-            return true;
-         };
-         Func<string[], string[]> remove_bad_cols = (col_names) => {
-            HashSet<string> schema_col_map = new HashSet<string>(dp.schema.columnNames);
-            List<string> cols_list = new List<string>(col_names);
-            cols_list.RemoveWhere(c => !schema_col_map.Contains(c));
-            return cols_list.ToArray();
-         };
-         if (null_or_empty(_join_spec.data_col_names)) {
-            _join_spec.data_col_names = dp.schema.columnNames;
-         }
-         _join_spec.data_col_names = remove_bad_cols(_join_spec.data_col_names);
-         if (null_or_empty(_join_spec.data_col_names)) {
-            _join_spec.data_col_names = dp.schema.columnNames;
-         }
-         if (null_or_empty(_join_spec.projected_col_names)) {
-            _join_spec.projected_col_names = _join_spec.data_col_names;
-         }
-      }
-
-      public void init() {
-         _join_spec.stream.onChange += _on_stream_cb;
-      }
-
-      public void _on_stream_cb(object dp_obj) {
-         //Log.Gen.info($"NonIncJoinBookkeeper._on_stream_cb dp_obj: {dp_obj}");
-         var timer = Stopwatch.StartNew();
-         DataPacket dp = (DataPacket)dp_obj;
-         _fill_missing_data(dp);
-         if (_schema == null) {
-            _schema = dp.schema;
-            _join_col_indices = BlueDataUtility.get_column_indices(_schema, _join_spec.join_col_names);
-            _data_col_indices = BlueDataUtility.get_column_indices(_schema, _join_spec.data_col_names);
-         }
-
-         if (_is_left_most) {
-            _row_list.Clear();
-            foreach (IBlueRow r in dp.newRows)
-               _row_list.Add(new Pair<int, IBlueRow>(_row_hasher.get_hash(r, _join_col_indices), r));
-         } else {
-            if (_is_tag_along)
-               _solitary_row = BlueDataUtility.extractRow(dp.newRows, 0);
-            else {
-               _row_map.Clear();
-               foreach (IBlueRow r in dp.newRows)
-                  _row_map[_row_hasher.get_hash(r, _join_col_indices)] = r;
-            }
-         }
-         timer.Stop();
-         _n_total_fires++;
-         if (timer.ElapsedMilliseconds >= 50) {
-            _n_expensive_fires++;
-            Log.Gen.info($"NonIncJoinBookkeepingData._on_stream_cb {dp.newRows.Count} rows {_join_col_indices.Length} "
-               + $"key cols {_join_position} join pos time {timer.Elapsed} ratio {_n_expensive_fires}/{_n_total_fires}");
-         }
-         if (on_new_data != null)
-            on_new_data();
-      }
-
-      public bool is_ready { get { return _schema != null; } }
-
-      public void constribute_cols_to_schema(List<ColumnInfo> cis) {
-         for (int i = 0; i < _join_spec.data_col_names.Length; i++)
-            cis.Add(new ColumnInfo(_join_spec.projected_col_names[i], cis.Count, _schema[_join_spec.data_col_names[i]].colType));
-      }
-
-      public List<Pair<int, IBlueRow>> get_list() { return _row_list; }
-
-      public void copy_master_row_to_output_row(int row_index, object[] output_row) {
-         IBlueRow this_row = _row_list[row_index].second;
-         BlueDataUtility.copy_to_row(this_row, _data_col_indices, output_row, _start_index_in_output);
-      }
-
-      public void copy_matching_row_to_output_row(int row_hash, object[] output_row) {
-         if (_is_tag_along) {
-            if (_solitary_row != null)
-               BlueDataUtility.copy_to_row(_solitary_row, _data_col_indices, output_row, _start_index_in_output);
-            else
-               copy_dbnull_to_output_row(output_row);
-         } else {
-            IBlueRow this_row = null;
-            if (_row_map.TryGetValue(row_hash, out this_row))
-               BlueDataUtility.copy_to_row(this_row, _data_col_indices, output_row, _start_index_in_output);
-            else
-               copy_dbnull_to_output_row(output_row);
-         }
-      }
-
-      private void copy_dbnull_to_output_row(object[] output_row) {
-         for (int i = 0; i < _data_col_indices.Length; i++)
-            output_row[i + _start_index_in_output] = DBNull.Value;
-      }
-
-      #region IDisposable
-
-      public void Dispose() { _join_spec.stream.onChange -= _on_stream_cb; }
-
-      #endregion
-   }
-}
@@ -1,67 +0,0 @@
-using System;
-using System.Collections.Generic;
-using System.Linq;
-using System.Text;
-using System.Threading.Tasks;
-
-using BCG;
-
-using mbts;
-
-namespace bts.BlueOrb {   
-   // this a collection of spec, theres no joining logic in here
-   public class NonIncJoinSpec {
-      public ViewStream stream;
-      // join col names are the names on which the streams are joined (for equality),
-      // data col names are the names in the original stream
-      // projected col names are their new names, ie if its x:l_x, then x is in the 
-      // data_col_names and l_x in projected cols
-      public string[] join_col_names, data_col_names, projected_col_names;
-
-      public NonIncJoinSpec() { }
-
-      public NonIncJoinSpec(ViewStream the_stream, string[] the_join_col_names, string[] the_data_col_names)
-         : this(the_stream, the_join_col_names, the_data_col_names, the_data_col_names) { }
-
-      public NonIncJoinSpec(ViewStream the_stream, string[] the_join_col_names, string[] the_data_col_names, string[] the_projected_col_names) {
-         stream = the_stream;
-         join_col_names = the_join_col_names;
-         data_col_names = the_data_col_names;
-         projected_col_names = the_projected_col_names;
-      }
-
-      public static NonIncJoinSpec[] from_triplet(params Triplet<ViewStream, string, string>[] join_specs) {
-         List<NonIncJoinSpec> rv = new List<NonIncJoinSpec>();
-         foreach (Triplet<ViewStream, string, string> r in join_specs) {
-            string[] keys = String.IsNullOrEmpty(r.second) ? new string[] { } : r.second.Split(' ');
-            string[] visible_cols = r.third.Split(' ');
-            List<string> data_col_names = new List<string>(), mapped_col_names = new List<string>();
-            foreach (string s in visible_cols) {
-               string[] toks = s.Split(':');
-               if (toks.Length == 1) {
-                  data_col_names.Add(toks[0]);
-                  mapped_col_names.Add(toks[0]);
-               } else if (toks.Length == 2) {
-                  data_col_names.Add(toks[0]);
-                  mapped_col_names.Add(toks[1]);
-               }
-            }
-            rv.Add(new NonIncJoinSpec(r.first, keys, data_col_names.ToArray(), mapped_col_names.ToArray()));
-         }
-         return rv.ToArray();
-      }
-
-      public override string ToString() {
-         Func<string[], string> arr_txt = (a) => {
-            string rv = "";
-            foreach (string s in a) {
-               rv += "{s} ";
-            }
-            rv = rv.Trim();
-            return rv;
-         };
-         string rv = $"stream: {stream} join_col_names: {arr_txt(join_col_names)} data_col_names: {arr_txt(data_col_names)} projected_col_names: {arr_txt(projected_col_names)}";
-         return rv;
-      }
-   }
-}
@@ -1,165 +0,0 @@
-using bts;
-
-using mbts;
-
-using System;
-using System.Collections.Generic;
-using System.Diagnostics;
-using BCG;
-
-namespace bts.BlueOrb {   
-   [DataStreamUserRespresentation("JoinStream")]
-   public class NonIncJoinStream : ViewStream, BCG.Utilities.General.IWorker {
-      private readonly NonIncJoinSpec[] _join_specs;
-      private readonly INonIncJoinInlineMapper _output_mapper;
-      private readonly BlueRowHasher _row_hasher;
-      private readonly TimeSpan _throttle_period;
-      private readonly ViewStreamDatabase _vsd;
-
-      private List<NonIncJoinBookkeeper> _book_keeping_data = new List<NonIncJoinBookkeeper>();
-      private IBlueTableSchema _output_schema = null;
-      private DateTime _last_fire_time = DateTime.MinValue;
-      private bool _is_dirty = false;
-
-      public NonIncJoinStream(ViewStream left, string left_join_cols, string left_proj,
-                              ViewStream right, string right_join_cols, string right_proj)
-         : this(new Triplet<ViewStream, string, string>[]
-               {
-                  new Triplet<ViewStream, string, string> (left, left_join_cols, left_proj),
-                  new Triplet<ViewStream, string, string> (right, right_join_cols, right_proj)
-               }) { }
-
-      public NonIncJoinStream(params Triplet<ViewStream, string, string>[] join_specs)
-         : this(NonIncJoinSpec.from_triplet(join_specs), null, 0) { }
-
-      public NonIncJoinStream(ViewStreamDatabase vsd, INonIncJoinInlineMapper output_mapper, int throttles_ms,
-                              params Triplet<ViewStream, string, string>[] join_specs)
-         : this(vsd, NonIncJoinSpec.from_triplet(join_specs), output_mapper, throttles_ms) { 
-      }
-
-      public NonIncJoinStream(NonIncJoinSpec[] join_specs, INonIncJoinInlineMapper output_mapper, int throttles_ms = 0)
-         : this(ViewStreamDatabase.instance, join_specs, output_mapper, throttles_ms) { }
-
-      public NonIncJoinStream(ViewStreamDatabase vsd, NonIncJoinSpec[] join_specs, INonIncJoinInlineMapper output_mapper, int throttles_ms = 0) {
-         _join_specs = join_specs;
-         _output_mapper = output_mapper;
-         _row_hasher = new BlueRowHasher(_join_specs[0].join_col_names.Length);
-         _throttle_period = throttles_ms == 0 ? TimeSpan.Zero : new TimeSpan(0, 0, 0, 0, throttles_ms);
-         _vsd = vsd;
-      }
-
-      protected override void myOnFirstSubscriber() {
-         int data_col_no = 1;
-         for (int jp = 0; jp < _join_specs.Length; jp++) {
-            NonIncJoinBookkeeper s = new NonIncJoinBookkeeper(_join_specs[jp], jp, data_col_no, _row_hasher);
-            //Log.Gen.info($"NonIncJoinStream.myOnFirstSubscriber jp: {jp} spec: {_join_specs[jp]}");
-            data_col_no += _join_specs[jp].data_col_names.Length;
-            s.on_new_data += _stream_callback;
-            _book_keeping_data.Add(s);
-         }
-         if (_output_mapper != null)
-            _output_mapper.onMapperChange += _on_mapper_change;
-         _vsd.addWorker(this);
-         foreach (var bkd in _book_keeping_data)
-            bkd.init();
-      }
-
-      protected override void myOnLastSubscriber() {
-         foreach (NonIncJoinBookkeeper d in _book_keeping_data)
-            d.Dispose();
-         if (_output_mapper != null)
-            _output_mapper.onMapperChange -= _on_mapper_change;
-         _vsd.removeWorker(this);
-      }
-
-      private void _stream_callback() {
-         //Log.Gen.info("NonIncJoinStream._stream_callback received callback");
-         _is_dirty = true;
-         doWork();
-      }
-
-      private void _on_mapper_change() {
-         _is_dirty = true;
-         doWork();
-      }
-
-      private bool _is_ready() {
-         bool rv = true;
-         foreach (NonIncJoinBookkeeper d in _book_keeping_data)
-            rv &= d.is_ready;
-         return rv;
-      }
-
-      private void _construct_schema() {
-         if (_output_schema == null) {
-            List<ColumnInfo> cis = new List<ColumnInfo>(new ColumnInfo[] { new ColumnInfo("id", 0, typeof(int)) });
-            foreach (NonIncJoinBookkeeper d in _book_keeping_data)
-               d.constribute_cols_to_schema(cis);
-            _output_schema = new BlueTableSchema("NonIncJoinStream", cis.ToArray());
-         }
-      }
-
-      private int _n_total_fires = 0, _n_expensive_fires = 0;
-      private void _form_and_fire() {
-         //Log.Gen.info($"NonIncJoinStream._form_and_fire _is_ready(): {_is_ready()}");
-         if (_is_ready()) {
-            _construct_schema();
-            var timer = Stopwatch.StartNew();
-            IList<Pair<int, IBlueRow>> left_row_list = _book_keeping_data[0].get_list();
-            List<object[]> output_rows = new List<object[]>();
-
-            for (int i = 0; i < left_row_list.Count; i++) {
-               object[] or = new object[_output_schema.columnInfo.Length];
-               or[0] = i;
-               output_rows.Add(or);
-            }
-
-            for (int ji = 0; ji < _book_keeping_data.Count; ji++) {
-               NonIncJoinBookkeeper bk_data = _book_keeping_data[ji];
-               for (int i = 0; i < left_row_list.Count; i++) {
-                  if (ji == 0)
-                     bk_data.copy_master_row_to_output_row(i, output_rows[i]);
-                  else
-                     bk_data.copy_matching_row_to_output_row(left_row_list[i].first, output_rows[i]);
-               }
-            }
-            IBlueRowCollection dp_output_rows = _apply_map(new SequentialIDBlueTable(_output_schema, output_rows));
-            timer.Stop();
-            _n_total_fires++;
-            //if (timer.ElapsedMilliseconds >= 250) {
-            _n_expensive_fires++;
-            //string log_msg = $"Join and map time for {_book_keeping_data.Count} "
-            //   + $"tables {timer.Elapsed.TotalMilliseconds} msecs, ratio {_n_expensive_fires}/{_n_total_fires}";
-            //Log.Gen.info(log_msg);
-            //}
-            var dp_out = BlueDataUtility.build_dp_inc(dp_output_rows, myLastValue as DataPacket, true);
-            myFireEvent(dp_out);
-            //myFireEvent(new DataPacket(dp_output_rows, dp_output_rows, myLastValue == null ? new BlueTable(_output_schema) : (myLastValue as DataPacket).newRows));
-         }
-      }
-
-      private IBlueRowCollection _apply_map(IBlueRowCollection unmapped_rows) {
-         if (_output_mapper != null) {
-            IBlueRowCollection output_rows;
-            var timer = Stopwatch.StartNew();
-            output_rows = _output_mapper.map_rows(unmapped_rows);
-            timer.Stop();
-            if (timer.ElapsedMilliseconds >= 50) {
-               Log.Gen.info($"Time to map {unmapped_rows.Count} rows into {output_rows.Count} rows {timer.Elapsed}");
-            }
-            return output_rows;
-         } else {
-            return unmapped_rows;
-         }
-      }
-
-      public void doWork() {
-         //Log.Gen.info($"NonIncJoinStream.doWork _last_fire_time: {_last_fire_time} _is_dirty: {_is_dirty}");
-         if (DateTime.Now - _last_fire_time >= _throttle_period && _is_dirty) {
-            _is_dirty = false;
-            _last_fire_time = DateTime.Now;
-            _form_and_fire();
-         }
-      }
-   }
-}
@@ -1,28 +0,0 @@
-//namespace bts.Blueorb {
-//   using BCG;
-
-//   public class JsonSaverStream : ViewStream {
-//      private ViewStream _stream;
-//      private string _file_name;
-//      public JsonSaverStream(ViewStream vs, string file_name) {
-//         _stream = vs;
-//         _file_name = file_name;
-//      }
-
-//      protected override void myOnFirstSubscriber() {
-//         _stream.onChange += _on_data;
-//      }
-
-//      private void _on_data(object dp_obj) {
-//         if (dp_obj == null) {
-//            return;
-//         }
-//         DataPacket dp = dp_obj as DataPacket;
-//         dp.newRows.write_to_json_file(dp.newRows, _file_name);
-//      }
-
-//      protected override void myOnLastSubscriber() {
-//         _stream.onChange -= _on_data;
-//      }
-//   }
-//}
@@ -1,91 +0,0 @@
-using System;
-using System.Collections;
-using System.Collections.Generic;
-
-using Amazon.Runtime.Internal.Util;
-
-using BCG;
-using BCG.Utilities;
-
-using bts;
-
-namespace bts.BlueOrb{
-   public class LiteralStream : ViewStream {
-
-      public static List<Tuple<ColumnInfo, object>> parse_columns(string key_value_pairs) {
-         List<Tuple<ColumnInfo, object>> rv = new List<Tuple<ColumnInfo, object>>();
-         string[] pairs = key_value_pairs.Split(' ');
-         foreach (string p in pairs) {
-            string[] kvp = p.Split(':');
-            string col_name = kvp[0];
-            string val = kvp[1];
-            var tup = BlueDataUtility.guess_type(val);
-            var ci = new ColumnInfo(col_name, rv.Count, tup.Item1);
-            rv.Add(Tuple.Create(ci, tup.Item2));
-         }
-         return rv;
-      }
-
-      //literal("table1", "x:123, y:1.001, z:abc;x:123, y:1.001, z:abc")
-      // col_name: col_value ... ; col_name: col_value
-      public LiteralStream(string table_name, string rows) {
-         Func<string, int, IBlueRow> get_row = (json_row, id) => {
-            Dictionary<string, object> map = new Dictionary<string, object>();
-            string[] toks = json_row.Split(',', StringSplitOptions.TrimEntries);
-            //Logger.assert(col_vals.Length == 2);
-            foreach (string t in toks) {
-               var col_vals = t.Split(':', StringSplitOptions.TrimEntries);
-               string col = col_vals[0];
-               string val = col_vals[1];
-               map.Add(col, val);
-            }
-            map["id"] = id;
-            return new BlueRow(map);
-         };
-         string[] row_arr = rows.Split(';', StringSplitOptions.TrimEntries);
-         BlueTable table = null;
-         foreach (string json_string in row_arr) {
-            IBlueRow r = //BlueDataUtility.convert_json_to_row(json_string, table == null ? 0 : table.Count);
-            get_row(json_string, table == null ? 0 : table.Count);
-            table = table ?? new BlueTable(r.schema);
-            table.addRow(r);
-         }
-         ViewStream str = new BlueTableStream(table);
-         str = new SchemaGuesserStream(str);
-         mySetRealSource(str);
-      }
-
-      public LiteralStream(string tableName, string schema, string value) {
-         string[] columns = schema.Split(' ');
-
-
-         ColumnInfo[] ci = new ColumnInfo[columns.Length + 1];
-         ci[0] = new ColumnInfo("id", 0, typeof(int));
-         int n = 1;
-         foreach (string c in columns) {
-            string[] columnAndType = c.Split('/');
-            if (columnAndType.Length == 1)
-               ci[n] = new ColumnInfo(c, n, typeof(double));
-            else
-               ci[n] = new ColumnInfo(columnAndType[0], n, Type.GetType("System." + columnAndType[1], true));
-            n++;
-         }
-
-
-         BlueTable t = new BlueTable(new BlueTableSchema(tableName, ci));
-         string[] rows = value.Split(';');
-         n = 0;
-         foreach (string r in rows)
-            try {
-               if (r == string.Empty)
-                  continue;
-               t.addRow(
-                  BlueDataUtility.createRowFromCSVFileRow(r, t.schema, ++n));
-            } catch (Exception e) {
-               throw (new Exception("Error occured at line: " + n.ToString() + " " + e.Message));
-            }
-
-         mySetRealSource(new BlueRowCollectionStaticStream(t));
-      }
-   }
-}
@@ -1,89 +0,0 @@
-using System;
-
-using BCG.Utilities.General;
-using BCG;
-using bts;
-
-namespace bts.BlueOrb {
-   [DataStreamUserRespresentation("MillisecondStream")]
-   public class MillisecondStream : ViewStream, IWorker {
-      private DateTime _last_fire_time = DateTime.MinValue;
-      private readonly TimeSpan _period;
-      private readonly int _size;
-      private readonly ViewStreamDatabase _vsd;
-      private IBlueTableSchema _schema;
-      private bts.CircularBufferTable _table;
-
-      public MillisecondStream(int interval_msecs, int size)
-      : this(ViewStreamDatabase.instance, interval_msecs, size) {
-      }
-
-      public MillisecondStream(ViewStreamDatabase vsd, int interval_msecs, int size) {
-         _vsd = vsd;
-         _period = TimeSpan.FromMilliseconds(interval_msecs);
-         _size = size;
-         _schema = BlueDataUtility.form_schema("Milliseconds",
-               Tuple.Create("id", TypeCode.Int32),
-               Tuple.Create("time", TypeCode.DateTime),
-               Tuple.Create("display_time", TypeCode.String));
-         _table = new CircularBufferTable(_schema, _size);
-      }
-
-      protected override void myOnFirstSubscriber() {
-         _vsd.addWorker(this);
-      }
-
-      protected override void myOnLastSubscriber() {
-         _vsd.removeWorker(this);
-      }
-
-      private int _id = 0;
-      public void doWork() {
-         if (DateTime.Now - _last_fire_time >= _period) {
-            _last_fire_time = DateTime.Now;
-            IBlueRow nr = new BlueRow(_schema, _id++, _last_fire_time,
-            _last_fire_time.ToString("yyyy-MM-dd HH:mm:ss.fff"));
-            _table.add_row(nr);
-            var dp = BlueDataUtility.build_dp_inc(_table, myLastValue as DataPacket);
-            myFireEvent(dp);
-         }
-      }
-   }
-   public class MSecondStream : TimeUnitStream {
-      public MSecondStream(int intervalInms, int size)
-         : base(DateTime.Now, new TimeSpan(0, 0, 0, 0, intervalInms), size, ViewStreamDatabase.instance) { }
-
-      protected override DateTime myRoundTime(DateTime dt) { return dt; }
-   }
-
-   public class DisplayableMillisecondStream : ViewStream, IStreamMapper {
-      private IBlueTableSchema mySchema = new BlueTableSchema("ms",
-                                                               new ColumnInfo("id", 0, typeof(int)),
-                                                               new ColumnInfo("time", 1, typeof(string)));
-      public DisplayableMillisecondStream(int intervalInMs, int sz) {
-         mySetRealSource(new MapStream(new MSecondStream(intervalInMs, sz), this));
-      }
-      #region IStreamMapper Members
-
-      public IBlueTableSchema getOutputSchema(IBlueTableSchema inputSchema) {
-         return mySchema;
-      }
-
-      #endregion
-
-      #region IDynamicStreamMapper Members
-
-      public IBlueRow map(IBlueRow row) {
-         DateTime time = row.get_datetime("time");
-         var time_txt = time.ToString("yyyy-MM-dd HH:mm:ss.fff");
-         return new BlueRow(mySchema, row["id"], time_txt);
-      }
-
-      #endregion
-
-      private string myToString(DateTime dt) {
-         string sep = ":";
-         return dt.Hour + sep + dt.Minute + sep + dt.Second + sep + dt.Millisecond;
-      }
-   }
-}
@@ -1,127 +0,0 @@
-namespace bts.BlueOrb {
-   using BCG;
-
-   using System;
-   using System.Collections.Generic;
-   using System.Linq;
-   //using MathNet.Numerics.Statistics;
-
-
-   public enum MultiRowOp {
-      Mean,
-      Median,
-      Mode,
-      Range,
-      StdDev,
-      Max,
-      Min,
-      First,
-      Last,
-      Sum,
-      EdgeDiff,
-      EdgeDiffPercentage,
-      //IsEdgeDiff,
-   }
-   public class MultiRowOpHelper {
-      public readonly List<Tuple<string, MultiRowOp>> operations;
-
-      public static MultiRowOpHelper parse_ops(string text) {
-         // text is inp_col: op, inp_col: op, inp_col:op
-         List<Tuple<string, MultiRowOp>> rv = new List<Tuple<string, MultiRowOp>>();
-         string[] calcs = text.Split(',');
-         foreach (string c in calcs) {
-            string[] toks = c.Split(':');
-            string inp_col = toks[0].Trim();
-            string op_txt = toks[1].Trim();
-            MultiRowOp op = Enum.Parse<MultiRowOp>(op_txt, true);
-            rv.Add(Tuple.Create(inp_col, op));
-         }
-         return new MultiRowOpHelper(rv);
-      }
-
-      public MultiRowOpHelper(List<Tuple<string, MultiRowOp>> operations_) {
-         operations = operations_;
-      }
-
-      // aggregates multiple rows into one - ie does group_by compute
-      public static IBlueRow group_by_compute(List<IBlueRow> rows, MultiRowOpHelper parameters) {
-         if (rows.Count() == 0) {
-            return null;
-         }
-
-         IBlueRow ir = rows.ElementAt(0);
-         IBlueTableSchema in_schema = ir.schema;
-         IBlueTableSchema out_schema = in_schema;
-         IDictionary<string, Tuple<string, MultiRowOp>> out_col_map =
-            new Dictionary<string, Tuple<string, MultiRowOp>>();
-         foreach (var t in parameters.operations) {
-            string out_col_name = $"{t.Item1}_{t.Item2}".ToLower();
-            out_schema = out_schema.add_column(TypeCode.Double, out_col_name);
-            out_col_map[out_col_name] = t;
-         }
-         MutableBlueRow or = new MutableBlueRow(out_schema, ir);
-         foreach (var kv in out_col_map) {
-            string out_col_name = kv.Key;
-            string in_col_name = kv.Value.Item1;
-            MultiRowOp op = kv.Value.Item2;
-            IEnumerable<double> col_vals = BlueDataUtility.get_colvalues<double>(rows, in_col_name);
-            double comp_res = compute_op(col_vals, op);
-            or.setValue(out_col_name, comp_res);
-         }
-         return or;
-      }
-
-      // rolling_rows are the rolling window of rows. the 0th row is the one thats gettin
-      // calculated. rows 1 - n-1 are its window rows. the schema is based on the operations.
-
-      public IBlueRow rolling_calc(List<IBlueRow> rolling_rows) {
-         if (rolling_rows.Count() == 0) {
-            return null;
-         }
-         IBlueRow ir = rolling_rows.ElementAt(0);
-         IBlueTableSchema in_schema = ir.schema;
-         IBlueTableSchema out_schema = in_schema;
-         IDictionary<string, Tuple<string, MultiRowOp>> out_col_map =
-            new Dictionary<string, Tuple<string, MultiRowOp>>();
-         foreach (var t in operations) {
-            string out_col_name = $"{t.Item1}_{t.Item2}".ToLower();
-            out_schema = out_schema.add_column(TypeCode.Double, out_col_name);
-            out_col_map[out_col_name] = t;
-         }
-         MutableBlueRow or = new MutableBlueRow(out_schema, ir);
-         foreach (var kv in out_col_map) {
-            string out_col_name = kv.Key;
-            string in_col_name = kv.Value.Item1;
-            MultiRowOp op = kv.Value.Item2;
-            IEnumerable<double> col_vals = BlueDataUtility.get_colvalues<double>(rolling_rows, in_col_name);
-            double comp_res = compute_op(col_vals, op);
-            or.setValue(out_col_name, comp_res);
-         }
-         return or;
-      }
-
-      public static double compute_op(IEnumerable<double> col_vals, MultiRowOp op) {
-         switch (op) {
-            case MultiRowOp.Max:
-               return col_vals.Max<double>();
-            case MultiRowOp.Min:
-               return col_vals.Min<double>();
-            case MultiRowOp.Mean:
-               return col_vals.Sum() / col_vals.Count();
-            case MultiRowOp.Sum:
-               return col_vals.Sum();
-            case MultiRowOp.First:
-               return col_vals.First();
-            case MultiRowOp.Last:
-               return col_vals.Last();
-            case MultiRowOp.StdDev:
-               return MathNet.Numerics.Statistics.Statistics.StandardDeviation(col_vals);
-            default:
-               // TODO
-               Log.assert(false);
-               return Double.NaN;
-         }
-      }
-
-   }
-}
@@ -1,207 +0,0 @@
-namespace bts.BlueOrb {
-   using BCG;
-
-   using ProtoBuf;
-
-   using System;
-   using System.Collections;
-   using System.Collections.Generic;
-   using System.Linq;
-
-   #region mappings
-   internal class ProjectData {
-      // schema from the base stream
-      public readonly IBlueTableSchema in_schema;
-      // this is the projected schema
-      public readonly IBlueTableSchema out_schema;
-      // projected cols to the input cols, for cols who's names are changed
-      public readonly IDictionary<string, string> mapped_to_orig_col_map = new Dictionary<string, string>();
-      public ProjectData(IBlueTableSchema in_schema_, IBlueTableSchema out_schema_,
-                  IDictionary<string, string> mapped_to_orig_col_map_) {
-         in_schema = in_schema_;
-         out_schema = out_schema_;
-         mapped_to_orig_col_map = mapped_to_orig_col_map_;
-      }
-   }
-   #endregion
-
-
-#region Row with mapipings build in
-   internal class NewProjectRow : IBlueRow {
-      #region IBlueRow members
-      private ProjectData _project_data;
-      private IBlueRow _row;
-      public NewProjectRow(ProjectData project_data, IBlueRow row) {
-         _project_data = project_data;
-         _row = row;
-      }
-      public IBlueTableSchema schema { get { return _project_data.out_schema; } }
-      #endregion
-
-      #region ICollection members
-      public int Count { get { return schema.columnNames.Length; } }
-
-      public bool IsSynchronized { get { return false; } }
-
-      public object SyncRoot { get { return this; } }
-
-      public void CopyTo(Array array, int index) {
-         int i = index;
-         foreach (object o in this as IEnumerable)
-            array.SetValue(o, i++);
-      }
-      #endregion
-
-      #region IDictionary members
-      public bool IsFixedSize { get { return true; } }
-
-      public bool IsReadOnly { get { return true; } }
-
-      public object this[object key] {
-         get {
-            if (key is string)
-               return this[(string)key];
-            return this[(int)key];
-         }
-         set { throw new NotSupportedException("The IBlueRow is readonly"); }
-      }
-
-      public object this[int index] {
-         get {
-            return this[schema.columnInfo[index].colName];
-         }
-      }
-
-      public object this[string col_name] {
-         get {
-            string mapped_to_col = null;
-            if (_project_data.mapped_to_orig_col_map.TryGetValue(col_name, out mapped_to_col)) {
-               return _row[mapped_to_col];
-            }
-            return _row[col_name];
-         }
-      }
-
-      public ICollection Keys { get { return schema.columnNames; } }
-
-      public ICollection Values {
-         get {
-            object[] retVal = new object[schema.columnNames.Length];
-            CopyTo(retVal, 0);
-            return retVal;
-         }
-      }
-
-      public void Add(object key, object value) { throw new NotSupportedException("The IBlueRow is readonly"); }
-
-      public void Clear() { throw new NotSupportedException("The IBlueRow is readonly"); }
-
-      public bool Contains(object key) { return schema[(string)key] != null; }
-
-      /// <summary>
-      /// Note, the default enumerator is the IDictionary's enumerator, so if user needs to
-      /// iterate through the values, typecast this into an IEnumerable
-      /// </summary>
-      /// <returns></returns>
-      public IDictionaryEnumerator GetEnumerator() { return new BlueRowDictionaryEnumerator(this); }
-
-      IEnumerator IEnumerable.GetEnumerator() { return new BlueRowEnumerator(this); }
-
-      public void Remove(object key) { throw new NotSupportedException("The IBlueRow is readonly"); }
-      #endregion
-   }
-   #endregion
-
-   [DataStreamUserRespresentation("ProjectStream")]
-   public class NewProjectStream : ViewStream, IStreamMapper {
-
-      private ViewStream _stream;
-      private string _project_expression;
-      private ProjectData _project_data = null;
-      //public ProjectData 
-      public NewProjectStream(ViewStream vs, string project_expression) {
-         _stream = vs;
-         _project_expression = project_expression;
-         _stream = new MapStream(vs, this);
-         mySetRealSource(_stream);
-      }
-
-      public IBlueTableSchema getOutputSchema(IBlueTableSchema input_schema) {
-         _project_data = _create_project_data(input_schema, _project_expression);
-         return _project_data.out_schema;
-      }
-
-      public IBlueRow map(IBlueRow row) {
-         return new NewProjectRow(_project_data, row);
-      }
-
-      private static ProjectData _create_project_data(IBlueTableSchema input_schema,
-                                                      string project_expression) {
-         project_expression = project_expression.Trim();
-         string[] project_toks = project_expression.Split(' ');
-         HashSet<string> removed_cols = new HashSet<string>();
-         Dictionary<string, string> mapped_to_orig_map = new Dictionary<string, string>();
-         Dictionary<string, string> orig_to_mapped = new Dictionary<string, string>();
-         // by default every col will show up, so if we have the projection as (a:a1)
-         // it will convert a to a1, 
-         // examples: if a schema has cols a b c
-         // project(parent_str, "a:aa ") -> aa b c
-         // project(parent_str, "a:aa !c") -> aa b
-         // project(parent_str, "a:aa !") -> aa
-         // project(parent_str, "a:aa b")) -> aa b
-         bool hide_all = false;
-         foreach (string p_tok in project_toks) {
-            if (p_tok.StartsWith("!")) {
-               if (p_tok.Length == 1) {
-                  // if it is a "blanket "!"=> hide all cols except the one that
-                  // are stated explicitly
-                  hide_all = true;
-               } else {
-                  string removed_col = p_tok.Substring(1);
-                  removed_cols.Add(removed_col);
-               }
-            } else if (p_tok.Contains(":")) {
-               string[] map_toks = p_tok.Split(":");
-               string orig_col = map_toks[0];
-               string mapped_col = map_toks[1];
-               mapped_to_orig_map.Add(mapped_col, orig_col);
-               orig_to_mapped.Add(orig_col, mapped_col);
-            } else {
-               if (input_schema.columnNames.Contains(p_tok)) {
-                  string orig_col = p_tok;
-                  string mapped_col = orig_col;
-                  orig_to_mapped.Add(orig_col, mapped_col);
-               }
-            }
-         }
-
-         // make the new schema
-         List<ColumnInfo> new_cols = new List<ColumnInfo>();
-         foreach (ColumnInfo ci in input_schema.columnInfo) {
-            if (ci.colName == "id") {
-               ColumnInfo nc = new ColumnInfo(ci.colName, new_cols.Count, ci.colType);
-               new_cols.Add(nc);
-               continue;
-            }
-            if (removed_cols.Contains(ci.colName)) {
-               continue;
-            } else if (orig_to_mapped.ContainsKey(ci.colName)) {
-               string mapped_name = orig_to_mapped[ci.colName];
-               ColumnInfo nc = new ColumnInfo(mapped_name, new_cols.Count, ci.colType);
-               new_cols.Add(nc);
-            } else {
-               if (hide_all)
-                  continue;
-               //if (ci.colName == "id" || removed_cols.Count > 0) {
-               // unless columns are  explicitely hidden, all columns show up
-               ColumnInfo nc = new ColumnInfo(ci.colName, new_cols.Count, ci.colType);
-               new_cols.Add(nc);
-               //}
-            }
-         }
-         IBlueTableSchema new_schema = new BlueTableSchema(input_schema.tableName, new_cols.ToArray());
-         return new ProjectData(input_schema, new_schema, mapped_to_orig_map);
-      }
-
-   }
-}
@@ -1,30 +0,0 @@
-using BCG;
-
-namespace bts.BlueOrb {
-   public class SchemaGuesserStream : ViewStream {
-
-      private IBlueRowCollection _rows = null;
-      private ViewStream _vs;
-      public SchemaGuesserStream(ViewStream vs) {
-         _vs = vs;
-      }
-
-      protected override void myOnFirstSubscriber() {
-         _vs.onChange += _on_data;
-      }
-      protected override void myOnLastSubscriber() {
-         _vs.onChange -= _on_data;
-      }
-
-      private void _on_data(object obj) {
-         DataPacket dp = obj as DataPacket;
-         if (_rows == null) {
-            _rows = BlueDataUtility.get_typed_table(dp.newRows);
-         }
-         var dp_out = BlueDataUtility.build_dp_inc(_rows, myLastValue as DataPacket);
-         myFireEvent(dp_out);
-
-      }
-
-   }
-}
@@ -1,67 +0,0 @@
-using Bts.ArithmeticExpressions;
-
-using System;
-using System.Collections.Generic;
-using System.Linq;
-using System.Text;
-using BCG;
-
-namespace bts.BlueOrb {
-
-   [DataStreamUserRespresentation("SingleRowMathStream")]
-   public class MathStream : ViewStream, IStreamMapper {
-      private IArithmeticProgram myExp;
-      private BooleanExpressionFilter _bool;
-      List<string> myExtraColumns = new List<string>();
-      BlueTableSchema mySchema;
-
-
-      public MathStream(ViewStream vs, string mathExpression)
-         : this(vs, mathExpression, string.Empty) {
-      }
-
-      public MathStream(ViewStream vs, string mathExpression, string boolean_expression) {
-         myExp = ArithmeticProgramParser.Parse(mathExpression) ??
-            throw new Exception($"Failed to parse ArithmeticExpression '{mathExpression}'");
-         myExtraColumns = myExp.OutputIdentifiers.ToList();
-         if (!string.IsNullOrEmpty(boolean_expression))
-            _bool = new BooleanExpressionFilter(boolean_expression);
-         mySetRealSource(new MapStream(vs, this));
-      }
-
-      #region IStreamMapper Members
-
-      public IBlueTableSchema getOutputSchema(IBlueTableSchema inputSchema) {
-         if (mySchema == null) {
-            Dictionary<string, ColumnInfo> colIdx = new Dictionary<string, ColumnInfo>();
-            foreach (ColumnInfo ci in inputSchema.columnInfo)
-               colIdx[ci.colName] = ci;
-
-            foreach (string ncn in myExtraColumns) {
-               ColumnInfo newC;
-               if (colIdx.ContainsKey(ncn))
-                  newC = new ColumnInfo(colIdx[ncn].colName, colIdx[ncn].colNo, typeof(double));
-               else
-                  newC = new ColumnInfo(ncn, colIdx.Count, typeof(double));
-               colIdx[ncn] = newC;
-            }
-            mySchema = new BlueTableSchema(colIdx.Values.ToArray(), "math(" + inputSchema.tableName + ")");
-         }
-         return mySchema;
-      }
-
-      #endregion
-
-      #region IDynamicStreamMapper Members
-
-      public IBlueRow map(IBlueRow row) {
-         MutableBlueRow result = new MutableBlueRow(mySchema, row);
-         if (_bool == null || _bool.filter(row)) {
-            myExp.Evaluate(result.AsDictionary());
-         }
-         return result;
-      }
-
-      #endregion
-   }
-}
@@ -1,11 +0,0 @@
-namespace bts.BlueOrb {
-   using BCG;
-   [DataStreamUserRespresentation("TopStream")]
-   public class TopStream : ViewStream { 
-      public TopStream(ViewStream vs, int n_rows, string sort_col) {
-         ViewStream rs = new BCG.TopStream (vs, n_rows, sort_col);
-         mySetRealSource(rs);
-      }
-   }
-
-}
@@ -1,261 +0,0 @@
-using System;
-using System.Collections.Generic;
-
-using mbts;
-using BCG;
-
-namespace bts.BlueOrb {
-   internal class MutableAppendTable : IBlueRowCollection {
-      private class MyRow : IBlueRow {
-         private IBlueRow myRow;
-         private int myID;
-
-         public MyRow(IBlueRow r, int id) {
-            myRow = r;
-            myID = id;
-         }
-
-         #region IBlueRow Members
-
-         public IBlueTableSchema schema { get { return myRow.schema; } }
-
-         public object this[int index] {
-            get {
-               if (myRow.schema[index].colName == "id")
-                  return myID;
-               else
-                  return myRow[index];
-            }
-         }
-
-         public object this[string colName] { get { return colName == "id" ? (object)myID : myRow[colName]; } }
-
-         #endregion
-
-         #region IDictionary Members
-
-         public void Add(object key, object value) { throw new Exception("Row is read only."); }
-
-         public void Clear() { throw new Exception("Row is read only."); }
-
-         public bool Contains(object key) { return myRow.Contains(key); }
-
-         public System.Collections.IDictionaryEnumerator GetEnumerator() { return new BlueRowDictionaryEnumerator(this); }
-
-         public bool IsFixedSize { get { return true; } }
-
-         public bool IsReadOnly { get { return true; } }
-
-         public System.Collections.ICollection Keys { get { return myRow.Keys; } }
-
-         public void Remove(object key) { throw new Exception("The method or operation is not implemented."); }
-
-         public System.Collections.ICollection Values {
-            get {
-               object[] vals = new object[myRow.Count];
-               CopyTo(vals, 0);
-               return vals;
-            }
-         }
-
-         public object this[object key] {
-            get { return key is string ? this[key as string] : this[(int)key]; }
-            set { throw new Exception("The method or operation is not implemented."); }
-         }
-
-         #endregion
-
-         #region ICollection Members
-
-         public void CopyTo(Array array, int index) {
-            myRow.CopyTo(array, index);
-            array.SetValue(myID, index + myRow.schema["id"].colNo);
-         }
-
-         public int Count { get { return myRow.Count; } }
-
-         public bool IsSynchronized { get { return myRow.IsSynchronized; } }
-
-         public object SyncRoot { get { return myRow.SyncRoot; } }
-
-         #endregion
-
-         #region IEnumerable Members
-
-         System.Collections.IEnumerator System.Collections.IEnumerable.GetEnumerator() { return new BlueRowEnumerator(this); }
-
-         #endregion
-      }
-
-      private BlueTable myRows;
-      private int myID = 0;
-      /// <summary>
-      /// maps the table no to a dictionary containing the
-      /// local IDs to a row in the final big table
-      /// </summary>
-      private Dictionary<int, Dictionary<int, IBlueRow>> myIDMap = new Dictionary<int, Dictionary<int, IBlueRow>>();
-
-      public MutableAppendTable(IBlueTableSchema schema) { myRows = new BlueTable(schema); }
-
-      public IBlueRow addRow(IBlueRow r, int tableNo) {
-         if (!myIDMap.ContainsKey(tableNo))
-            myIDMap.Add(tableNo, new Dictionary<int, IBlueRow>());
-         IBlueRow nr = new MyRow(r, myID++); // BlueDataUtility.getNewReplacedRow(r, myID++, "id");
-         myRows.addRow(nr);
-         myIDMap[tableNo].Add((int)r["id"], nr);
-         return nr;
-      }
-
-      public IBlueRow removeRow(IBlueRow r, int tableNo) {
-         IBlueRow dr = myIDMap[tableNo][(int)r["id"]];
-         myRows.deleteRow(dr);
-         myIDMap[tableNo].Remove((int)r["id"]);
-         return dr;
-      }
-
-      public IBlueRowCollection removeTable(int tableNo) {
-         if (!myIDMap.ContainsKey(tableNo))
-            return null;
-         else {
-            BlueTable rv = new BlueTable(myRows.schema);
-            foreach (IBlueRow r in myIDMap[tableNo].Values)
-               rv.addRow(r);
-            myRows.deleteRows(rv);
-            myIDMap.Remove(tableNo);
-            return rv;
-         }
-      }
-
-      public void clear() {
-         myRows.clear();
-         myIDMap.Clear();
-      }
-
-      #region IBlueRowCollection Members
-
-      public IBlueRow this[int id] { get { return myRows[id]; } }
-
-      public IBlueTableSchema schema { get { return myRows.schema; } }
-
-      public bool containsID(int id) { return myRows.containsID(id); }
-
-      #endregion
-
-      #region ICollection Members
-
-      public void CopyTo(Array array, int index) { myRows.CopyTo(array, index); }
-
-      public int Count { get { return myRows.Count; } }
-
-      public bool IsSynchronized { get { return myRows.IsSynchronized; } }
-
-      public object SyncRoot { get { return myRows.SyncRoot; } }
-
-      #endregion
-
-      #region IEnumerable Members
-
-      public System.Collections.IEnumerator GetEnumerator() { return myRows.GetEnumerator(); }
-
-      #endregion
-   }
-   /// <summary>
-   /// MutableStream can append multiple streams
-   /// as well as give the user the flexibility
-   /// to append (or remove) more streams at
-   /// runtime
-   /// </summary>
-   }
-namespace bts.BlueOrb { 
-   [DataStreamUserRespresentation("UnionStream")]
-   public class MutableAppendStream : ViewStream {
-      private bool mySubscriptionActive = false;
-      private MutableAppendTable myTable;
-      private int myMovingIndex;
-      private Dictionary<int, Pair<ViewStream, ViewStream>> myStreamIndex = new Dictionary<int, Pair<ViewStream, ViewStream>>();
-      private bool _allign_col_orders = true;
-
-      public MutableAppendStream(params ViewStream[] streams)
-         : this(new List<ViewStream>(streams)) { }
-
-      public MutableAppendStream(bool allign_col_order, params ViewStream[] streams)
-         : this(allign_col_order, new List<ViewStream>(streams)) { }
-
-      public MutableAppendStream(ICollection<ViewStream> streams)
-         : this(true, streams) { }
-
-      public MutableAppendStream(bool allign_col_orders, ICollection<ViewStream> streams) {
-         _allign_col_orders = allign_col_orders;
-         foreach (ViewStream vs in streams) {
-            myMovingIndex++;
-            ViewStream pj = new ChangeTableNameStream(vs, myMovingIndex.ToString());
-            if (_allign_col_orders)
-               pj = new ProjectStream(pj, "!");
-            myStreamIndex.Add(myMovingIndex, new Pair<ViewStream, ViewStream>(vs, pj));
-         }
-      }
-
-      public void addStream(ViewStream vs) {
-         myMovingIndex++;
-         ///the change name is done to identify the callbacks, and the project to
-         ///make all the schemas col orders the same
-         ViewStream pj = new ChangeTableNameStream(vs, myMovingIndex.ToString());
-         if (_allign_col_orders)
-            pj = new ProjectStream(pj, "!");
-         myStreamIndex.Add(myMovingIndex, new Pair<ViewStream, ViewStream>(vs, pj));
-         if (mySubscriptionActive)
-            pj.onChange += new ChangeDelegate(myHandleStreamChanges);
-      }
-
-      public void removeStream(ViewStream vs) {
-         int index = -1;
-         foreach (KeyValuePair<int, Pair<ViewStream, ViewStream>> p in myStreamIndex) {
-            if (p.Value.first == vs) {
-               index = p.Key;
-               if (mySubscriptionActive) {
-                  p.Value.second.onChange -= new ChangeDelegate(myHandleStreamChanges);
-                  if (myTable != null) {
-                     IBlueRowCollection delRows = myTable.removeTable(p.Key);
-                     if (delRows != null)
-                        myFireEvent(new DataPacket(myTable, new BlueTable(myTable.schema), delRows));
-                  }
-               }
-               break;
-            }
-            if (index > 0)
-               myStreamIndex.Remove(index);
-         }
-      }
-
-      protected override void myOnFirstSubscriber() {
-         mySubscriptionActive = true;
-         foreach (KeyValuePair<int, Pair<ViewStream, ViewStream>> p in myStreamIndex)
-            p.Value.second.onChange += new ChangeDelegate(myHandleStreamChanges);
-      }
-
-      protected override void myOnLastSubscriber() {
-         mySubscriptionActive = false;
-         foreach (KeyValuePair<int, Pair<ViewStream, ViewStream>> p in myStreamIndex)
-            p.Value.second.onChange -= new ChangeDelegate(myHandleStreamChanges);
-         if (myTable != null)
-            myTable.clear();
-      }
-
-      private void myHandleStreamChanges(object newValue) {
-         DataPacket dp = newValue as DataPacket;
-         myTable = myTable ?? new MutableAppendTable(new BlueTableSchema("MutableAppend", dp.schema.columnInfo));
-         BlueTable adds = new BlueTable(myTable.schema),
-                     dels = new BlueTable(myTable.schema);
-         int index = Int32.Parse(dp.schema.tableName);
-         foreach (IBlueRow r in dp.deletedRows) {
-            IBlueRow dr = myTable.removeRow(r, index);
-            dels.addRow(dr);
-         }
-         foreach (IBlueRow r in dp.addedRows) {
-            IBlueRow nr = myTable.addRow(r, index);
-            adds.addRow(nr);
-         }
-         myFireEvent(new DataPacket(myTable, adds, dels));
-      }
-   }
-}
@@ -1,7 +1,7 @@
 <Project Sdk="Microsoft.NET.Sdk">

  <PropertyGroup>
-    <TargetFramework>net8.0</TargetFramework>
+    <TargetFramework>net7.0</TargetFramework>
    <Nullable>enable</Nullable>
  </PropertyGroup>

@@ -18,13 +18,11 @@

  <ItemGroup>
    <PackageReference Include="HtmlAgilityPack" Version="1.11.57" />
-    <PackageReference Include="IronPython" Version="3.4.2" />
    <PackageReference Include="Microsoft.Data.Analysis" Version="0.21.1" />
    <PackageReference Include="MongoDB.Bson" Version="2.23.1" />
    <PackageReference Include="MongoDB.Driver" Version="2.23.1" />
    <PackageReference Include="Parquet.Net" Version="5.0.1" />
    <PackageReference Include="System.Data.SQLite" Version="1.0.118" />
-    <PackageReference Include="MathNet.Numerics" Version="5.0.0" />
  </ItemGroup>

 </Project>
@@ -1,47 +1,52 @@
 using System;

-namespace bts.BlueOrb {
-   using BCG;
-   [DataStreamUserRespresentation("CountStream")]
-   public class CountStream : ViewStream {
+namespace BCG
+{
+   [DataStreamUserRespresentation ("count")]
+	public class CountStream : ViewStream
+	{
      private DataPacket myLastFiredDataPacket;
-      private IBlueTableSchema mySchema = new BlueTableSchema("count",
-         new ColumnInfo("id", 0, typeof(int)),
-         new ColumnInfo("count", 1, typeof(int)));
+      private IBlueTableSchema mySchema = new BlueTableSchema ("count", 
+         new ColumnInfo ("id", 0, typeof(int)),
+         new ColumnInfo ("count", 1, typeof(int)));
      private int myId = 0;
      private ViewStream myVS;

-      public CountStream(ViewStream vs) {
+      public CountStream (ViewStream vs)
+      {
         myVS = vs;
      }

-      public CountStream(string tableName)
-         : this(new TableStream(tableName)) { }
+      public CountStream (string tableName)
+         : this (new TableStream (tableName))
+      {}

-      protected override void myOnFirstSubscriber() {
+      protected override void myOnFirstSubscriber () {
         myVS.onChange += new ChangeDelegate(myHandleChanges);
      }
-      protected override void myOnLastSubscriber() {
+      protected override void myOnLastSubscriber () {
         myVS.onChange -= new ChangeDelegate(myHandleChanges);
      }


-      private void myHandleChanges(object newValue) {
-         myFireEvent(
+      private void myHandleChanges(object newValue)
+      {
+         myFireEvent (
            myLastFiredDataPacket =
               myLastFiredDataPacket == null ?
-                  new DataPacket(myGetRows(newValue, ++myId), myGetRows(newValue, myId),
-                  new BlueTable(mySchema))
+                  new DataPacket (myGetRows (newValue, ++myId), myGetRows (newValue, myId), 
+                  new BlueTable (mySchema))
                  :
-                  new DataPacket(myGetRows(newValue, ++myId), myGetRows(newValue, myId),
+                  new DataPacket (myGetRows (newValue, ++myId), myGetRows (newValue, myId), 
                  myLastFiredDataPacket.newRows)
            );
      }

-      private IBlueRowCollection myGetRows(object dp, int id) {
-         BlueTable retVal = new BlueTable(mySchema);
+      private IBlueRowCollection myGetRows (object dp, int id)
+      {
+         BlueTable retVal = new BlueTable (mySchema);
         retVal.addRow
-            (new BlueRow(mySchema, id, (dp as DataPacket).newRows.Count));
+            (new BlueRow (mySchema, id, (dp as DataPacket).newRows.Count));
         return retVal;
      }

@@ -5,20 +5,25 @@ using mbts.config;
 using System;
 using System.Text.RegularExpressions;

-namespace BCG.StreamWorld.Addition {
-   [DataStreamUserRespresentation(false)]
-   public class FilterStream : ViewStream {
+namespace BCG.StreamWorld.Addition
+{
+   [DataStreamUserRespresentation (false)]
+   public class FilterStream : ViewStream
+   {
      private ViewStream myBaseStream;
      private BlueTable myBaseTable, myStateTable;
      private IStreamFilter myFilter;

      public FilterStream(ViewStream vs)
-         : this(vs, "1=1") { }
+         : this(vs, "1=1")
+      { }

      public FilterStream(ViewStream vs, string filterExp)
-         : this(vs, new BooleanExpressionFilter(filterExp)) { }
+         : this(vs, new BooleanExpressionFilter(filterExp))
+      { }

-      public FilterStream(ViewStream vs, IStreamFilter filter) {
+      public FilterStream(ViewStream vs, IStreamFilter filter)
+      {
         myBaseStream = vs;
         myFilter = filter;
         if (filter is IDynamicStreamFilter)
@@ -26,35 +31,43 @@ namespace BCG.StreamWorld.Addition {
      }


-      protected override void myOnFirstSubscriber() {
+      protected override void myOnFirstSubscriber()
+      {
         myBaseStream.onChange += new ChangeDelegate(myHandleChanges);
      }

-      protected override void myOnLastSubscriber() {
+      protected override void myOnLastSubscriber()
+      {
         myBaseStream.onChange -= new ChangeDelegate(myHandleChanges);
         myBaseTable.clear();
         myStateTable.clear();
      }

-      private void myHandleChanges(object newValue) {
+      private void myHandleChanges(object newValue)
+      {
         DataPacket dp = newValue as DataPacket;
-         if (myStateTable == null) {
+         if (myStateTable == null)
+         {
            bts.Log.assert(myBaseTable == null);
            myStateTable = new BlueTable(dp.schema);
            myBaseTable = new BlueTable(dp.schema);
         }
         BlueTable dels = new BlueTable(dp.schema),
                     adds = new BlueTable(dp.schema);
-         foreach (IBlueRow r in dp.deletedRows) {
+         foreach (IBlueRow r in dp.deletedRows)
+         {
            myBaseTable.deleteRow(r);
-            if (myStateTable.containsID((int)r["id"])) {
+            if (myStateTable.containsID((int)r["id"]))
+            {
               dels.addRow(r);
               myStateTable.deleteRow(r);
            }
         }
-         foreach (IBlueRow r in dp.addedRows) {
+         foreach (IBlueRow r in dp.addedRows)
+         {
            myBaseTable.addRow(r);
-            if (myFilter.filter(r)) {
+            if (myFilter.filter(r))
+            {
               adds.addRow(r);
               myStateTable.addRow(r);
            }
@@ -63,10 +76,13 @@ namespace BCG.StreamWorld.Addition {
            myFireEvent(new DataPacket(myStateTable, adds, dels));
      }

-      public IStreamFilter filter {
-         set {
+      public IStreamFilter filter
+      {
+         set
+         {
            myFilter = value;
-            if (myBaseTable != null) {
+            if (myBaseTable != null)
+            {
               IBlueRowCollection oldState = new BlueTable(myStateTable);
               myStateTable.clear();
               foreach (IBlueRow r in myBaseTable)
@@ -4,46 +4,42 @@ namespace bts.SilverPine {
   using System;
   using System.Collections.Generic;
   using System.Linq;
-   //using MathNet.Numerics.Statistics;

+   using ZstdSharp.Unsafe;

-   public enum MultiRowOp {
+   public enum GroupByBundleOp {
      Mean,
      Median,
      Mode,
-      Range,
      StdDev,
      Max,
      Min,
      First,
      Last,
-      Sum,
-      EdgeDiff,
-      EdgeDiffPercentage,
-      //IsEdgeDiff,
+      Sum
   }
-   public class MultiRowOpHelper {
-      public readonly List<Tuple<string, MultiRowOp>> operations;
+   public class GroupByBundle {
+      public readonly List<Tuple<string, GroupByBundleOp>> operations;

-      public static MultiRowOpHelper parse(string text) {
+      public static GroupByBundle parse(string text) {
         // text is inp_col: op, inp_col: op, inp_col:op
-         List<Tuple<string, MultiRowOp>> rv = new List<Tuple<string, MultiRowOp>>();
+         List<Tuple<string, GroupByBundleOp>> rv = new List<Tuple<string, GroupByBundleOp>>();
         string[] calcs = text.Split(',');
         foreach (string c in calcs) {
            string[] toks = c.Split(':');
            string inp_col = toks[0].Trim();
            string op_txt = toks[1].Trim();
-            MultiRowOp op = Enum.Parse<MultiRowOp>(op_txt, true);
+            GroupByBundleOp op = Enum.Parse<GroupByBundleOp>(op_txt, true);
            rv.Add(Tuple.Create(inp_col, op));
         }
-         return new MultiRowOpHelper(rv);
+         return new GroupByBundle(rv);
      }

-      public MultiRowOpHelper(List<Tuple<string, MultiRowOp>> operations_) {
+      public GroupByBundle(List<Tuple<string, GroupByBundleOp>> operations_) {
         operations = operations_;
      }

-      public static IBlueRow group_by(List<IBlueRow> rows, MultiRowOpHelper parameters) {
+      public static IBlueRow group_by(List<IBlueRow> rows, GroupByBundle parameters) {
         if (rows.Count() == 0) {
            return null;
         }
@@ -51,8 +47,8 @@ namespace bts.SilverPine {
         IBlueRow ir = rows.ElementAt(0);
         IBlueTableSchema in_schema = ir.schema;
         IBlueTableSchema out_schema = in_schema;
-         IDictionary<string, Tuple<string, MultiRowOp>> out_col_map =
-            new Dictionary<string, Tuple<string, MultiRowOp>>();
+         IDictionary<string, Tuple<string, GroupByBundleOp>> out_col_map =
+            new Dictionary<string, Tuple<string, GroupByBundleOp>>();
         foreach (var t in parameters.operations) {
            string out_col_name = $"{t.Item1}_{t.Item2}".ToLower();
            out_schema = out_schema.add_column(TypeCode.Double, out_col_name);
@@ -62,7 +58,7 @@ namespace bts.SilverPine {
         foreach (var kv in out_col_map) {
            string out_col_name = kv.Key;
            string in_col_name = kv.Value.Item1;
-            MultiRowOp op = kv.Value.Item2;
+            GroupByBundleOp op = kv.Value.Item2;
            IEnumerable<double> col_vals = BlueDataUtility.get_colvalues<double>(rows, in_col_name);
            double comp_res = compute_op(col_vals, op);
            or.setValue(out_col_name, comp_res);
@@ -70,51 +66,20 @@ namespace bts.SilverPine {
         return or;
      }

-      // rolling_rows are the rolling window of rows. the 0th row is the one thats gettin
-      // calculated. rows 1 - n-1 are its window rows. the schema is based on the operations.
-
-      public IBlueRow rolling_calc(List<IBlueRow> rolling_rows) {
-         if (rolling_rows.Count() == 0) {
-            return null;
-         }
-         IBlueRow ir = rolling_rows.ElementAt(0);
-         IBlueTableSchema in_schema = ir.schema;
-         IBlueTableSchema out_schema = in_schema;
-         IDictionary<string, Tuple<string, MultiRowOp>> out_col_map =
-            new Dictionary<string, Tuple<string, MultiRowOp>>();
-         foreach (var t in operations) {
-            string out_col_name = $"{t.Item1}_{t.Item2}".ToLower();
-            out_schema = out_schema.add_column(TypeCode.Double, out_col_name);
-            out_col_map[out_col_name] = t;
-         }
-         MutableBlueRow or = new MutableBlueRow(out_schema, ir);
-         foreach (var kv in out_col_map) {
-            string out_col_name = kv.Key;
-            string in_col_name = kv.Value.Item1;
-            MultiRowOp op = kv.Value.Item2;
-            IEnumerable<double> col_vals = BlueDataUtility.get_colvalues<double>(rolling_rows, in_col_name);
-            double comp_res = compute_op(col_vals, op);
-            or.setValue(out_col_name, comp_res);
-         }
-         return or;
-      }
-
-      public static double compute_op(IEnumerable<double> col_vals, MultiRowOp op) {
+      public static double compute_op(IEnumerable<double> col_vals, GroupByBundleOp op) {
         switch (op) {
-            case MultiRowOp.Max:
+            case GroupByBundleOp.Max:
               return col_vals.Max<double>();
-            case MultiRowOp.Min:
+            case GroupByBundleOp.Min:
               return col_vals.Min<double>();
-            case MultiRowOp.Mean:
+            case GroupByBundleOp.Mean:
               return col_vals.Sum() / col_vals.Count();
-            case MultiRowOp.Sum:
+            case GroupByBundleOp.Sum:
               return col_vals.Sum();
-            case MultiRowOp.First:
+            case GroupByBundleOp.First:
               return col_vals.First();
-            case MultiRowOp.Last:
+            case GroupByBundleOp.Last:
               return col_vals.Last();
-            case MultiRowOp.StdDev:
-               return MathNet.Numerics.Statistics.Statistics.StandardDeviation(col_vals);
            default:
               // TODO
               Log.assert(false);
@@ -5,7 +5,7 @@ using System.Collections.Generic;
 using System.Linq;

 namespace BCG {
-   internal class GroupByStream : MethodDrivenStream {
+   public class GroupByStream : MethodDrivenStream {
      private class MyFuncGroupByEvaluator : IGroupByEvaluator {
         private string[] _groupby_cols;
         private Func<DataPacket, IBlueRow> _groupby_func;
@@ -1,20 +0,0 @@
-namespace bts.SilverPine {
-   using BCG;
-
-   using System;
-
-   [DataStreamUserRespresentation("AggregationStream")]
-   public class AgregationStream : ViewStream {
-
-      public AgregationStream(ViewStream parent_str, string operation_txt) {
-         ViewStream rs = new NonIncGroupByStream(parent_str, String.Empty, operation_txt);
-         mySetRealSource(rs);
-      }
-
-      public AgregationStream(ViewStream parent_str, string groupby_col, string operation_txt) {
-         ViewStream rs = new NonIncGroupByStream(parent_str, groupby_col, operation_txt);
-         mySetRealSource(rs);
-      }
-   }
-
-}
@@ -8,7 +8,7 @@ using bts;
 using bts.SilverPine;

 namespace BCG {
-   [DataStreamUserRespresentation("AggregateStream")]
+   [DataStreamUserRespresentation("aggregate")]
   public class NonIncGroupByStream : ViewStream {
      private class DelNonIncGroupByEvaluator : INonIncGroupByEvaluator {
         Func<string[]> _groupby_cols_func;
@@ -53,18 +53,15 @@ namespace BCG {
                                 Func<List<IBlueRow>, List<IBlueRow>> evaluate_func)
                                 : this(main_stream, new DelNonIncGroupByEvaluator(groupby_cols_func, evaluate_func)) { }

-      public NonIncGroupByStream(ViewStream parent_str, string groupby_col, string standard_group_by_text)
-      : this(parent_str, groupby_col, standard_group_by_text, String.Empty) { }
-
-      public NonIncGroupByStream(ViewStream parent_str, string groupby_col, string standard_group_by_text, string optional_sort_col) {
-         MultiRowOpHelper std_group_by = MultiRowOpHelper.parse(standard_group_by_text);
+      public NonIncGroupByStream(ViewStream parent_str, string groupby_col, string standard_group_by_text, string optional_sort_col = null) {
+         GroupByBundle std_group_by = GroupByBundle.parse(standard_group_by_text);
         Func<string[]> groupby_cols_func = () => {
            return String.IsNullOrEmpty(groupby_col) ? null : new string[] { groupby_col };
         };
         Func<List<IBlueRow>, List<IBlueRow>> evaluate_func = (group_rows) => {
            if (!String.IsNullOrEmpty(optional_sort_col))
               group_rows.Sort((r1, r2) => BlueDataUtility.compare_rows(r2, r1, optional_sort_col));
-            IBlueRow rv = MultiRowOpHelper.group_by(group_rows, std_group_by);
+            IBlueRow rv = GroupByBundle.group_by(group_rows, std_group_by);
            return new IBlueRow[] { rv }.ToList();
         };
         ViewStream rs = new NonIncGroupByStream(parent_str, groupby_cols_func, evaluate_func);
@@ -74,7 +74,7 @@ namespace BCG {
      }

      public void _on_stream_cb(object dp_obj) {
-         //Log.Gen.info($"NonIncJoinBookkeeper._on_stream_cb dp_obj: {dp_obj}");
+         Log.Gen.info($"NonIncJoinBookkeeper._on_stream_cb dp_obj: {dp_obj}");
         var timer = Stopwatch.StartNew();
         DataPacket dp = (DataPacket)dp_obj;
         _fill_missing_data(dp);
@@ -50,7 +50,7 @@ namespace BCG {
         int data_col_no = 1;
         for (int jp = 0; jp < _join_specs.Length; jp++) {
            NonIncJoinBookkeeper s = new NonIncJoinBookkeeper(_join_specs[jp], jp, data_col_no, _row_hasher);
-            //Log.Gen.info($"NonIncJoinStream.myOnFirstSubscriber jp: {jp} spec: {_join_specs[jp]}");
+            Log.Gen.info($"NonIncJoinStream.myOnFirstSubscriber jp: {jp} spec: {_join_specs[jp]}");
            data_col_no += _join_specs[jp].data_col_names.Length;
            s.on_new_data += _stream_callback;
            _book_keeping_data.Add(s);
@@ -71,7 +71,7 @@ namespace BCG {
      }

      private void _stream_callback() {
-         //Log.Gen.info("NonIncJoinStream._stream_callback received callback");
+         Log.Gen.info("NonIncJoinStream._stream_callback received callback");
         _is_dirty = true;
         doWork();
      }
@@ -99,7 +99,7 @@ namespace BCG {

      private int _n_total_fires = 0, _n_expensive_fires = 0;
      private void _form_and_fire() {
-         //Log.Gen.info($"NonIncJoinStream._form_and_fire _is_ready(): {_is_ready()}");
+         Log.Gen.info($"NonIncJoinStream._form_and_fire _is_ready(): {_is_ready()}");
         if (_is_ready()) {
            _construct_schema();
            var timer = Stopwatch.StartNew();
@@ -126,11 +126,11 @@ namespace BCG {
            _n_total_fires++;
            //if (timer.ElapsedMilliseconds >= 250) {
            _n_expensive_fires++;
-            //string log_msg = $"Join and map time for {_book_keeping_data.Count} "
-            //   + $"tables {timer.Elapsed.TotalMilliseconds} msecs, ratio {_n_expensive_fires}/{_n_total_fires}";
-            //Log.Gen.info(log_msg);
+            string log_msg = $"Join and map time for {_book_keeping_data.Count} "
+               + $"tables {timer.Elapsed.TotalMilliseconds} msecs, ratio {_n_expensive_fires}/{_n_total_fires}";
+            Log.Gen.info(log_msg);
            //}
-            var dp_out = BlueDataUtility.build_dp_inc(dp_output_rows, myLastValue as DataPacket, true);
+            var dp_out = BlueDataUtility.build_dp_inc(dp_output_rows, myLastValue as DataPacket);
            myFireEvent(dp_out);
            //myFireEvent(new DataPacket(dp_output_rows, dp_output_rows, myLastValue == null ? new BlueTable(_output_schema) : (myLastValue as DataPacket).newRows));
         }
@@ -152,7 +152,7 @@ namespace BCG {
      }

      public void doWork() {
-         //Log.Gen.info($"NonIncJoinStream.doWork _last_fire_time: {_last_fire_time} _is_dirty: {_is_dirty}");
+         Log.Gen.info($"NonIncJoinStream.doWork _last_fire_time: {_last_fire_time} _is_dirty: {_is_dirty}");
         if (DateTime.Now - _last_fire_time >= _throttle_period && _is_dirty) {
            _is_dirty = false;
            _last_fire_time = DateTime.Now;
@@ -69,6 +69,4 @@ namespace bts {
         myFireEvent(dp_out);
      }
   }
-
-
 }
@@ -13,8 +13,9 @@ namespace BCG {
   /// desired row ie newRow = f(row), where f is the mapping fucntion. Note that the id 
   /// is preserved across the rows, ie row["id"] = f(row)["id"]
   /// </summary>
-   [DataStreamUserRespresentation(false)]
-   public class MapStream : MethodDrivenStream {
+   [DataStreamUserRespresentation (false)]
+   public class MapStream : MethodDrivenStream
+   {
      protected IStreamMapper myMapper;
      protected IChangeableStreamMapper myChangeableMapper = null;
      private bool myUseNewMap = Configuration.get<DataStreamConfig>().UseNewMap;
@@ -1,14 +0,0 @@
-namespace bts.SilverPine
-{
-   using BCG;
-
-   [DataStreamUserRespresentation("RollingComputationStream")]
-   public class RollingComputationStream : ViewStream
-   {
-      private ViewStream _parent_str;
-      public RollingComputationStream(ViewStream parent_str, string operation_txt, string roll_column, uint window_sz)
-      {
-
-      }
-   }
-}
@@ -1,10 +0,0 @@
-namespace bts.SilverPine {
-   using BCG;
-
-
-   [DataStreamUserRespresentation("RollingComputationStream")]
-   public class RollingComputationStream : ViewStream {
-      public RollingComputationStream(ViewStream parent_str, string operation_txt, string roll_column, uint window_sz) {
-      }
-   }
-}
@@ -108,7 +108,6 @@ namespace bts {

   }

-   [DataStreamUserRespresentation("Project")]
   public class NewProjectStream : ViewStream, IStreamMapper {

      private ViewStream _stream;
@@ -37,42 +37,4 @@ namespace BCG
      }
         
   }
-
-   [DataStreamUserRespresentation("throttle")]
-   public class ThrottleStream : ViewStream{
-      private readonly ViewStream _main_stream;
-      private readonly TimeSpan _ts;
-      private readonly ViewStreamDatabase _vsd;
-      private DateTime _last_fire_time = DateTime.MinValue;
-
-      public ThrottleStream (ViewStream main_stream, int throttle_seconds)
-      :
-      this(main_stream, TimeSpan.FromSeconds(throttle_seconds), ViewStreamDatabase.instance)
-      {
-      }
-
-      public ThrottleStream (ViewStream main_stream, TimeSpan ts, ViewStreamDatabase vsd){
-         _main_stream = main_stream;
-         _ts = ts;
-         _vsd = vsd;
-      }
-
-       protected override void myOnFirstSubscriber() {
-         _main_stream.onChange += _handle_base_stream_update;
-      }
-
-      protected override void myOnLastSubscriber() {
-         _main_stream.onChange -= _handle_base_stream_update;
-      }
-
-      private void _handle_base_stream_update(object dp_obj){
-         if (DateTime.Now - _last_fire_time >= _ts){
-            DataPacket dp_in = dp_obj as DataPacket;
-            _last_fire_time = DateTime.Now;
-            DataPacket dp = BlueDataUtility.build_dp_inc(dp_in.newRows, myLastValue as DataPacket);
-            myFireEvent(dp);
-         }
-      }
- }
-
 }
@@ -1,40 +0,0 @@
-namespace bts.SilverPine{
-   using BGC;
-   using System;
-   [DataStreamUserRespresentation("throttle")]
-   public class ThrottleStream : ViewStream{
-      private readonly ViewStream _main_stream;
-      private readonly TimeSpan _ts;
-      private readonly ViewStreamDatabase _vsd;
-      private DateTime _last_fire_time = DateTime.MinValue;
-
-      public ThrottleStream (ViewStream main_stream, int throttle_seconds)
-      :
-      this(main_stream, TimeSpan.FromSeconds(throttle_seconds), ViewStreamDatabase.instance)
-      {
-      }
-
-      public ThrottleStream (ViewStream main_stream, TimeSpan ts, ViewStreamDatabase vsd){
-         _main_stream = main_stream;
-         _ts = ts;
-         _vsd;
-      }
-
-       protected override void myOnFirstSubscriber() {
-         _main_stream.onChange += _handle_base_stream_update;
-      }
-
-      protected override void myOnLastSubscriber() {
-         _main_stream.onChange -= _handle_base_stream_update;
-      }
-
-      private void _handle_base_stream_update(object dp_obj){
-         if (DateTime.Now - _last_fire_time >= _ts){
-            DataPacket dp_in = dp_obj;
-            _last_fire_time = DateTime.Now;
-            DataPacket dp = BlueDataUtility.build_dp_inc(dp_in.newRow, myLastValue as DataPacket);
-            myFireEvent(dp);
-         }
-      }
- }
-}
@@ -40,4 +40,4 @@ namespace BCG {
            myFireSwitch = myLastFiredPacket.addedRows.Count + myLastFiredPacket.deletedRows.Count == 0 ? false : true;
        }
    }
-}
+}
@@ -1,15 +1,18 @@
 using System;
 using System.Collections.Generic;
-
 using mbts;

-namespace BCG {
-   internal class MutableAppendTable : IBlueRowCollection {
-      private class MyRow : IBlueRow {
+namespace BCG
+{
+   internal class MutableAppendTable : IBlueRowCollection
+   {
+      private class MyRow : IBlueRow
+      {
         private IBlueRow myRow;
         private int myID;

-         public MyRow(IBlueRow r, int id) {
+         public MyRow(IBlueRow r, int id)
+         {
            myRow = r;
            myID = id;
         }
@@ -18,8 +21,10 @@ namespace BCG {

         public IBlueTableSchema schema { get { return myRow.schema; } }

-         public object this[int index] {
-            get {
+         public object this[int index]
+         {
+            get
+            {
               if (myRow.schema[index].colName == "id")
                  return myID;
               else
@@ -47,17 +52,20 @@ namespace BCG {

         public System.Collections.ICollection Keys { get { return myRow.Keys; } }

-         public void Remove(object key) { throw new Exception("The method or operation is not implemented."); }
+         public void Remove(object key) {throw new Exception("The method or operation is not implemented.");}

-         public System.Collections.ICollection Values {
-            get {
+         public System.Collections.ICollection Values
+         {
+            get
+            {
               object[] vals = new object[myRow.Count];
               CopyTo(vals, 0);
               return vals;
            }
         }

-         public object this[object key] {
+         public object this[object key]
+         {
            get { return key is string ? this[key as string] : this[(int)key]; }
            set { throw new Exception("The method or operation is not implemented."); }
         }
@@ -66,7 +74,8 @@ namespace BCG {

         #region ICollection Members

-         public void CopyTo(Array array, int index) {
+         public void CopyTo(Array array, int index)
+         {
            myRow.CopyTo(array, index);
            array.SetValue(myID, index + myRow.schema["id"].colNo);
         }
@@ -81,7 +90,8 @@ namespace BCG {

         #region IEnumerable Members

-         System.Collections.IEnumerator System.Collections.IEnumerable.GetEnumerator() { return new BlueRowEnumerator(this); }
+         System.Collections.IEnumerator System.Collections.IEnumerable.GetEnumerator()
+         { return new BlueRowEnumerator(this); }

         #endregion
      }
@@ -96,7 +106,8 @@ namespace BCG {

      public MutableAppendTable(IBlueTableSchema schema) { myRows = new BlueTable(schema); }

-      public IBlueRow addRow(IBlueRow r, int tableNo) {
+      public IBlueRow addRow(IBlueRow r, int tableNo)
+      {
         if (!myIDMap.ContainsKey(tableNo))
            myIDMap.Add(tableNo, new Dictionary<int, IBlueRow>());
         IBlueRow nr = new MyRow(r, myID++); // BlueDataUtility.getNewReplacedRow(r, myID++, "id");
@@ -105,17 +116,20 @@ namespace BCG {
         return nr;
      }

-      public IBlueRow removeRow(IBlueRow r, int tableNo) {
+      public IBlueRow removeRow(IBlueRow r, int tableNo)
+      {
         IBlueRow dr = myIDMap[tableNo][(int)r["id"]];
         myRows.deleteRow(dr);
         myIDMap[tableNo].Remove((int)r["id"]);
         return dr;
      }

-      public IBlueRowCollection removeTable(int tableNo) {
+      public IBlueRowCollection removeTable(int tableNo)
+      {
         if (!myIDMap.ContainsKey(tableNo))
            return null;
-         else {
+         else
+         {
            BlueTable rv = new BlueTable(myRows.schema);
            foreach (IBlueRow r in myIDMap[tableNo].Values)
               rv.addRow(r);
@@ -125,7 +139,8 @@ namespace BCG {
         }
      }

-      public void clear() {
+      public void clear()
+      {
         myRows.clear();
         myIDMap.Clear();
      }
@@ -136,7 +151,7 @@ namespace BCG {

      public IBlueTableSchema schema { get { return myRows.schema; } }

-      public bool containsID(int id) { return myRows.containsID(id); }
+      public bool containsID(int id) {return myRows.containsID(id);}

      #endregion

@@ -154,7 +169,7 @@ namespace BCG {

      #region IEnumerable Members

-      public System.Collections.IEnumerator GetEnumerator() { return myRows.GetEnumerator(); }
+      public System.Collections.IEnumerator GetEnumerator() {return myRows.GetEnumerator();}

      #endregion
   }
@@ -164,8 +179,8 @@ namespace BCG {
   /// to append (or remove) more streams at
   /// runtime
   /// </summary>
-   [DataStreamUserRespresentation("UnionStream")]
-   public class MutableAppendStream : ViewStream {
+   public class MutableAppendStream : ViewStream
+   {
      private bool mySubscriptionActive = false;
      private MutableAppendTable myTable;
      private int myMovingIndex;
@@ -173,17 +188,22 @@ namespace BCG {
      private bool _allign_col_orders = true;

      public MutableAppendStream(params ViewStream[] streams)
-         : this(new List<ViewStream>(streams)) { }
+         : this(new List<ViewStream>(streams))
+      { }

      public MutableAppendStream(bool allign_col_order, params ViewStream[] streams)
-         : this(allign_col_order, new List<ViewStream>(streams)) { }
+         : this(allign_col_order, new List<ViewStream>(streams))
+      { }

      public MutableAppendStream(ICollection<ViewStream> streams)
-         : this(true, streams) { }
+         : this(true, streams)
+      { }

-      public MutableAppendStream(bool allign_col_orders, ICollection<ViewStream> streams) {
+      public MutableAppendStream(bool allign_col_orders, ICollection<ViewStream> streams)
+      {
         _allign_col_orders = allign_col_orders;
-         foreach (ViewStream vs in streams) {
+         foreach (ViewStream vs in streams)
+         {
            myMovingIndex++;
            ViewStream pj = new ChangeTableNameStream(vs, myMovingIndex.ToString());
            if (_allign_col_orders)
@@ -192,7 +212,8 @@ namespace BCG {
         }
      }

-      public void addStream(ViewStream vs) {
+      public void addStream(ViewStream vs)
+      {
         myMovingIndex++;
         ///the change name is done to identify the callbacks, and the project to
         ///make all the schemas col orders the same
@@ -204,14 +225,19 @@ namespace BCG {
            pj.onChange += new ChangeDelegate(myHandleStreamChanges);
      }

-      public void removeStream(ViewStream vs) {
+      public void removeStream(ViewStream vs)
+      {
         int index = -1;
-         foreach (KeyValuePair<int, Pair<ViewStream, ViewStream>> p in myStreamIndex) {
-            if (p.Value.first == vs) {
+         foreach (KeyValuePair<int, Pair<ViewStream, ViewStream>> p in myStreamIndex)
+         {
+            if (p.Value.first == vs)
+            {
               index = p.Key;
-               if (mySubscriptionActive) {
+               if (mySubscriptionActive)
+               {
                  p.Value.second.onChange -= new ChangeDelegate(myHandleStreamChanges);
-                  if (myTable != null) {
+                  if (myTable != null)
+                  {
                     IBlueRowCollection delRows = myTable.removeTable(p.Key);
                     if (delRows != null)
                        myFireEvent(new DataPacket(myTable, new BlueTable(myTable.schema), delRows));
@@ -224,13 +250,15 @@ namespace BCG {
         }
      }

-      protected override void myOnFirstSubscriber() {
+      protected override void myOnFirstSubscriber()
+      {
         mySubscriptionActive = true;
         foreach (KeyValuePair<int, Pair<ViewStream, ViewStream>> p in myStreamIndex)
            p.Value.second.onChange += new ChangeDelegate(myHandleStreamChanges);
      }

-      protected override void myOnLastSubscriber() {
+      protected override void myOnLastSubscriber()
+      {
         mySubscriptionActive = false;
         foreach (KeyValuePair<int, Pair<ViewStream, ViewStream>> p in myStreamIndex)
            p.Value.second.onChange -= new ChangeDelegate(myHandleStreamChanges);
@@ -238,17 +266,20 @@ namespace BCG {
            myTable.clear();
      }

-      private void myHandleStreamChanges(object newValue) {
+      private void myHandleStreamChanges(object newValue)
+      {         
         DataPacket dp = newValue as DataPacket;
         myTable = myTable ?? new MutableAppendTable(new BlueTableSchema("MutableAppend", dp.schema.columnInfo));
         BlueTable adds = new BlueTable(myTable.schema),
                     dels = new BlueTable(myTable.schema);
         int index = Int32.Parse(dp.schema.tableName);
-         foreach (IBlueRow r in dp.deletedRows) {
+         foreach (IBlueRow r in dp.deletedRows)
+         {
            IBlueRow dr = myTable.removeRow(r, index);
            dels.addRow(dr);
         }
-         foreach (IBlueRow r in dp.addedRows) {
+         foreach (IBlueRow r in dp.addedRows)
+         {
            IBlueRow nr = myTable.addRow(r, index);
            adds.addRow(nr);
         }
@@ -280,28 +280,6 @@ namespace BCG {

      #region json

-      public static void write_to_json_file(this IBlueRowCollection table, string file_path) { 
-         write_dict_to_json_file(file_path, table.to_namevalue_pairs());
-      }
-
-      public static void write_dict_to_json_file(string file_path, List<Dictionary<string, string>> data) {
-         var options = new JsonSerializerOptions { WriteIndented = true };
-         string json = System.Text.Json.JsonSerializer.Serialize(data, options);
-         File.WriteAllText(file_path, json);
-      }
-
-      public static List<Dictionary<string, string>> ReadFromFile(string filePath) {
-         string json = File.ReadAllText(filePath);
-         return System.Text.Json.JsonSerializer.Deserialize<List<Dictionary<string, string>>>(json);
-      }
-
-      public static void write_table_to_json(IBlueRowCollection table) {
-         List<Dictionary<string, string>> rows = new List<Dictionary<string, string>>();
-         foreach (IBlueRow row in table) {
-            row.to_json();
-         }
-      }
-
      public static Dictionary<string, string> convert_json_to_dictionary(string json_string) {
         Dictionary<string, string> dictionary = JsonConvert.DeserializeObject<Dictionary<string, string>>(json_string);
         return dictionary;
@@ -370,14 +348,6 @@ namespace BCG {
         }
         return rv;
      }
-
-      public static List<Dictionary<string,string>> to_namevalue_pairs(this IBlueRowCollection table) {
-         List<Dictionary<string, string>> rv = new List<Dictionary<string, string>>();
-         foreach (IBlueRow r in table) {
-            rv.Add(r.to_dict());
-         }
-         return rv;
-      }
      #region Schema types
      // these are the only types allowed in schemas
      private static IReadOnlyDictionary<TypeCode, Type> _typecode_to_type = new Dictionary<TypeCode, Type>
@@ -1255,20 +1225,6 @@ namespace BCG {
         return rv;
      }

-      public static Dictionary<string, string> to_dict(this IBlueRow r) {
-         Dictionary<string, string> map = new Dictionary<string, string>();
-         foreach (ColumnInfo c in r.schema.columnInfo) {
-            map.Add(c.colName, r[c.colName].ToString());
-         }
-         return map;
-      }
-
-      public static string to_json(this IBlueRow r) {
-         Dictionary<string, string> map = r.to_dict();
-         string json = System.Text.Json.JsonSerializer.Serialize(map);
-         return json;
-      }
-
      public static void compute_differences_fast(IBlueRowCollection new_rows, IBlueRowCollection old_rows,
                                                  out IBlueRowCollection out_d_rows, out IBlueRowCollection out_a_rows) {
         if (old_rows == null || old_rows.Count == 0) {
@@ -2,7 +2,7 @@

  <PropertyGroup>
    <AllowUnsafeBlocks>true</AllowUnsafeBlocks>
-    <TargetFramework>net8.0</TargetFramework>
+    <TargetFramework>net7.0</TargetFramework>
    <Nullable>disable</Nullable>
  </PropertyGroup>

@@ -19,8 +19,8 @@
  </ItemGroup>

  <ItemGroup>
-    <Proto Include="../../lib_spark/spark/config/protos/data_stream_config.proto" CSName="DataStreamConfig">
-      <OutDir>$(ProtoOutputPath)/lib_spark/spark/config/protos</OutDir>
+    <Proto Include="../../lib/config/protos/data_stream_config.proto" CSName="DataStreamConfig">
+      <OutDir>$(ProtoOutputPath)/lib/config/protos</OutDir>
    </Proto>
  </ItemGroup>

@@ -47,10 +47,10 @@ namespace BCG {
      /// </summary>
      /// <param name="assemblyName"></param>
      public void loadAssembly(string assemblyName) {
-         Func<Type, bool> is_blue_orb = (t) => {
-            //if (!assemblyName.Contains("SharedStreams.dll"))
-            //   return true;
-            return t.Namespace == "bts.BlueOrb";// || t.Namespace == "BCG";
+         Func<Type, bool> is_silverpine_only = (t) => {
+            if (!assemblyName.Contains("SharedStreams.dll"))
+               return true;
+            return t.Namespace == "bts.SilverPine";
         };
         bts.Log.Gen.info($"PrimitiveManager.loadAssembly assemblyName: {assemblyName}");
         if (myAssemblyNames.Contains(assemblyName))
@@ -61,7 +61,7 @@ namespace BCG {
            Assembly.LoadFrom(assemblyName);

         foreach (Type type in loadedAssembly.GetTypes())
-            if (!type.IsInterface && is_blue_orb(type)
+            if (!type.IsInterface && is_silverpine_only(type)
               && !type.IsAbstract
               && type.IsSubclassOf(typeof(ViewStream))
               && type.IsPublic) {
@@ -77,7 +77,7 @@ namespace BCG {
                     break;
                  }
               if (isVisible) {
-                 // Log.assert_d(myTypeLookupTable[userRep] == null, type.FullName);
+                  Log.assert_d(myTypeLookupTable[userRep] == null, type.FullName);
                  myTypeLookupTable[userRep] = type;
               }
            }
@@ -1,7 +1,7 @@
 <Project Sdk="Microsoft.NET.Sdk">

  <PropertyGroup>
-    <TargetFramework>net8.0</TargetFramework>
+    <TargetFramework>net7.0</TargetFramework>
    <ImplicitUsings>enable</ImplicitUsings>
    <Nullable>enable</Nullable>
  </PropertyGroup>
@@ -2,7 +2,7 @@

  <PropertyGroup>
    <OutputType>Exe</OutputType>
-    <TargetFramework>net8.0</TargetFramework>
+    <TargetFramework>net7.0</TargetFramework>
    <Nullable>enable</Nullable>
    <AllowUnsafeBlocks>true</AllowUnsafeBlocks>
  </PropertyGroup>
@@ -2,7 +2,7 @@

  <PropertyGroup>
    <OutputType>Exe</OutputType>
-    <TargetFramework>net8.0-windows</TargetFramework>
+    <TargetFramework>net7.0-windows</TargetFramework>
    <Nullable>enable</Nullable>
    <UseWindowsForms>true</UseWindowsForms>
  </PropertyGroup>
@@ -1,7 +1,7 @@
 <Project Sdk="Microsoft.NET.Sdk">

  <PropertyGroup>
-    <TargetFramework>net8.0</TargetFramework>
+    <TargetFramework>net7.0</TargetFramework>
    <ImplicitUsings>enable</ImplicitUsings>
    <Nullable>enable</Nullable>

@@ -1,7 +1,7 @@
 <Project Sdk="Microsoft.NET.Sdk">

  <PropertyGroup>
-    <TargetFramework>net8.0</TargetFramework>
+    <TargetFramework>net7.0</TargetFramework>
    <ImplicitUsings>enable</ImplicitUsings>
    <Nullable>enable</Nullable>
  </PropertyGroup>
@@ -1,7 +1,7 @@
 <Project Sdk="Microsoft.NET.Sdk">

  <PropertyGroup>
-    <TargetFramework>net8.0-windows</TargetFramework>
+    <TargetFramework>net7.0-windows</TargetFramework>
    <ImplicitUsings>enable</ImplicitUsings>
    <Nullable>enable</Nullable>
    <UseWindowsForms>true</UseWindowsForms>
@@ -6,18 +6,12 @@ namespace BCG
 	public interface IPublisher
 	{
      void publish( IPubItem pi );
-#pragma warning disable SYSLIB0011 // Type or member is obsolete
      void publish( IPubItem pi, IFormatter f );
-#pragma warning restore SYSLIB0011 // Type or member is obsolete
      void publish( IPubItem[] pi );
-#pragma warning disable SYSLIB0011 // Type or member is obsolete
      void publish( IPubItem[] pi, IFormatter f );
-#pragma warning restore SYSLIB0011 // Type or member is obsolete
   }

-#pragma warning disable SYSLIB0011 // Type or member is obsolete
   public interface IPublisher<T> where T : IFormatter
-#pragma warning restore SYSLIB0011 // Type or member is obsolete
   {
      void publish (IPubItem pi);
      void publish (IPubItem[] pi);
@@ -5,21 +5,23 @@ using System.Runtime.Serialization.Formatters.Binary;

 namespace BCG
 {
-#pragma warning disable SYSLIB0011 // Type or member is obsolete
   public class GenericFormatter<F> : IGenericFormatter where F : IFormatter, new ()
   {
      IFormatter myFormatter = new F ();

      public T Deserialize<T> (Stream serializationStream)
      {
+         #pragma warning disable 
         return (T) myFormatter.Deserialize (serializationStream);
+         #pragma warning restore SYSLIB0011
      }
      public void Serialize<T> (Stream serializationStream, T graph)
      {
+         #pragma warning disable 
         myFormatter.Serialize (serializationStream, graph);
+         #pragma warning restore SYSLIB0011
      }
   }

   public class GenericBinaryFormatter : GenericFormatter<BinaryFormatter> { }
-#pragma warning restore SYSLIB0011 // Type or member is obsolete
 }
@@ -6,7 +6,7 @@

  <PropertyGroup>
      <AllowUnsafeBlocks>true</AllowUnsafeBlocks>
-    <TargetFramework>net8.0</TargetFramework>
+    <TargetFramework>net7.0</TargetFramework>
    <Nullable>enable</Nullable>
  </PropertyGroup>

@@ -2,7 +2,7 @@

  <PropertyGroup>
    <OutputType>Exe</OutputType>
-    <TargetFramework>net8.0</TargetFramework>
+    <TargetFramework>net7.0</TargetFramework>
    <Nullable>enable</Nullable>
  </PropertyGroup>

@@ -1,18 +0,0 @@
-namespace bts.BlueOrb {
-   using bts.SilverPine;
-   using BCG;
-   using System;
-
-   [DataStreamUserRespresentation("SpatialArbScoreStream")]
-   public class SpatialArbScoreStream : ViewStream {
-      public SpatialArbScoreStream(int window_sz_secs, int sample_period_sz)
-         : this(TimeSpan.FromSeconds(window_sz_secs), TimeSpan.FromSeconds(sample_period_sz), ViewStreamDatabase.instance) { }
-
-      public SpatialArbScoreStream(TimeSpan window_sz, TimeSpan sample_period, ViewStreamDatabase vsd) {
-         ViewStream rs = new SpatialArbStatsStream(window_sz, sample_period, vsd);
-         rs = new ThrottleStream(rs, TimeSpan.FromSeconds(1), vsd);
-         rs = new SortStream(rs, "-n_trade_ticks");
-         mySetRealSource(rs);
-      }
-   }
-}
@@ -1,113 +0,0 @@
-namespace bts.SilverPine {
-   using BCG;
-
-   using SharpCompress.Common.Rar;
-
-   using System;
-   using System.Collections.Generic;
-   using System.Linq;
-   using System.Text;
-   using System.Threading.Tasks;
-
-   // For all the pairs generated by SpatialFuturePairsStream, this stream 
-   // generates the statistics, by appending together a SpreadStatsStream
-   [DataStreamUserRespresentation("SpatialArbStatsStream")]
-   public class SpatialArbStatsStream : ViewStream{
-      private ViewStream _spatial_pairs_stream;
-      private ViewStreamDatabase _vsd;
-      private ViewStream _tot_str = null;
-      private HashSet<string> _key_map = new HashSet<string>();
-      private ViewStream _bbo_stream;
-      private readonly TimeSpan _window_sz, _sampling_period_sz;
-
-      public SpatialArbStatsStream(int window_sz_seconds, int sampling_period_sz_seconds)
-         : this(TimeSpan.FromSeconds(window_sz_seconds), TimeSpan.FromSeconds(sampling_period_sz_seconds), ViewStreamDatabase.instance) { }
-
-      public SpatialArbStatsStream(TimeSpan window_sz, TimeSpan sampling_period_sz, ViewStreamDatabase vsd) {
-         _vsd = vsd;
-         _window_sz = window_sz;
-         _sampling_period_sz = sampling_period_sz;
-         _spatial_pairs_stream = new SpatialFuturePairsStream(vsd);
-         _bbo_stream = new BBOStream(vsd);
-      }
-
-      protected override void myOnFirstSubscriber() {
-         _spatial_pairs_stream.onChange += _spatial_pairs_stream_onchange;
-      }
-
-      protected override void myOnLastSubscriber() {
-         _spatial_pairs_stream.onChange -= _spatial_pairs_stream_onchange;
-      }
-
-      private ViewStream _build_mapped_stream(ViewStream parent_str) {
-         Func<IBlueTableSchema, IBlueTableSchema> build_schema = (inp_schema) => {
-            IBlueTableSchema rv = new BlueTableSchema("SpreaddStats", new ColumnInfo("id", 0, TypeCode.Int32));
-            rv = rv.add_column(TypeCode.String, "spread");
-            rv = rv.add_column(TypeCode.String, "window");
-            // bids
-            rv = rv.add_column(TypeCode.Double, "bid_mean");
-            rv = rv.add_column(TypeCode.Double, "bid_stddev");
-            rv = rv.add_column(TypeCode.Double, "bid_skewness");
-            rv = rv.add_column(TypeCode.Double, "bid_kurtosis");
-            rv = rv.add_column(TypeCode.Double, "bid_max");
-            rv = rv.add_column(TypeCode.Double, "bid_min");
-            // asks
-            rv = rv.add_column(TypeCode.Double, "ask_mean");
-            rv = rv.add_column(TypeCode.Double, "ask_stddev");
-            rv = rv.add_column(TypeCode.Double, "ask_skewness");
-            rv = rv.add_column(TypeCode.Double, "ask_kurtosis");
-            rv = rv.add_column(TypeCode.Double, "ask_max");
-            rv = rv.add_column(TypeCode.Double, "ask_min");
-            // tail distribution
-            rv = rv.add_column(TypeCode.Double, "bid Z ≥ 2σ");
-            rv = rv.add_column(TypeCode.Double, "ask Z < -2σ");
-            // trading ticks
-            rv = rv.add_column(TypeCode.Int32, "n_buy_ticks");
-            rv = rv.add_column(TypeCode.Double, "cumm_buy_price");
-            rv = rv.add_column(TypeCode.Int32, "n_sell_ticks");
-            rv = rv.add_column(TypeCode.Double, "cumm_sell_price");
-            rv = rv.add_column(TypeCode.Int32, "n_trade_ticks");
-            return rv;
-         };
-         Func<IBlueRow, IBlueRow> build_row = (r) => {
-            IBlueTableSchema s = build_schema(r.schema);
-            return new BlueRow(s, r["id"], r["spread"], r["window"],
-                           r["bid_mean"], r["bid_stddev"], r["bid_skewness"], r["bid_kurtosis"], r["bid_max"], r["bid_min"],
-                           r["ask_mean"], r["ask_stddev"], r["ask_skewness"], r["ask_kurtosis"], r["ask_max"], r["ask_min"],
-                           r["bid Z ≥ 2σ"], r["ask Z < -2σ"],
-                           r["n_buy_ticks"], r["cumm_buy_price"], r["n_sell_ticks"], r["cumm_sell_price"], r["n_trade_ticks"]);
-         };
-         MapStream rs = new MapStream(parent_str, inp_schema => build_schema(inp_schema),
-            inp_r => build_row(inp_r));
-         return rs;
-      }
-
-      private void _spatial_pairs_stream_onchange(object dp_obj) {
-         if (_tot_str != null)
-            return;
-         MutableAppendStream append_str = new MutableAppendStream();
-         _tot_str = _build_mapped_stream(append_str);
-         //_tot_str = new ThrottleStream(_tot_str, 1000);
-         //_tot_str = new SortStream(_tot_str, "-n_trade_ticks");
-         _tot_str.onChange += _handle_tot_stream;
-
-         DataPacket dp = dp_obj as DataPacket;         
-         foreach (IBlueRow r in dp.newRows) {
-            string s1 = r.get<string>("s1");
-            string s2 = r.get<string>("s2");
-            string key = SpatialFuturePairsStream.get_key(s1, s2);
-            if (!_key_map.Contains(key)) {
-               _key_map.Add(key);
-               ViewStream spread_stats_stream = new SpreadStatsStream(s1, s2, 1, 1, Convert.ToInt32(_window_sz.TotalSeconds), 
-                                                                     Convert.ToInt32(_sampling_period_sz.TotalSeconds), _bbo_stream, _vsd);
-               append_str.addStream(spread_stats_stream);
-            }
-         }
-      }
-
-      private void _handle_tot_stream(object dp_obj) {
-         myFireEvent(dp_obj);
-      }
-   }
-
-}
@@ -1,89 +0,0 @@
-namespace bts.SilverPine {
-   using BCG;
-
-   using bts.api;
-
-   using System;
-   using System.Collections.Generic;
-   using System.Linq;
-   using System.Text;
-   using System.Threading.Tasks;
-
-   [DataStreamUserRespresentation("SpatialFuturePairsStream")]
-   public class SpatialFuturePairsStream : ViewStream {
-      private ViewStream _main_stream;
-      private IBlueTableSchema _schema;
-      private HashSet<string> _key_map = new HashSet<string>();
-      public SpatialFuturePairsStream()
-         : this(ViewStreamDatabase.instance) { }
-
-      public SpatialFuturePairsStream(ViewStreamDatabase vsd) {
-         _main_stream = new InstrumentsStream(vsd);
-         _main_stream = new FilterStream(_main_stream, "sec_type='Future'");
-         _schema = new BlueTableSchema("SpatialFutures", new ColumnInfo("id", 0, TypeCode.Int32));
-         _schema = _schema.add_column(TypeCode.String, "s1");
-         _schema = _schema.add_column(TypeCode.String, "venue_1");
-         _schema = _schema.add_column(TypeCode.String, "s2");
-         _schema = _schema.add_column(TypeCode.String, "venue_2");
-
-      }
-
-      protected override void myOnFirstSubscriber() {
-         _main_stream.onChange += _main_stream_onchange;
-      }
-
-      protected override void myOnLastSubscriber() {
-         _main_stream.onChange -= _main_stream_onchange;
-      }
-
-      public static string get_key(string s1, string s2) {
-         string key = string.Compare(s1, s2, StringComparison.Ordinal) <= 0 ? s1 + s2 : s2 + s1;
-         return key;
-      }
-
-      public static string get_key(IBlueRow r1, IBlueRow r2) {
-         string s1 = r1.get<string>("sec");
-         string s2 = r2.get<string>("sec");
-         return get_key(s1, s2);
-      }
-
-      private bool _combine(IBlueRow r1, IBlueRow r2) {
-         string key = get_key(r1, r2);
-         if (_key_map.Contains(key))
-            return false;
-         if (r1.get<string>("symbol") != r2.get<string>("symbol"))
-            return false;
-         if (r1.get_enum<Venue>("venue") == r2.get_enum<Venue>("venue"))
-            return false;
-         if (r1.get<int>("sec_key") == r2.get<int>("sec_key")) {
-            return false;
-         }
-         DateTime e1 = r1.get_datetime("expiration");
-         DateTime e2 = r2.get_datetime("expiration");
-         bool exps_eq = e1 == e2 || (e1.Year >= 2100 && e2.Year >= 2100);
-         if (!exps_eq)
-            return false;         
-         return true;
-      }
-
-      private void _main_stream_onchange(object dp_obj) {
-         _key_map.Clear();
-         DataPacket dp = dp_obj as DataPacket;
-         BlueTable table = new BlueTable(_schema);
-         var rows = dp.newRows.to_list();
-         foreach (var r1 in rows) {
-            foreach (var r2 in rows) {
-               if (_combine(r1, r2)) {
-                  IBlueRow nr = new BlueRow(_schema, table.Count, r1.get<string>("sec"), r1.get<string>("venue"),
-                     r2.get<string>("sec"), r2.get<string>("venue"));
-                  table.addRow(nr);
-                  string key = get_key(r1, r2);
-                  _key_map.Add(key);
-               }
-            }
-         }
-         var dp_out = BlueDataUtility.build_dp(table, myLastValue as DataPacket);
-         myFireEvent(dp_out);
-      }
-   }
-}
@@ -1,656 +0,0 @@
-using System;
-using System.Collections.Generic;
-using System.Linq;
-using MathNet.Numerics.Statistics;
-using MathNet.Numerics.LinearRegression;
-using System.Collections.Generic;
-using System.Linq;
-using MathNet.Numerics.Statistics;
-using MathNet.Numerics.LinearAlgebra;
-using Amazon.Runtime;
-using System;
-using System.Collections.Generic;
-using System.Linq;
-using MathNet.Numerics.Statistics;
-using System;
-using System.Collections.Generic;
-using System.Linq;
-using MathNet.Numerics.Statistics;
-using System;
-using System.Collections.Generic;
-using System.Linq;
-using MathNet.Numerics.Statistics;
-// stats calculator
-using System;
-using System.Collections.Generic;
-using System.Linq;
-using MathNet.Numerics.Statistics;
-
-/// <summary>
-/// Provides evaluation metrics to assess the quality of a synthetic spread for mean-reversion trading.
-/// Assumes the input spread data is a trailing, time-causal window of recent ticks (e.g., last 120 seconds).
-/// </summary>
-public static class SpreadPerformanceEvaluator
-{
-
-   public static class SpreadStatistics {
-      public class SpreadStatsResult {
-         public Dictionary<string, double> bid_zscore_distribution { get; set; }
-         public Dictionary<string, double> ask_zscore_distribution { get; set; }
-         public Dictionary<string, double> mid_zscore_distribution { get; set; }
-
-         public DescriptiveStatistics bid_stats { get; set; }
-         public DescriptiveStatistics ask_stats { get; set; }
-         public DescriptiveStatistics mid_stats { get; set; }
-      }
-
-      private static readonly string[] _zscore_labels = new[]
-      {
-        "Z < -2σ",
-        "-2σ ≤ Z < -1σ",
-        "-1σ ≤ Z < 0σ",
-        "0σ ≤ Z < 1σ",
-        "1σ ≤ Z < 2σ",
-        "Z ≥ 2σ"
-    };
-
-      public static SpreadStatsResult analyze_spread_quotes(List<(DateTime timestamp, double bid, double bid_qty, double ask, double ask_qty)> quotes) {
-         if (quotes == null || quotes.Count == 0)
-            throw new ArgumentException("Spread quote list cannot be null or empty.");
-
-         var bid_prices = quotes.Select(q => q.bid).ToList();
-         var ask_prices = quotes.Select(q => q.ask).ToList();
-         var mid_prices = quotes.Select(q => (q.bid + q.ask) / 2.0).ToList();
-
-         var bid_stats = new DescriptiveStatistics(bid_prices);
-         var ask_stats = new DescriptiveStatistics(ask_prices);
-         var mid_stats = new DescriptiveStatistics(mid_prices);
-
-         var bid_zscores = bid_prices.Select(x => zscore(x, bid_stats.Mean, bid_stats.StandardDeviation)).ToList();
-         var ask_zscores = ask_prices.Select(x => zscore(x, ask_stats.Mean, ask_stats.StandardDeviation)).ToList();
-         var mid_zscores = mid_prices.Select(x => zscore(x, mid_stats.Mean, mid_stats.StandardDeviation)).ToList();
-
-         var bid_distribution = bin_zscores_to_percentages(bid_zscores);
-         var ask_distribution = bin_zscores_to_percentages(ask_zscores);
-         var mid_distribution = bin_zscores_to_percentages(mid_zscores);
-
-         return new SpreadStatsResult {
-            bid_zscore_distribution = bid_distribution,
-            ask_zscore_distribution = ask_distribution,
-            mid_zscore_distribution = mid_distribution,
-            bid_stats = bid_stats,
-            ask_stats = ask_stats,
-            mid_stats = mid_stats
-         };
-      }
-
-      private static double zscore(double x, double mean, double std_dev) {
-         return std_dev == 0 ? 0 : (x - mean) / std_dev;
-      }
-
-      private static Dictionary<string, double> bin_zscores_to_percentages(List<double> zscores) {
-         var bins = _zscore_labels.ToDictionary(label => label, _ => 0);
-
-         foreach (var z in zscores) {
-            if (z < -2) bins["Z < -2σ"]++;
-            else if (z < -1) bins["-2σ ≤ Z < -1σ"]++;
-            else if (z < 0) bins["-1σ ≤ Z < 0σ"]++;
-            else if (z < 1) bins["0σ ≤ Z < 1σ"]++;
-            else if (z < 2) bins["1σ ≤ Z < 2σ"]++;
-            else bins["Z ≥ 2σ"]++;
-         }
-
-         double total = zscores.Count;
-         return bins.ToDictionary(kvp => kvp.Key, kvp => total == 0 ? 0 : (kvp.Value / total) * 100.0);
-      }
-
-      public static double score_spread(SpreadStatsResult result) {
-         var ask_bins = result.ask_zscore_distribution;
-         var bid_bins = result.bid_zscore_distribution;
-         var mid_bins = result.mid_zscore_distribution;
-
-         double buy_opportunity = ask_bins["Z < -2σ"] + ask_bins["-2σ ≤ Z < -1σ"];
-         double sell_opportunity = bid_bins["Z ≥ 2σ"] + bid_bins["1σ ≤ Z < 2σ"];
-         double opportunity_score = buy_opportunity + sell_opportunity;
-
-         double below_mean = mid_bins["Z < -2σ"] + mid_bins["-2σ ≤ Z < -1σ"] + mid_bins["-1σ ≤ Z < 0σ"];
-         double above_mean = mid_bins["Z ≥ 2σ"] + mid_bins["1σ ≤ Z < 2σ"] + mid_bins["0σ ≤ Z < 1σ"];
-         double symmetry_score = 1.0 - Math.Abs(below_mean - above_mean) / (below_mean + above_mean + 1e-6);
-
-         double amplitude_score = result.mid_stats.StandardDeviation;
-         double skew_penalty = 1.0 / (1.0 + Math.Abs(result.mid_stats.Skewness));
-         double kurtosis_bonus = Math.Min(result.mid_stats.Kurtosis, 6.0) / 6.0;
-
-         double score =
-             opportunity_score *
-             symmetry_score *
-             amplitude_score *
-             skew_penalty *
-             (1.0 + kurtosis_bonus * 0.1);
-
-         return score;
-      }
-   }
-
-
-
-
-
-/// <summary>
-/// Represents a single trade action (entry or exit) taken during the simulation.
-/// </summary>
-public struct TradeAction
-    {
-        public bool IsEntry { get; set; }    // True if this action opened a position
-        public bool IsLong { get; set; }     // True = Long, False = Short
-        public double Price { get; set; }    // Executed price
-        public int Index { get; set; }       // Tick index of the action
-
-        public override string ToString()
-        {
-            string side = IsLong ? "Long" : "Short";
-            string type = IsEntry ? "Open" : "Close";
-            return $"{type} {side} @ {Price:F4} (Tick {Index})";
-        }
-    }
-
-    /// <summary>
-/// Simulates a mean-reversion strategy using rolling, causal mid-price stats.
-/// ✅ Entry:
-///   - Buy at ask when ask < rollingMean - entryThreshold × rollingStdDev
-///   - Sell at bid when bid > rollingMean + entryThreshold × rollingStdDev
-/// ✅ Exit:
-///   - Close when mid returns within rollingMean ± exitThreshold × rollingStdDev
-///   - Long exits at bid, short exits at ask
-/// This version uses only past and current data at each step (no forward-looking bias).
-/// </summary>
-public static (double TotalPnL, List<TradeAction> TradeActions, int BuyTurns, int SellTurns) EvaluateSimulatedPnL(
-    List<Tuple<double, double, double, double>> spreadData,
-    int statWindow = 30,
-    double entryThresholdMultiplier = 0.5,
-    double exitThresholdMultiplier = 0.1)
-{
-    var trades = new List<TradeAction>();
-    var mids = spreadData.Select(x => (x.Item1 + x.Item3) / 2.0).ToList();
-
-    if (spreadData.Count < statWindow)
-        return (0.0, trades, 0, 0);
-
-    bool positionOpen = false;
-    bool isLong = false;
-    double entryPrice = 0.0;
-    double pnl = 0.0;
-    int buyTurns = 0;
-    int sellTurns = 0;
-
-    for (int i = statWindow; i < spreadData.Count; i++)
-    {
-        double bid = spreadData[i].Item1;
-        double ask = spreadData[i].Item3;
-        double mid = mids[i];
-
-        // Rolling window
-        var window = mids.Skip(i - statWindow).Take(statWindow).ToList();
-        double mean = window.Average();
-        double stddev = Math.Sqrt(window.Select(x => Math.Pow(x - mean, 2)).Average());
-
-        double lowerThreshold = mean - entryThresholdMultiplier * stddev;
-        double upperThreshold = mean + entryThresholdMultiplier * stddev;
-        double exitBand = exitThresholdMultiplier * stddev;
-
-        if (!positionOpen)
-        {
-            if (ask < lowerThreshold)
-            {
-                positionOpen = true;
-                isLong = true;
-                entryPrice = ask;
-                trades.Add(new TradeAction { IsEntry = true, IsLong = true, Price = entryPrice, Index = i });
-            }
-            else if (bid > upperThreshold)
-            {
-                positionOpen = true;
-                isLong = false;
-                entryPrice = bid;
-                trades.Add(new TradeAction { IsEntry = true, IsLong = false, Price = entryPrice, Index = i });
-            }
-        }
-        else
-        {
-            if (Math.Abs(mid - mean) < exitBand)
-            {
-                double exitPrice = isLong ? bid : ask;
-                pnl += isLong ? (exitPrice - entryPrice) : (entryPrice - exitPrice);
-
-                trades.Add(new TradeAction { IsEntry = false, IsLong = isLong, Price = exitPrice, Index = i });
-
-                if (isLong) buyTurns++;
-                else sellTurns++;
-
-                positionOpen = false;
-            }
-        }
-    }
-
-    return (pnl, trades, buyTurns, sellTurns);
-}
-
-
-
-    /// <summary>
-    /// Estimates the Hurst exponent of the spread mid-prices.
-    /// Assumes the input is a trailing window of recent data.
-    /// Hurst < 0.5 implies mean-reversion; ideal is 0.35–0.45.
-    /// </summary>
-    public static double EvaluateHurstExponent(List<Tuple<double, double, double, double>> spreadData)
-    {
-        const int MinLength = 21;
-
-        if (spreadData == null || spreadData.Count < MinLength)
-            return double.NaN;
-
-        var spreadMids = spreadData.Select(x => (x.Item1 + x.Item3) / 2.0).ToArray();
-        int n = spreadMids.Length;
-
-        var lags = new List<double>();
-        var tau = new List<double>();
-
-        for (int lag = 2; lag <= 20; lag++)
-        {
-            if (n - lag <= 1)
-                continue;
-
-            double[] diffs = new double[n - lag];
-            for (int i = 0; i < diffs.Length; i++)
-                diffs[i] = spreadMids[i + lag] - spreadMids[i];
-
-            double stddev = diffs.StandardDeviation();
-
-            if (stddev > 0)
-            {
-                lags.Add(Math.Log(lag));
-                tau.Add(Math.Log(stddev));
-            }
-        }
-
-        if (lags.Count < 2)
-            return double.NaN;
-
-        var (intercept, slope) = SimpleRegression.Fit(lags.ToArray(), tau.ToArray());
-        double hurst = slope * 0.5 + 0.5;
-        return hurst;
-    }
-
-    /// <summary>
-/// Computes the mean and standard deviation of the spread mid-prices
-/// over a trailing, causal window.
-/// </summary>
-/// <param name="spreadData">List of (bid, bidQty, ask, askQty)</param>
-/// <returns>Tuple of (Mean, StdDev). Returns (NaN, NaN) if input is too short.</returns>
-public static (double Mean, double StdDev) EvaluateMeanAndStdDev(List<Tuple<double, double, double, double>> spreadData)
-{
-    if (spreadData == null || spreadData.Count < 2)
-        return (double.NaN, double.NaN);
-
-    var mids = spreadData.Select(x => (x.Item1 + x.Item3) / 2.0).ToList();
-    double mean = mids.Average();
-    double stddev = Math.Sqrt(mids.Select(x => Math.Pow(x - mean, 2)).Average());
-
-    return (mean, stddev);
-}
-
-
-   /// <summary>
-   /// Counts how many times the spread ask dropped below (mean - threshold) [buy signal]
-   /// and how many times the spread bid rose above (mean + threshold) [sell signal].
-   /// Assumes spreadData is a trailing, causal window.
-   /// </summary>
-   /// <returns>(BuyExcursions, SellExcursions)</returns>
-   public static (uint BuyExcursions, double cumm_buy_price, uint SellExcursions, double cumm_sell_price) EvaluateExcursionCount(
-       List<Tuple<double, double, double, double>> spreadData,
-       double entryThresholdMultiplier = 0.5) {
-      if (spreadData == null || spreadData.Count < 2)
-         return (0, Double.NaN, 0, Double.NaN);
-
-      var midPrices = spreadData.Select(x => (x.Item1 + x.Item3) / 2.0).ToList();
-      double mean = midPrices.Average();
-      double stddev = midPrices.StandardDeviation();
-      double buy_threshold = mean - entryThresholdMultiplier * stddev;
-      double sell_threshold = mean + entryThresholdMultiplier * stddev;
-
-      int buyCount = 0;
-      int sellCount = 0;
-      //bool in_buy_zone = false;
-      //bool in_sell_zone = false;
-
-      double cumm_buy_price = 0.0, cumm_sell_price = 0.0;
-      foreach (var (bid, _, ask, _) in spreadData) {
-         if (ask <= buy_threshold) {
-            //if (!in_buy_zone) {
-            cumm_buy_price += ask;
-            buyCount++;
-            //in_buy_zone = true;
-         }
-         //} else {
-         //   in_buy_zone = false;
-         //}
-
-         if (bid >= sell_threshold) {
-            //if (!in_sell_zone) {
-            cumm_sell_price += bid;
-            sellCount++;
-            //in_sell_zone = true;
-         }
-         //} else {
-         //   in_sell_zone = false;
-         //}
-      }
-
-      return ((uint)buyCount, cumm_buy_price / buyCount, (uint)sellCount, cumm_sell_price / sellCount);
-   }
-
-/// <summary>
-/// Counts how many times the spread mid-price crosses its mean level.
-/// Each crossing indicates a potential oscillation turn (cycle midpoint).
-/// Assumes the input is a trailing, causal window of spread ticks.
-/// </summary>
-/// <param name="spreadData">List of (bid, bidQty, ask, askQty)</param>
-/// <returns>Number of zero crossings in the mid-price series</returns>
-public static UInt32 EvaluateZeroCrossingCount(List<Tuple<double, double, double, double>> spreadData)
-{
-    if (spreadData == null || spreadData.Count < 2)
-        return 0;
-
-    var mids = spreadData.Select(x => (x.Item1 + x.Item3) / 2.0).ToList();
-    double mean = mids.Average();
-    int zeroCrossings = 0;
-
-    for (int i = 1; i < mids.Count; i++)
-    {
-        double prevOffset = mids[i - 1] - mean;
-        double currOffset = mids[i] - mean;
-
-        if (prevOffset * currOffset < 0) // Sign changed
-            zeroCrossings++;
-    }
-
-    return (UInt32)zeroCrossings;
-}
-
-   /// <summary>
-   /// Computes the half-life of mean reversion.
-   /// Lower half-life means the spread reverts to its mean more quickly.
-   /// </summary>
-   public static double ComputeHalfLife(List<Tuple<double, double, double, double>> spreadData) {
-      var prices = spreadData.Select(x => (x.Item1 + x.Item3) / 2.0).ToList();
-      var lagged = prices.Take(prices.Count - 1).ToArray();
-      var delta = prices.Skip(1).Zip(lagged, (cur, prev) => cur - prev).ToArray();
-
-      if (lagged.Length < 2) return double.NaN;
-
-      var x = Vector<double>.Build.Dense(lagged);
-      var y = Vector<double>.Build.Dense(delta);
-      var beta = x.DotProduct(y) / x.DotProduct(x);
-
-      return -Math.Log(2) / beta;
-   }
-
-   /// <summary>
-   /// Computes the lag-1 autocorrelation of the spread.
-   /// Lower values indicate quicker mean-reversion.
-   /// </summary>
-   public static double ComputeLag1Autocorrelation(List<Tuple<double, double, double, double>> spreadData) {
-      var prices = spreadData.Select(x => (x.Item1 + x.Item3) / 2.0).ToList();
-      var mean = prices.Average();
-      var numerator = 0.0;
-      var denominator = 0.0;
-
-      for (int i = 1; i < prices.Count; i++) {
-         numerator += (prices[i] - mean) * (prices[i - 1] - mean);
-         denominator += Math.Pow(prices[i - 1] - mean, 2);
-      }
-
-      return denominator == 0 ? 0 : numerator / denominator;
-   }
-
-   /// <summary>
-   /// Estimates how quickly extreme Z-score values decay back toward the mean.
-   /// Shorter average decay time indicates stronger mean-reversion.
-   /// </summary>
-   public static double ComputeAverageZScoreDecay(List<Tuple<double, double, double, double>> spreadData, int zScoreWindow = 50) {
-      var prices = spreadData.Select(x => (x.Item1 + x.Item3) / 2.0).ToList();
-      if (prices.Count < zScoreWindow + 2) return double.NaN;
-
-      var decayTimes = new List<int>();
-      for (int i = zScoreWindow; i < prices.Count - 1; i++) {
-         var window = prices.Skip(i - zScoreWindow).Take(zScoreWindow).ToList();
-         double mean = window.Average();
-         double stdDev = Statistics.StandardDeviation(window);
-         if (stdDev == 0) continue;
-
-         double z = (prices[i] - mean) / stdDev;
-         if (Math.Abs(z) > 2) {
-            for (int j = 1; j < prices.Count - i; j++) {
-               double zNext = (prices[i + j] - mean) / stdDev;
-               if (Math.Abs(zNext) < 1) {
-                  decayTimes.Add(j);
-                  break;
-               }
-            }
-         }
-      }
-
-      return decayTimes.Count == 0 ? double.NaN : decayTimes.Average();
-   }
-
-   /// <summary>
-   /// Computes the variance ratio: short-term variance divided by long-term variance.
-   /// Lower ratio suggests stronger mean-reversion.
-   /// </summary>
-   public static double ComputeVarianceRatio(List<Tuple<double, double, double, double>> spreadData, int shortWindow, int longWindow) {
-      var prices = spreadData.Select(x => (x.Item1 + x.Item3) / 2.0).ToList();
-      if (prices.Count < longWindow) return double.NaN;
-
-      double shortVar = Statistics.Variance(prices.Skip(prices.Count - shortWindow).Take(shortWindow));
-      double longVar = Statistics.Variance(prices.Skip(prices.Count - longWindow).Take(longWindow));
-
-      return longVar == 0 ? double.NaN : shortVar / longVar;
-   }
-
-   /// <summary>
-   /// Computes the highest bid and the lowest ask from a spread tick series.
-   /// Useful for evaluating the best market opportunity range.
-   /// </summary>
-   /// <param name="spreadData">List of (bid, bidQty, ask, askQty)</param>
-   /// <returns>Tuple of (MaxBid, MinAsk)</returns>
-   public static (double MaxBid, double MinAsk) EvaluateMaxBidMinAsk(List<Tuple<double, double, double, double>> spreadData)
-{
-    if (spreadData == null || spreadData.Count == 0)
-        return (double.NaN, double.NaN);
-
-    double maxBid = double.NegativeInfinity;
-    double minAsk = double.PositiveInfinity;
-
-    foreach (var (bid, _, ask, _) in spreadData)
-    {
-        if (bid > maxBid) maxBid = bid;
-        if (ask < minAsk) minAsk = ask;
-    }
-
-    return (maxBid, minAsk);
-}
-
-/// <summary>
-/// Computes the distribution of spread mid-prices around the mean,
-/// grouped by caller-defined distance bands (in standard deviations).
-///
-/// This helps answer:
-/// - Is the spread tightly hugging the mean?
-/// - Are there wide excursions or heavy tails?
-///
-/// You define breakpoints like: [0.5, 1.0, 2.0] → it will return:
-/// - "≤ 0.5σ"
-/// - "0.5σ – 1.0σ"
-/// - "1.0σ – 2.0σ"
-/// - "> 2.0σ"
-///
-/// How to Interpret It:
-/// - ≤ 0.5σ        → Tight clustering near the mean (strong mean-reversion)
-/// - 0.5σ – 1.0σ   → Mild oscillation (common trade region)
-/// - 1.0σ – 2.0σ   → Moderate deviation (potential trade signals)
-/// - > 2.0σ        → Rare, extreme moves (tail events, potential entries/exits)
-///
-/// Use this to tune entry thresholds or detect regime shifts.
-/// </summary>
-/// <param name="spreadData">List of (bid, bidQty, ask, askQty)</param>
-/// <param name="regions">List of positive breakpoints in std deviations (e.g., [0.5, 1.0, 2.0])</param>
-/// <returns>Dictionary mapping label to % of mid-prices in that region</returns>
-public static Dictionary<string, double> EvaluateDistributionAroundMean(
-    List<Tuple<double, double, double, double>> spreadData,
-    List<double> regions)
-{
-    var result = new Dictionary<string, double>();
-
-    if (spreadData == null || spreadData.Count < 2 || regions == null || regions.Count == 0)
-        return result;
-
-    var mids = spreadData.Select(x => (x.Item1 + x.Item3) / 2.0).ToList();
-    double mean = mids.Average();
-    double stddev = Math.Sqrt(mids.Select(x => Math.Pow(x - mean, 2)).Average());
-
-    if (stddev == 0)
-    {
-        result["Zero Volatility"] = 100.0;
-        return result;
-    }
-
-    var buckets = new int[regions.Count + 1];
-    foreach (var mid in mids)
-    {
-        double distance = Math.Abs(mid - mean);
-        bool placed = false;
-
-        for (int i = 0; i < regions.Count; i++)
-        {
-            if (distance <= regions[i])
-            {
-                buckets[i]++;
-                placed = true;
-                break;
-            }
-        }
-
-        if (!placed)
-            buckets[regions.Count]++;
-    }
-
-    int total = mids.Count;
-    for (int i = 0; i < regions.Count; i++)
-    {
-        string label = i == 0
-            ? $"≤ {regions[i]:0.##}σ"
-            : $"{regions[i - 1]:0.##}σ – {regions[i]:0.##}σ";
-        result[label] = 100.0 * buckets[i] / total;
-    }
-
-    result[$"> {regions.Last():0.##}σ"] = 100.0 * buckets.Last() / total;
-
-    return result;
-}
-
-/// <summary>
-/// Computes two drift metrics indicating how much the spread's mean changes over time,
-/// based on the standard deviation of rolling mean values.
-///
-/// ✅ Absolute Drift: Standard deviation of the rolling mean
-/// ✅ Relative Drift: Drift normalized by the overall mean (unitless ratio)
-///
-/// This shows whether the center of the spread is stable — critical for mean-reversion.
-///
-/// How to Interpret the Output:
-/// - ✅ Relative Drift < 1% → Very stable mean — ideal
-/// - ⚠️ 1–5% → Mild drift — consider shorter holds or adaptive mean
-/// - ❌ >5% → Drifting mean — fixed mean may no longer work
-/// </summary>
-/// <param name="spreadData">List of (bid, bidQty, ask, askQty)</param>
-/// <param name="window">Window size for rolling mean (e.g., 20 ticks)</param>
-/// <returns>
-/// Tuple of (Absolute Drift, Relative Drift) — where:
-/// - Absolute Drift = StdDev of rolling means
-/// - Relative Drift = Drift / global mean
-/// </returns>
-public static (double AbsoluteDrift, double RelativeDrift) EvaluateMeanDriftScore(
-    List<Tuple<double, double, double, double>> spreadData,
-    int window = 20)
-{
-    if (spreadData == null || spreadData.Count < window)
-        return (double.NaN, double.NaN);
-
-    var mids = spreadData.Select(x => (x.Item1 + x.Item3) / 2.0).ToArray();
-    var rollingMeans = new List<double>();
-
-    for (int i = window; i <= mids.Length; i++)
-    {
-        double rollingMean = mids.Skip(i - window).Take(window).Average();
-        rollingMeans.Add(rollingMean);
-    }
-
-
-    double overallMean = mids.Average(); // use full mean for normalization
-    //double drift = Math.Sqrt(rollingMeans.Select(m => Math.Pow(m - rollingMeans.Average(), 2)).Average());
-    double drift = Math.Sqrt(rollingMeans.Select(m => Math.Pow(m - rollingMeans.Average(), 2)).Average());
-
-
-double relativeDrift = Math.Abs(overallMean) > 1e-8 ? drift / Math.Abs(overallMean) : double.NaN;
-    return (drift, relativeDrift);
-}
-
-/// <summary>
-/// Returns the latest Kalman Filter estimate of the true mean of the spread mid-prices.
-/// This tracks the spread's central value while allowing for gradual drift and suppressing noise.
-///
-/// Internally, this applies a recursive 1D Kalman filter over the input mid-prices.
-/// 
-/// Tuning Parameters:
-/// - Q: Process noise variance (higher = faster adaptation to drift)
-/// - R: Measurement noise variance (higher = more smoothing)
-/// 
-/// Use this instead of SMA if your mean drifts or if price series is noisy.
-///
-/// </summary>
-/// <param name="spreadData">List of (bid, bidQty, ask, askQty)</param>
-/// <param name="Q">Process noise variance (default = 0.0001)</param>
-/// <param name="R">Measurement noise variance (default = 0.01)</param>
-/// <returns>Latest estimated Kalman-filtered mean, or NaN if input is invalid</returns>
-public static double GetLatestKalmanMean(
-    List<Tuple<double, double, double, double>> spreadData,
-    double Q = 0.0001,
-    double R = 0.01)
-{
-    if (spreadData == null || spreadData.Count == 0)
-        return double.NaN;
-
-    var mids = spreadData.Select(x => (x.Item1 + x.Item3) / 2.0).ToList();
-
-    double x = mids[0];  // initial estimate of mean
-    double P = 1.0;       // initial uncertainty
-
-    for (int i = 1; i < mids.Count; i++)
-    {
-        // Predict step
-        double x_prior = x;
-        double P_prior = P + Q;
-
-        // Update step
-        double K = P_prior / (P_prior + R);
-        x = x_prior + K * (mids[i] - x_prior);
-        P = (1 - K) * P_prior;
-    }
-
-    return x; // the latest estimated mean
-}
-
-
-
-
-}
@@ -1,137 +0,0 @@
-namespace bts.SilverPine {
-   using BCG;
-
-   using System.Diagnostics;
-
-   public class SpreadPerformanceStream : ViewStream {
-      ViewStream _spread_price_stream = null;
-      private readonly IBlueTableSchema _schema;
-      private readonly int _window_secs;
-      private readonly double _buy_threshold, _sell_threshold, _flat_threshold;
-      private ViewStream _bbo_stream;
-      public SpreadPerformanceStream(string s1, string s2, double w1, double w2, int window_secs,
-                                       double buy_threshold, double sell_threshold, double flat_threshold)
-      : this(s1, s2, w1, w2, window_secs, buy_threshold, sell_threshold, flat_threshold, ViewStreamDatabase.instance) { }
-
-      public SpreadPerformanceStream(string s1, string s2, double w1, double w2, int window_secs,
-                                      double buy_threshold, double sell_threshold, double flat_threshold,
-                                      ViewStreamDatabase vsd) {
-         _bbo_stream = new BBOStream(vsd);
-         _schema = BlueDataUtility.form_schema("SpreadPerformance", Tuple.Create("id", TypeCode.Int32),                                                
-                                                Tuple.Create("t0", TypeCode.DateTime),
-                                                Tuple.Create("t1", TypeCode.DateTime),
-                                                Tuple.Create("n_ticks", TypeCode.UInt32),
-                                                Tuple.Create("curr_windows_seconds", TypeCode.Double),
-                                                // means / std dev                                          
-                                                Tuple.Create("stddev", TypeCode.Double),
-                                                Tuple.Create("mean", TypeCode.Double),
-                                                Tuple.Create("kalman_filter_mean", TypeCode.Double),
-                                                Tuple.Create("absolute_mean_drift", TypeCode.Double),
-                                                Tuple.Create("relative_mean_drift", TypeCode.Double),
-                                                // statistical measures
-                                                Tuple.Create("hurst_exponent", TypeCode.Double),
-                                                Tuple.Create("half_life", TypeCode.Double),
-                                                Tuple.Create("lag1_autocorrelation", TypeCode.Double),
-                                                Tuple.Create("average_zscore_decay", TypeCode.Double),
-                                                Tuple.Create("variance_ratio", TypeCode.Double),
-                                                Tuple.Create("n_zero_crossings", TypeCode.UInt32),                                                
-                                                // distributions
-                                                Tuple.Create("Z ≤ 0.5σ", TypeCode.Double),
-                                                Tuple.Create("0.5σ – 1.0σ", TypeCode.Double),
-                                                Tuple.Create("1.0σ – 2.0σ", TypeCode.Double),
-                                                Tuple.Create("> 2.0σ", TypeCode.Double),
-                                                ////////////////////////////////////////                                                   
-                                                Tuple.Create("max_bid", TypeCode.Double),
-                                                Tuple.Create("min_ask", TypeCode.Double),
-                                                Tuple.Create("n_buy_signals", TypeCode.UInt32),
-                                                Tuple.Create("cumm_buy_price", TypeCode.Double),
-                                                Tuple.Create("n_sell_signals", TypeCode.UInt32),
-                                                Tuple.Create("cumm_sell_price", TypeCode.Double)
-                                                //Tuple.Create("pnl", TypeCode.Double),
-                                                //Tuple.Create("buy_turns", TypeCode.UInt32),
-                                                //Tuple.Create("sell_turns", TypeCode.UInt32),
-                                                //Tuple.Create("trades", TypeCode.String)
-                                                );
-         _window_secs = window_secs;
-         _buy_threshold = buy_threshold;
-         _sell_threshold = sell_threshold;
-         _flat_threshold = flat_threshold;
-         _spread_price_stream = new SyntheticSpreadStream(s1, s2, w1, w2, _bbo_stream, vsd);
-         _spread_price_stream = new TimeSeriesStream(_spread_price_stream);
-         _spread_price_stream = new ThrottleStream(_spread_price_stream, TimeSpan.FromSeconds(5.0), vsd);
-      }
-
-      protected override void myOnFirstSubscriber() {
-         _spread_price_stream.onChange += _handle_spread_prices;
-      }
-
-      protected override void myOnLastSubscriber() {
-         _spread_price_stream.onChange -= _handle_spread_prices;
-      }
-
-      private void _handle_spread_prices(object dp_obj) {
-         DataPacket dp = dp_obj as DataPacket;
-         List<Tuple<DateTime, double, double, double, double>> spread_ts_prices = new List<Tuple<DateTime, double, double, double, double>>();
-         DateTime min_time = DateTime.MaxValue; DateTime max_time = DateTime.MinValue;
-         var sw = Stopwatch.StartNew();
-         sw.Start();
-         foreach (IBlueRow r in dp.newRows) {
-            DateTime ts = r.get_datetime("timestamp");
-            TimeSpan elapsed_time = DateTime.Now - ts;
-            if (elapsed_time.TotalSeconds >= _window_secs)
-               continue;
-            spread_ts_prices.Add(Tuple.Create(
-                                          r.get_datetime("timestamp"),
-                                          r.get<double>("spread_bid_price"),
-                                          r.get<double>("spread_bid_sz"),
-                                          r.get<double>("spread_ask_price"),
-                                          r.get<double>("spread_ask_sz")));
-            if (ts <= min_time) {
-               min_time = ts;
-            }
-            if (ts >= max_time) {
-               max_time = ts;
-            }
-         }
-         List<Tuple<double, double, double, double>> spread_prices = spread_ts_prices
-                                                                                             .OrderBy(t => t.Item1)
-                                                                                             .Select(t => Tuple.Create(t.Item2, t.Item3, t.Item4, t.Item5))
-                                                                                             .ToList();
-
-         uint n_ticks = (uint)spread_prices.Count;
-         double curr_windows_seconds = (max_time - min_time).TotalSeconds;
-         double hurst_exponent = SpreadPerformanceEvaluator.EvaluateHurstExponent(spread_prices);
-         double half_life = SpreadPerformanceEvaluator.ComputeHalfLife (spread_prices);
-         double lag1_autocorrelation = SpreadPerformanceEvaluator.ComputeLag1Autocorrelation(spread_prices);
-         double average_zscore_decay = SpreadPerformanceEvaluator.ComputeAverageZScoreDecay(spread_prices);
-         double variance_ratio = SpreadPerformanceEvaluator.ComputeVarianceRatio(spread_prices, 60, 360);
-
-         (double mean, double stddev) = SpreadPerformanceEvaluator.EvaluateMeanAndStdDev(spread_prices);
-         double kalman_filter_mean = SpreadPerformanceEvaluator.GetLatestKalmanMean(spread_prices);
-         uint n_zero_crossings = SpreadPerformanceEvaluator.EvaluateZeroCrossingCount(spread_prices);
-         (double max_bid, double min_ask) = SpreadPerformanceEvaluator.EvaluateMaxBidMinAsk(spread_prices);
-         (uint n_buy_signals, double cumm_buy_price, uint n_sell_signals, double cumm_sell_price) = SpreadPerformanceEvaluator.EvaluateExcursionCount(spread_prices, _buy_threshold);
-         //(double pnl, List<SpreadPerformanceEvaluator.TradeAction> trades, int buy_turns, int sell_turns) = SpreadPerformanceEvaluator.EvaluateSimulatedPnL(spread_prices, 30, _buy_threshold, _flat_threshold);
-         //string trade_str = String.Empty;
-         //foreach (var tr in trades) {
-         //   trade_str += tr.ToString() + " , ";
-         //}
-         Dictionary<string, double> dist_map = SpreadPerformanceEvaluator.EvaluateDistributionAroundMean(spread_prices,
-                                                                       new List<double>(new double[] { 0.5, 1.0, 2.0 }));
-         var dist_vals = dist_map.Count < 4 ? new double[] { Double.NaN, Double.NaN, Double.NaN, Double.NaN } : dist_map.Values.ToArray();
-         (double absolute_drift, double relative_drift) = SpreadPerformanceEvaluator.EvaluateMeanDriftScore(spread_prices);
-         IBlueRow nr = new BlueRow(_schema, 0, min_time, max_time, n_ticks, curr_windows_seconds, stddev, mean, kalman_filter_mean, absolute_drift, relative_drift,
-            hurst_exponent, half_life, lag1_autocorrelation, average_zscore_decay, variance_ratio, n_zero_crossings,
-                                    dist_vals[0], dist_vals[1], dist_vals[2], dist_vals[3],
-                                    max_bid, min_ask, n_buy_signals, cumm_buy_price, n_sell_signals, cumm_sell_price);
-         BlueTable t = new BlueTable(_schema);
-         t.addRow(nr);
-         var dp_out = BlueDataUtility.build_dp_inc(t, myLastValue as DataPacket, true);
-         var perf_msecs = sw.ElapsedMilliseconds;
-         sw.Stop();
-         Log.Gen.info($"SpreadPerformanceStream._handle_spread_prices comp_time: {perf_msecs} msecs");
-         myFireEvent(dp_out);
-
-      }
-   }
-}
@@ -1,68 +0,0 @@
-namespace bts.SilverPine{
-   using BCG;
-   public class SpreadStatArbsWeightLinearRegressionStream : ViewStream {
-      private IBlueTableSchema _schema;
-      private ViewStream _leg_price_stream;
-      private readonly double _window_secs;
-      private readonly string _s1, _s2;
-
-      public SpreadStatArbsWeightLinearRegressionStream(string s1, string s2, double window_secs, double sample_period_secs)
-         : this(s1, s2, window_secs, sample_period_secs, ViewStreamDatabase.instance) { }
-
-
-      public SpreadStatArbsWeightLinearRegressionStream(string s1, string s2, double window_secs, double sample_period_secs, ViewStreamDatabase vsd) {
-         _window_secs = window_secs;
-         _s1 = s1;
-         _s2 = s2;
-         _schema = BlueDataUtility.form_schema("LinearRegressionWeights",
-                                                Tuple.Create("id", TypeCode.Int32),
-                                                Tuple.Create("t0", TypeCode.DateTime),
-                                                Tuple.Create("t1", TypeCode.DateTime),
-                                                Tuple.Create("curr_windows_seconds", TypeCode.Double),
-                                                Tuple.Create("n_ticks", TypeCode.UInt32),
-                                                Tuple.Create("s1", TypeCode.String),
-                                                Tuple.Create("w1", TypeCode.Double),
-                                                Tuple.Create("s2", TypeCode.String),
-                                                Tuple.Create("w2", TypeCode.String));
-
-         _leg_price_stream = new TimeSeriesStream(new SyntheticSpreadLegsStream(s1, s2));
-         _leg_price_stream = new ThrottleStream(_leg_price_stream, TimeSpan.FromSeconds(1), vsd);
-      }
-
-      protected override void myOnFirstSubscriber() {
-         _leg_price_stream.onChange += _handle_leg_prices;
-      }
-
-      protected override void myOnLastSubscriber() {
-         _leg_price_stream.onChange -= _handle_leg_prices;
-      }
-
-      private void _handle_leg_prices(object dp_obj){
-         DataPacket dp = dp_obj as DataPacket;
-         List<Tuple<double, double, DateTime>> leg_prices = new List<Tuple<double, double, DateTime>>();
-         foreach (IBlueRow r in dp.newRows){
-            double s1_mid = (r.get<double>("s1_bid_price") + r.get<double>("s1_ask_price")) /2.0;
-            double s2_mid = (r.get<double>("s2_bid_price") + r.get<double>("s2_ask_price")) /2.0;
-            DateTime ts = r.get_datetime("timestamp");
-            leg_prices.Add(new Tuple<double, double, DateTime>(s1_mid, s2_mid, ts));
-         }
-         if (leg_prices.Count <= 2)
-            return;
-         leg_prices = leg_prices.Where(t => (DateTime.Now - t.Item3).TotalSeconds <= _window_secs).ToList();
-         DateTime t0 = leg_prices.Select(t => t.Item3).Min();
-         DateTime t1 = leg_prices.Select(t => t.Item3).Max();
-         double curr_windows_seconds = (t1 - t0).TotalSeconds;
-         int n_ticks = leg_prices.Count;
-         List<double> s1_mids = leg_prices.Select(t => t.Item1).ToList();
-         List<double> s2_mids = leg_prices.Select(t => t.Item2).ToList();
-         (double w1, double w2) = SpreadWeightCalculator.CalculateWeightsLinearRegression(s1_mids, s2_mids);
-         BlueTable state = new BlueTable(_schema);
-         IBlueRow nr = new BlueRow(_schema, 0, t0, t1, curr_windows_seconds, n_ticks,_s1, w1, _s2, w2);
-         state.addRow(nr);
-         var dp_out = BlueDataUtility.build_dp_inc(state, myLastValue as DataPacket, true);
-         myFireEvent(dp_out);
-
-      }
-   }
-}
-
@@ -1,143 +0,0 @@
-namespace bts.SilverPine {
-   using BCG;
-
-   using MathNet.Numerics.LinearAlgebra.Factorization;
-
-   using System;
-   using System.Collections.Generic;
-   using System.Diagnostics;
-   using System.Linq;
-   using System.Text;
-   using System.Threading.Tasks;
-   [DataStreamUserRespresentation("SpreadStatsStream")]
-   // This is the fundamental stream that computes the stats (including buy and sell zones) 
-   // of a spread. the legs and the weights are passed as args
-   public class SpreadStatsStream : ViewStream {
-      private ViewStream _spread_price_stream;
-      private readonly IBlueTableSchema _schema;
-      private readonly TimeSpan _window_sz;
-      private readonly string _spread;
-      public SpreadStatsStream(string s1, string s2, double w1, double w2, int rolling_window_sz_seconds, int window_sampling_sz_secs)
-         : this(s1, s2, w1, w2, rolling_window_sz_seconds, window_sampling_sz_secs, new BBOStream(ViewStreamDatabase.instance), ViewStreamDatabase.instance) { }
-      public SpreadStatsStream(string s1, string s2, double w1, double w2, int rolling_window_sz_seconds, int window_sampling_sz_secs, ViewStream bbo_stream, ViewStreamDatabase vsd) {
-         _window_sz = TimeSpan.FromSeconds(rolling_window_sz_seconds);
-         _spread = $"{w1} X {s1} - {w2} X {s2}";
-         _schema = new BlueTableSchema("SpreaddStats", new ColumnInfo ("id", 0, TypeCode.Int32));
-         _schema = _schema.add_column(TypeCode.String, "spread");
-         _schema = _schema.add_column(TypeCode.String, "window");
-         // basic stats
-         _schema = _schema.add_column(TypeCode.Double, "bid_mean");
-         _schema = _schema.add_column(TypeCode.Double, "ask_mean");
-         _schema = _schema.add_column(TypeCode.Double, "bid_stddev");
-         _schema = _schema.add_column(TypeCode.Double, "ask_stddev");
-         _schema = _schema.add_column(TypeCode.Double, "bid_skewness");
-         _schema = _schema.add_column(TypeCode.Double, "ask_skewness");
-         _schema = _schema.add_column(TypeCode.Double, "bid_kurtosis");
-         _schema = _schema.add_column(TypeCode.Double, "ask_kurtosis");
-         _schema = _schema.add_column(TypeCode.Double, "bid_max");
-         _schema = _schema.add_column(TypeCode.Double, "ask_max");
-         _schema = _schema.add_column(TypeCode.Double, "bid_min");
-         _schema = _schema.add_column(TypeCode.Double, "ask_min");
-         // bid bins
-         _schema = _schema.add_column(TypeCode.Double, "bid Z < -2σ");
-         _schema = _schema.add_column(TypeCode.Double, "bid -2σ ≤ Z < -1σ");
-         _schema = _schema.add_column(TypeCode.Double, "bid -1σ ≤ Z < 0σ");
-         // positive z's
-         _schema = _schema.add_column(TypeCode.Double, "bid 0σ ≤ Z < 1σ");
-         _schema = _schema.add_column(TypeCode.Double, "bid 1σ ≤ Z < 2σ");
-         _schema = _schema.add_column(TypeCode.Double, "bid Z ≥ 2σ");
-         // ask bins
-         _schema = _schema.add_column(TypeCode.Double, "ask Z < -2σ");
-         _schema = _schema.add_column(TypeCode.Double, "ask -2σ ≤ Z < -1σ");
-         _schema = _schema.add_column(TypeCode.Double, "ask -1σ ≤ Z < 0σ");
-         // positive z's
-         _schema = _schema.add_column(TypeCode.Double, "ask 0σ ≤ Z < 1σ");
-         _schema = _schema.add_column(TypeCode.Double, "ask 1σ ≤ Z < 2σ");
-         _schema = _schema.add_column(TypeCode.Double, "ask Z ≥ 2σ");
-         // opps
-         _schema = _schema.add_column(TypeCode.Int32, "n_buy_ticks");
-         _schema = _schema.add_column(TypeCode.Double, "cumm_buy_price");
-         _schema = _schema.add_column(TypeCode.Int32, "n_sell_ticks");
-         _schema = _schema.add_column(TypeCode.Double, "cumm_sell_price");
-         _schema = _schema.add_column(TypeCode.Int32, "n_trade_ticks");
-         _spread_price_stream = new SyntheticSpreadStream(s1, s2, w1, w2, bbo_stream, vsd);
-         _spread_price_stream = new TimeSeriesStream(_spread_price_stream);
-         double window_sampling_sz_smsecs = window_sampling_sz_secs * 1000.0 + new Random(DateTime.Now.Second).Next(-100, 100);
-         TimeSpan sampling_period = TimeSpan.FromMilliseconds(window_sampling_sz_smsecs);
-         _spread_price_stream = new ThrottleStream(_spread_price_stream, sampling_period, vsd);
-      }
-
-      protected override void myOnFirstSubscriber() {
-         _spread_price_stream.onChange += _handle_spread_prices;
-      }
-
-      protected override void myOnLastSubscriber() {
-         _spread_price_stream.onChange -= _handle_spread_prices;
-      }
-
-      private void _handle_spread_prices(object dp_obj) {
-         List<IBlueRow> rows = (dp_obj as DataPacket).newRows.to_list();
-         Func<IBlueRow, bool> row_filter = (r) => {
-            DateTime ts = r.get_datetime("timestamp");
-            DateTime start_time = DateTime.Now.Subtract(_window_sz);
-            return ts >= start_time;
-         };
-         var rolling_win_rows = rows.Where(r => row_filter(r));
-         var spread_prices = rolling_win_rows.Select(r => (
-                                          r.get_datetime("timestamp"),
-                                          r.get<double>("spread_bid_price"),
-                                          r.get<double>("spread_bid_sz"),
-                                          r.get<double>("spread_ask_price"),
-                                          r.get<double>("spread_ask_sz"))).ToList();
-         spread_prices.Sort((a,b) => a.Item1.CompareTo(b.Item1));
-         DateTime t0 = spread_prices.First().Item1, t1 = spread_prices.Last().Item1;
-         var tot_secs = (t1 - t0).TotalSeconds;
-         var stopwatch = Stopwatch.StartNew();
-         Log.Gen.info($"SpreadStatsStream.analyze_spread_quotes executed in {stopwatch.ElapsedMilliseconds} ms");
-         var result = SpreadPerformanceEvaluator.SpreadStatistics.analyze_spread_quotes(spread_prices);
-         (uint n_buy_signals, double cumm_buy_price, uint n_sell_signals, double cumm_sell_price) = 
-              SpreadPerformanceEvaluator.EvaluateExcursionCount(spread_prices.Select(t => Tuple.Create(t.Item2, t.Item3, t.Item4, t.Item5)).ToList(), 2.0);
-         IBlueRow nr = new BlueRow(_schema,
-            0,
-            _spread,
-            $"n_ticks: {spread_prices.Count} t0: {t0.ToLongTimeString()} -> t1: {t1.ToLongTimeString()} ({tot_secs}) secs",
-            result.bid_stats.Mean,
-            result.ask_stats.Mean,
-            result.bid_stats.StandardDeviation,
-            result.ask_stats.StandardDeviation,
-            result.bid_stats.Skewness,
-            result.ask_stats.Skewness,
-            result.bid_stats.Kurtosis,
-            result.ask_stats.Kurtosis,
-            result.bid_stats.Maximum,
-            result.ask_stats.Maximum,
-            result.bid_stats.Minimum,
-            result.ask_stats.Minimum,
-            // bid distribution
-            result.bid_zscore_distribution["Z < -2σ"],
-            result.bid_zscore_distribution["-2σ ≤ Z < -1σ"],
-            result.bid_zscore_distribution["-1σ ≤ Z < 0σ"],
-            result.bid_zscore_distribution["0σ ≤ Z < 1σ"],
-            result.bid_zscore_distribution["1σ ≤ Z < 2σ"],
-            result.bid_zscore_distribution["Z ≥ 2σ"],
-            // ask distributions
-            result.ask_zscore_distribution["Z < -2σ"],
-            result.ask_zscore_distribution["-2σ ≤ Z < -1σ"],
-            result.ask_zscore_distribution["-1σ ≤ Z < 0σ"],
-            result.ask_zscore_distribution["0σ ≤ Z < 1σ"],
-            result.ask_zscore_distribution["1σ ≤ Z < 2σ"],
-            result.ask_zscore_distribution["Z ≥ 2σ"],
-            n_buy_signals,
-            cumm_buy_price,
-            n_sell_signals,
-            cumm_sell_price,
-            n_buy_signals + n_sell_signals
-            );
-         stopwatch.Stop();
-         BlueTable t = new BlueTable(_schema);
-         t.addRow(nr);
-         var dp_out = BlueDataUtility.build_dp(t, myLastValue as DataPacket);
-         myFireEvent(dp_out);
-      }
-   }
-}
@@ -1,49 +0,0 @@
-using System;
-using System.Collections.Generic;
-using MathNet.Numerics.LinearRegression;
-
-namespace bts.SilverPine {
-
-   public static class SpreadWeightCalculator {
-      /// <summary>
-      /// Estimates hedge weights (w1, w2) such that the spread s = w1 * s1 - w2 * s2 
-      /// is potentially mean-reverting. Uses linear regression: s1 = beta * s2 + ε.
-      /// 
-      /// Assumes mid-prices are precomputed and aligned over the same time window.
-      /// </summary>
-      /// <param name="s1MidPrices">List of mid-prices for asset 1 (dependent variable)</param>
-      /// <param name="s2MidPrices">List of mid-prices for asset 2 (independent variable)</param>
-      /// <param name="includeIntercept">
-      /// Whether to include an intercept in the regression model (default: true).
-      /// </param>
-      /// <returns>(w1, w2): Tuple of hedge weights</returns>
-      public static (double w1, double w2) CalculateWeightsLinearRegression(
-          List<double> s1MidPrices,
-          List<double> s2MidPrices,
-          bool includeIntercept = true) {
-         if (s1MidPrices == null || s2MidPrices == null)
-            throw new ArgumentNullException("Price series cannot be null.");
-         if (s1MidPrices.Count != s2MidPrices.Count)
-            throw new ArgumentException("Price series must be of equal length.");
-         if (s1MidPrices.Count < 2)
-            throw new ArgumentException("Need at least two data points for regression.");
-
-         double[] x = s2MidPrices.ToArray(); // Independent variable
-         double[] y = s1MidPrices.ToArray(); // Dependent variable
-
-         double beta;
-
-         if (includeIntercept) {
-            // Returns (intercept, slope)
-            (double intercept, double slope) = SimpleRegression.Fit(x, y);
-            beta = slope;
-         } else {
-            // Returns only slope when forcing regression through origin
-            beta = SimpleRegression.FitThroughOrigin(x, y);
-         }
-
-         // Return fixed w1 = 1.0 and w2 = beta
-         return (1.0, beta);
-      }
-   }
-}
@@ -1,118 +0,0 @@
-// TODO: Remove
-//// namespace bts.SilverPine 
-////    using bts;
-////    using BCG;
-////    using MongoDB.Driver.Linq;
-////    using System.ComponentModel.DataAnnotations;
-////    using ZstdSharp.Unsafe;
-
-////    [DataStreamUserRespresentation("VWAP")]
-////    public class VWAPStream : ViewStream {
-////       // returns the VWAP stream for each contract 
-////       private static ViewStream build_vwap_stream(string symbol, DateTime begin_dt, DateTime end_dt) {
-////          var ohlcv_params = OHLCVStream.get_params(begin_dt, end_dt);
-////          ViewStream ohlc_str = new OHLCVStream(symbol, ohlcv_params);
-////          // stream is grouped per contract and then vwap is calculated per contract
-////          GlobalMapStream rs = new GlobalMapStream(ohlc_str, "ts_event", "contract",
-////          (rows, idx) => {
-////             Log.Gen.info($"VWAPStream.GlobalMapStream delegate n_rows: {rows.Count} idx: {idx}");
-////             IBlueRow ir = rows.ElementAt((int)idx);
-////             var sel_rows = rows.Skip((int)idx).Where(r => r.get_datetime("ts_event") >= end_dt && r.get_datetime("ts_event") <= begin_dt);
-////             var volumeprices = sel_rows.Select(r => Tuple.Create(r.get<int>("volume"), r.get<double>("close"))).ToList();
-////             double vwap = IndicatorCalc.calculate_VWAP(volumeprices);
-////             IBlueTableSchema schema = ir.schema.add_columns(Tuple.Create(typeof(double), "vwap"));
-////             MutableBlueRow nr = new MutableBlueRow(schema, ir);
-////             //vwap = -313.987;
-////             nr.setValue("vwap", vwap);
-////             return nr;
-////          });
-////          return rs;
-////       }
-////       // the args want dt1 > dt2, if its nto we, swap them
-////       static void swap_if_necessary(ref DateTime dt1, ref DateTime dt2) {
-////          if (dt1 < dt2) {
-////             DateTime aux = dt1;
-////             dt1 = dt2;
-////             dt2 = aux;
-////          }
-////       }
-
-////       // this assumes the ohlc_str has diff contracts
-////       public VWAPStream(string symbol_or_contract, DateTime begin_dt, DateTime end_dt) {
-////          swap_if_necessary(ref begin_dt, ref end_dt);
-////          Tuple<ContractUtils.ContractCategory, string> ccat_sym_contr = ContractUtils.get_symbol_category(symbol_or_contract);
-////          string symbol = String.Empty;
-////          symbol = ccat_sym_contr.Item2;
-////          var ohlcv_params = OHLCVStream.get_params(begin_dt, end_dt);
-////          ViewStream vwap_str = build_vwap_stream(symbol, begin_dt, end_dt);
-////          // vwap_str now has vwap for all contracts. if the arg was a symbol, we need to filter it for the active contract,
-////          // whihc we do with FutureActivesContractStream. If the arg was a contract we'll simply filter it
-////          ViewStream rs = vwap_str;
-////          switch (ccat_sym_contr.Item1) {
-////             case ContractUtils.ContractCategory.FutureString:
-////             case ContractUtils.ContractCategory.SpreadString:
-////                rs = new FilterStream(vwap_str, r => r.get<string>("contract").CompareTo(ccat_sym_contr.Item2) == 0);
-////                break;
-////             case ContractUtils.ContractCategory.Symbol:
-////                rs = new FuturesActiveContractStream(vwap_str);
-////                break;
-////          }
-////          rs = new FilterStream(rs, r => r.get_datetime("ts_event").CompareTo(begin_dt) == 0);
-////          rs = new NewProjectStream(rs, "! vwap");
-////          mySetRealSource(rs);
-////       }
-
-////       public VWAPStream(string symbol_or_contract, string begin_dt, string end_dt)
-////       :
-////       this(symbol_or_contract, DateTime.Parse(begin_dt), DateTime.Parse(end_dt)) { }
-////    }
-//// }
-
-//using System;
-
-//using BCG;
-
-//using bts.api;
-
-//namespace bts.SilverPine {
-//   /*
-//      ----------------------------------------------------------------------
-//                                  *Expression*
-//      ----------------------------------------------------------------------
-//          //nl("Find Settlement price of the SP500 future in the first week of 2023.")
-
-//          r=FutureHistoricalBars("ES", "One_Day", "1/1/2024", "2/1/2024")
-//          r=math(r, "vprice = volume * close")
-//          r=aggregate(r, "vprice:Sum, volume:Sum")
-//          r=math(r, "vwap = vprice_sum / volume_sum")
-//          r=newProject(r, "vwap !")
-
-//      ----------------------------------------------------------------------
-//                                    *Schema*
-//      ----------------------------------------------------------------------
-//          id                              System.Int32
-//          vwap                            System.Double
-//   */
-
-//   [DataStreamUserRespresentation("VWAP")]
-//   public class VWAPStream : ViewStream {
-//      public VWAPStream(string symbol_or_contract, DateTime beg_dt, DateTime end_dt)
-//      :
-//      this(symbol_or_contract, beg_dt, end_dt, ViewStreamDatabase.instance) { }
-
-//      public VWAPStream(string symbol_or_contract, DateTime beg_dt, DateTime end_dt, ViewStreamDatabase db) {
-//         //var fc = new ContractUtils.FutureContract(symbol_or_contract);
-//         ViewStream r = new FutureHistoricalBarsStream(symbol_or_contract, beg_dt.ToString(), end_dt.ToString());
-//         // since this is already filtered, we can groupBy iit without a groupBy column
-//         r = new MathStream(r,
-//                 "vprice = volume * close");
-//         r = new NonIncGroupByStream(r,
-//                                      "vprice:Sum, volume:Sum");
-//         r = new MathStream(r,
-//                             "vwap = vprice_sum / volume_sum");
-//         r = new NewProjectStream(r,
-//                                   "vwap !");
-//         mySetRealSource(r);
-//      }
-//   }
-//}
@@ -13,6 +13,7 @@ namespace bts.SilverPine {
   using Spark.Api.MarketData;

   using SecurityKey = System.UInt32;
+   using System.Data.Entity.Infrastructure;

   public class FakeBBOStream : ViewStream, IWorker {
      private string _symbol;
@@ -104,14 +105,14 @@ namespace bts.SilverPine {
   public class BBOStream : ViewStream, IWorker {
      private readonly TimeSpan _poll_period = TimeSpan.FromMilliseconds(500);
      private DateTime _last_poll_time = DateTime.MinValue;
-      private ViewStreamDatabase? _vsd;
-      IReadOnlyDictionary<SecurityKey, BookTuple>? _cached_state = null;
-      MarketDataClient? _md_client;
+      private ViewStreamDatabase _vsd;
+      private int _id = 0;
+      IReadOnlyDictionary<SecurityKey, BookTuple> _cached_state = null;
+      MarketDataClient _md_client;
      internal static IBlueTableSchema _bbo_schema = BlueDataUtility.form_schema("bbo",
                                                Tuple.Create("id", TypeCode.Int32),
                                                Tuple.Create("sec_key", TypeCode.UInt32),
                                                Tuple.Create("sec", TypeCode.String),
-                                                Tuple.Create("exchange", TypeCode.String),
                                                Tuple.Create("bid_price", TypeCode.Double),
                                                Tuple.Create("bid_sz", TypeCode.Single),
                                                Tuple.Create("ask_price", TypeCode.Double),
@@ -123,18 +124,17 @@ namespace bts.SilverPine {
      }

      public BBOStream(ViewStreamDatabase vsd) {
-         bts.Log.Gen.info("BBOStream.cons");
-         _vsd = vsd;
-         _md_client = new MarketDataClient(new MDSource(TimeSpan.FromSeconds(0.5)));
-         //FakeBBOStream rs = new FakeBBOStream("Future", "CME", "ES", 5000, 0.25, vsd);
-         //mySetRealSource(rs);
+         // bts.Log.Gen.info("BBOStream.cons");
+         // _vsd = vsd;
+         // _md_client = new MarketDataClient(new MDSource(TimeSpan.FromSeconds(0.5)));
+         FakeBBOStream rs = new FakeBBOStream("Future", "CME", "ES", 5000, 0.25, vsd);
+         mySetRealSource(rs);
      }

      protected override void myOnFirstSubscriber() {
         _vsd.addWorker(this);
         _md_client.books().Subscribe(_on_books);
-         _vsd?.addWorker(this);
-         _md_client?.books().Subscribe(_on_books);
+
      }

      private void _log_updates(TableUpdate<BookTuple, uint> update) {
@@ -144,24 +144,24 @@ namespace bts.SilverPine {
         }
         foreach (Change<BookTuple> c in update.changes) {
            var bt = c.New;
-            //bts.Log.Gen.info($"BBOStream changes: idx: {i} bid: {bt.book.bids[0].price} / {bt.book.bids[0].size} ask: {bt.book.asks[0].price} / {bt.book.asks[0].size}");
+            bts.Log.Gen.info($"BBOStream changes: idx: {i} bid: {bt.book.bids[0].price} / {bt.book.bids[0].size} ask: {bt.book.asks[0].price} / {bt.book.asks[0].size}");
         }
      }

      private void _on_books(TableUpdate<BookTuple, uint> update) {
         _log_updates(update);
         // just cacching in here, will populate the rows in _poll_scan and fire the datapacket
-         //bts.Log.Gen.info($"BBOStream._on_books state: {update.state.Count()} changes: {update.changes.Count()} added: {update.added.Count()}");
+         bts.Log.Gen.info($"BBOStream._on_books state: {update.state.Count()} changes: {update.changes.Count()} added: {update.added.Count()}");
         _cached_state = update.state;
      }

      protected override void myOnLastSubscriber() {
-         _vsd?.removeWorker(this);
+         _vsd.removeWorker(this);
      }

      private void _scan_bbo() {
         // this shouldn be made incremental using the TableUpdate
-         //bts.Log.Gen.info("BBOStream._scan_bbo");
+         bts.Log.Gen.info("BBOStream._scan_bbo");
         if (_cached_state == null)
            return;
         BlueTable new_state = new BlueTable(_bbo_schema);
@@ -170,23 +170,22 @@ namespace bts.SilverPine {
            var sec = kv.Value.security;
            int id = Convert.ToInt32(skey);
            BookTuple bt = kv.Value;
-            var venue = sec.Exchange;
-            IBlueRow r = new BlueRow(_bbo_schema, id, skey, sec.ToString(), venue.ToString(), bt.book.bids[0].price, bt.book.bids[0].size,
+            IBlueRow r = new BlueRow(_bbo_schema, id, skey, sec.ToString(), bt.book.bids[0].price, bt.book.bids[0].size,
                                     bt.book.asks[0].price, bt.book.asks[0].size);
            new_state.addRow(r);
         }
-         var dp = accumulate(myLastValue as DataPacket, new_state);
+         DataPacket dp = null;
+         if (myLastValue == null) {
+            dp = new DataPacket(new_state, new_state, new BlueTable(_bbo_schema));
+         } else {
+            DataPacket old_dp = myLastValue as DataPacket;
+            IBlueRowCollection added_rows, del_rows;
+            BlueDataUtility.compute_differences_fast(new_state, old_dp.newRows, out del_rows, out added_rows);
+            dp = new DataPacket(new_state, added_rows, del_rows);
+         }
         if (dp.addedRows.Count == 0 && dp.deletedRows.Count == 0)
            return;
         myFireEvent(dp);
-
-         static DataPacket accumulate(DataPacket? old_dp, BlueTable new_state) {
-            if (old_dp is null) {
-               return new(new_state, new_state, new BlueTable(_bbo_schema));
-            }
-            BlueDataUtility.compute_differences_fast(new_state, old_dp.newRows, out var del_rows, out var added_rows);
-            return new(new_state, added_rows, del_rows);
-         }
      }

      public void doWork() {
@@ -196,4 +195,4 @@ namespace bts.SilverPine {
         }
      }
   }
-}
+}
@@ -1,5 +1,4 @@
-namespace bts.BlueOrb {
-   using bts.SilverPine;
+namespace bts.SilverPine {
   using BCG;

   using System.Runtime.CompilerServices;
@@ -46,7 +45,7 @@ namespace bts.BlueOrb {
   }

   [DataStreamUserRespresentation("FuturesHistorical1DayBars")]
-   internal class FuturesHistorical1DayBarsStream : ViewStream {
+   public class FuturesHistorical1DayBarsStream : ViewStream {

      // this gets the raw bars for a symbol (perhaps make it into a raw bars stream?)
      public FuturesHistorical1DayBarsStream(string symbol) :
@@ -104,7 +103,7 @@ namespace bts.BlueOrb {
   }

   [DataStreamUserRespresentation("FuturesHistorical1MinBars")]
-   internal class FuturesHistorical1MinuteBarsStream : ViewStream {
+   public class FuturesHistorical1MinuteBarsStream : ViewStream {
      // same as the above but for one min
      // this gets the raw bars for a symbol
      public FuturesHistorical1MinuteBarsStream(string symbol) :
@@ -188,6 +187,46 @@ namespace bts.BlueOrb {
      }
   }

+   [DataStreamUserRespresentation("FuturePriceDiff")]
+   public class FutureHistoricalBarsPriceDiffStream : ViewStream {
+
+      public FutureHistoricalBarsPriceDiffStream(string future_contract, string dt1, string dt2)
+      :
+      this(new ContractUtils.FutureContract(future_contract), DateTime.Parse(dt1), DateTime.Parse(dt2), "close", ViewStreamDatabase.instance) {
+      }
+
+
+      public FutureHistoricalBarsPriceDiffStream(string future_contract, DateTime dt1, DateTime dt2)
+      :
+      this(new ContractUtils.FutureContract(future_contract), dt1, dt2, "close", ViewStreamDatabase.instance) {
+      }
+
+      public FutureHistoricalBarsPriceDiffStream(string future_contract, DateTime dt1, DateTime dt2, string col)
+      :
+      this(new ContractUtils.FutureContract(future_contract), dt1, dt2, col, ViewStreamDatabase.instance) {
+      }
+
+
+      public FutureHistoricalBarsPriceDiffStream(ContractUtils.FutureContract fc, DateTime dt1, DateTime dt2, string col, ViewStreamDatabase vsd) {
+         BarSize bsize = dt2 - dt1 >= TimeSpan.FromDays(1.0) ? BarSize.ONE_DAY : BarSize.ONE_MIN;
+         ViewStream rs = new FutureHistoricalBarsStream(fc, BarSize.ONE_DAY, new DateTimeRange(dt1, dt2));
+
+         rs = new ColumnDifferenceStream(rs, r => {
+            DateTime r_dt = r.get_datetime("ts_event");
+            var rv = dt1.Equals(r_dt);
+            return rv;
+         },
+         //r => dt2.Equals(r.get_datetime("ts_event")), col, col);
+         r => {
+            DateTime r_dt = r.get_datetime("ts_event");
+            var rv = dt2.Equals(r_dt);
+            return rv;
+         }, col, col);
+         rs = new NewProjectStream(rs, "diff !");
+         mySetRealSource(rs);
+      }
+   }
+
   // takes the contract name and expands it into multuiple columns corresponding to the contracts.
   // the contract columns are a union, of all properties with some of them empty
   public class ContractExpanderStream : ViewStream {
@@ -208,29 +247,3 @@ namespace bts.BlueOrb {
   }

 }
-
-namespace bts.BlueOrb {
-   using BCG;
-
-   using bts.api;
-
-   using Newtonsoft.Json.Serialization;
-
-   [DataStreamUserRespresentation("FuturesContinousContract")]
-   public class FuturesContinousContractStream : ViewStream {
-      // the argument is symbol, like "ES" etc and not the contract itself
-      public FuturesContinousContractStream(string symbol) : this(symbol, ViewStreamDatabase.instance) { }
-      public FuturesContinousContractStream(string symbol, ViewStreamDatabase db) {
-         symbol = symbol.EndsWith("*") ? symbol : symbol + "*";
-         ViewStream r = new FuturesHistorical1DayBarsStream(symbol);
-         r = new ContractExpanderStream(r, "symbol");
-         r = new FilterStream(r, "contract_type='Future'");
-         // finding the max volume across symbols, if we need to do this for open_interest, then join here 
-         // with the open interest stream on symbol/contract
-         r = new NonIncGroupByStream(r, "ts_event", "volume:Max");
-         r = new NewProjectStream(r, "ts_event symbol !");
-         r = new SortStream(r, "-ts_event");
-         mySetRealSource(r);
-      }
-   }
-}
Author	SHA1	Message	Date
John Colvin	0e08bad534	cleanup: Remove unnecessary includes	2025-03-04 05:02:35 +00:00
John Colvin	09573167b8	feat: Add binance gw for cross margin orders Spots only	2025-03-04 05:02:35 +00:00
John Colvin	caa03dcfa7	refactor: Rename user_data_streams dir, ns without plural 's' There is one user data stream for Binance Spots, and another for Usds Futures. We model both of these user data streams in separate folders, so each "user_data_stream" dir represents a single stream.	2025-03-03 19:32:16 +00:00
John Colvin	d010ba7ae7	refactor: Abstract out common binance gw code For all sub-venues to use	2025-03-03 19:32:16 +00:00
John Colvin	c60df8bcfa	refactor: Rename binance exe gw filenames so they are unique The debugger gets confused when a particular lib contains multiple source files with the same name (in different dirs).	2025-03-03 19:32:16 +00:00
John Colvin	634e69f72b	fix: Trim account name string when converting to Fill proto	2025-03-03 19:32:16 +00:00
John Colvin	2397a15b23	fix: Disable CURLOPT_FAILONERROR to always get response body Even when the http error code indicates failure. The payload may contain useful information about the error.	2025-03-03 19:32:16 +00:00
John Colvin	dfacf73289	Improve logging/error reporting in binance execution gw In the process, consolidate binance exe report stores into single class.	2025-03-03 19:32:16 +00:00
John Colvin	0339fe9749	cleanup: Specify bts account directly in binance gw config This eliminates the need for an ExchangeSubAccountResolver.	2025-03-03 19:32:16 +00:00
John Colvin	ff1d71a36d	cleanup: Replace includes w/ fwd declares in header	2025-03-03 19:32:16 +00:00
John Colvin	5b9fc54ea6	fix: Reject orders w/ invalid TimeInForce at binance Do this internally (before sending them to the exchange)	2025-03-03 19:32:16 +00:00
John Colvin	24f0eef2eb	fix: Report internal reject msgs to ExecutionReport For Binance execution gateways (spots and usds futures)	2025-03-03 19:32:16 +00:00
John Colvin	01e90c2497	Add reconnect behavior to WebsocketClient	2025-03-03 19:32:16 +00:00
John Colvin	a62b4af667	cleanup: Perform basic init in WebsocketClient ctr	2025-03-03 19:32:16 +00:00
John Colvin	3aebd25838	Move config profiles dir to server/sparkd	2025-03-03 19:32:16 +00:00