Reversehedging fixed

Format: ran clang format on file
Add: config changes to make server controller work and set up hk zen server
2025-04-08 22:13:33 +00:00 · 2025-04-07 15:39:57 +00:00 · 2025-04-07 15:33:26 +00:00 · 2025-04-05 23:38:05 +00:00 · 2025-04-05 15:38:30 +00:00
2327 changed files with 12219 additions and 37389 deletions
@@ -65,7 +65,7 @@ RUN \
   && apt-get install -y cmake

 # Dotnet
-RUN apt-get update && apt-get install -y dotnet8
+RUN apt-get update && apt-get install -y dotnet7

 # Node
 ARG node_major=18
@@ -34,7 +34,7 @@
 #   src/server/sparkd/Dockerfile
 SPARKD__IMAGE=sparkd
 # The config dir for this service to use
-SPARKD__CONFIG_DIR=src/app_sparkd/configurations/dev
+SPARKD__CONFIG_DIR=src/server/sparkd/configurations/dev
 # Mount a specific volume or dir to "~/sparkdata/data" inside the docker.
 # Defaults to a volume named "sparkd_<config_port>" (e.g., sparkd_8000).
 # SPARKD__DATA_DIR=/home/bts/sparkdata/data
@@ -44,6 +44,5 @@ SPARKD__CONFIG_PORT=8000
 SPARKD__QUICKBUS_PORT=8001
 SPARKD__SIM_FIX_PORT=8002
 SPARKD__SHM_SIZE=4gb
-SPARKD__PYTHON_REQUIREMENTS=requirements.txt

 # TODO: add config for other services (pubsub, md-publisher, webapp, api-server, etc.)
@@ -60,4 +60,3 @@ trace_data/
 *.Generated.cs
 *.lst
 .env.local
-.secrets
@@ -22,7 +22,7 @@
        "--config",
        "${workspaceFolder}/src/DataStreamSuite/DataStreamPublisher/Configurations/${input:dataStreamPublisherConfig}.json"
      ],
-      "cwd": "build/Debug/bin/net8.0",
+      "cwd": "build/Debug/bin/net7.0",
      "stopAtEntry": false,
      "console": "integratedTerminal"
    },
@@ -30,12 +30,12 @@
      "name": "(gdb) Launch",
      "type": "cppdbg",
      "request": "launch",
-      "program": "${workspaceFolder}/build/sparkd-debug-clang/src/app_sparkd/sparkd",
+      "program": "/home/bts/spark/build/sparkd-debug-clang/server/sparkd",
      "args": [
        "run"
      ],
      "stopAtEntry": false,
-      "cwd": "${workspaceFolder}/build/sparkd-debug-clang/src/app_sparkd/",
+      "cwd": "/home/bts/spark/build/sparkd-debug-clang/server",
      "environment": [],
      "externalConsole": false,
      "MIMode": "gdb",
@@ -82,7 +82,7 @@
          "--config",
          "${workspaceFolder}/src/DataStreamSuite/DataStreamQueryAnalyzer/Configurations/DataStreamQueryAnalyzer.json"
        ],
-        "cwd": "build/Debug/bin/net8.0-windows",
+        "cwd": "build/Debug/bin/net7.0-windows",
        "env": {
          "PYTHON_EXE": "${workspaceFolder}/.venv/Scripts/python.exe"
        },
@@ -106,7 +106,7 @@
        "--config",
        "${workspaceFolder}/src/DataStreamSuite/PubSubServer/Configurations/${input:pubSubServerConfig}.json"
      ],
-      "cwd": "build/Debug/bin/net8.0",
+      "cwd": "build/Debug/bin/net7.0",
      "stopAtEntry": false,
      "console": "integratedTerminal"
    },
@@ -118,7 +118,7 @@
      "windows": {
        "preLaunchTask": "Build Spark (Debug)",
        "program": "Spark.exe",
-        "cwd": "build/Debug/bin/net8.0-windows",
+        "cwd": "build/Debug/bin/net7.0-windows",
      },
      "stopAtEntry": false,
      "console": "integratedTerminal"
@@ -130,7 +130,7 @@
      "program": "",
      "windows": {
        "program": "Spark.exe",
-        "cwd": "build/Debug/bin/net8.0-windows",
+        "cwd": "build/Debug/bin/net7.0-windows",
      },
      "stopAtEntry": false,
      "console": "integratedTerminal"
@@ -147,7 +147,7 @@
          "--sparkd",
          "${input:sparkConfigServer}"
        ],
-        "cwd": "build/Debug/bin/net8.0-windows",
+        "cwd": "build/Debug/bin/net7.0-windows",
      },
      "stopAtEntry": false,
      "console": "integratedTerminal"
@@ -160,7 +160,7 @@
      "windows": {
        "preLaunchTask": "Build Spark (Debug)",
        "program": "Spark.Console.exe",
-        "cwd": "build/Debug/bin/net8.0",
+        "cwd": "build/Debug/bin/net7.0",
        "args": [
          "test",
          "${input:sparkTest}"
@@ -176,7 +176,7 @@
      "program": "",
      "windows": {
        "program": "Spark.Console.exe",
-        "cwd": "build/Debug/bin/net8.0",
+        "cwd": "build/Debug/bin/net7.0",
        "args": [
          "test",
          "${input:sparkTest}"
@@ -191,27 +191,24 @@
      "request": "launch",
      "program": "",
      "linux": {
-        "program": "${workspaceFolder}/build/sparkd-debug-clang/src/app_sparkd/sparkd",
-      },
-      "args": [
-        "run"
-      ],
-      "cwd": "${workspaceFolder}/build/sparkd-debug-clang/src/app_sparkd",
-      "stopAtEntry": false,
-      "externalConsole": false,
-      "MIMode": "gdb",
-      "setupCommands": [
-        {
-          "description": "Enable pretty-printing for gdb",
-          "text": "-enable-pretty-printing",
-          "ignoreFailures": true
-        },
-        {
-          "description": "Set Disassembly Flavor to Intel",
-          "text": "-gdb-set disassembly-flavor intel",
-          "ignoreFailures": true
-        }
-      ]
+        "preLaunchTask": "Build Sparkd (Debug)",
+        "program": "${workspaceFolder}/build/sparkd-debug-clang/server/sparkd",
+        "args": [
+          "run"
+        ],
+        "stopAtEntry": false,
+        "cwd": "${workspaceFolder}/build/sparkd-debug-clang/server",
+        "environment": [],
+        "externalConsole": false,
+        "MIMode": "gdb",
+        "setupCommands": [
+          {
+            "description": "Enable pretty-printing for gdb",
+            "text": "-enable-pretty-printing",
+            "ignoreFailures": true
+          }
+        ]
+      }
    },
    {
      "name": "Sparkd (Skip Build)",
@@ -219,27 +216,74 @@
      "request": "launch",
      "program": "",
      "linux": {
-        "program": "${workspaceFolder}/build/sparkd-debug-clang/src/app_sparkd/sparkd",
-      },
-      "args": [
-        "run"
-      ],
-      "cwd": "${workspaceFolder}/build/sparkd-debug-clang/src/app_sparkd",
-      "stopAtEntry": false,
-      "externalConsole": false,
-      "MIMode": "gdb",
-      "setupCommands": [
-        {
-          "description": "Enable pretty-printing for gdb",
-          "text": "-enable-pretty-printing",
-          "ignoreFailures": true
-        },
-        {
-          "description": "Set Disassembly Flavor to Intel",
-          "text": "-gdb-set disassembly-flavor intel",
-          "ignoreFailures": true
-        }
-      ]
+        "program": "${workspaceFolder}/build/sparkd-debug-clang/server/sparkd",
+        "args": [
+          "run"
+        ],
+        "stopAtEntry": false,
+        "cwd": "${workspaceFolder}/build/sparkd-debug-clang/server",
+        "environment": [],
+        "externalConsole": false,
+        "MIMode": "gdb",
+        "setupCommands": [
+          {
+            "description": "Enable pretty-printing for gdb",
+            "text": "-enable-pretty-printing",
+            "ignoreFailures": true
+          }
+        ]
+      }
+    },
+    {
+      "name": "Sparkd Test",
+      "type": "cppdbg",
+      "request": "launch",
+      "program": "",
+      "linux": {
+        "preLaunchTask": "Build Sparkd (Debug)",
+        "program": "${workspaceFolder}/build/sparkd-debug-clang/server/sparkd",
+        "args": [
+          "test",
+          "${input:sparkdTest}",
+        ],
+        "stopAtEntry": false,
+        "cwd": "${workspaceFolder}/build/sparkd-debug-clang/server",
+        "environment": [],
+        "externalConsole": false,
+        "MIMode": "gdb",
+        "setupCommands": [
+          {
+            "description": "Enable pretty-printing for gdb",
+            "text": "-enable-pretty-printing",
+            "ignoreFailures": true
+          }
+        ]
+      }
+    },
+    {
+      "name": "Sparkd Test (Skip Build)",
+      "type": "cppdbg",
+      "request": "launch",
+      "program": "",
+      "linux": {
+        "program": "${workspaceFolder}/build/sparkd-debug-clang/server/sparkd",
+        "args": [
+          "test",
+          "${input:sparkdTest}",
+        ],
+        "stopAtEntry": false,
+        "cwd": "${workspaceFolder}/build/sparkd-debug-clang/server",
+        "environment": [],
+        "externalConsole": false,
+        "MIMode": "gdb",
+        "setupCommands": [
+          {
+            "description": "Enable pretty-printing for gdb",
+            "text": "-enable-pretty-printing",
+            "ignoreFailures": true
+          }
+        ]
+      }
    },
  ],
  "inputs": [
@@ -282,4 +326,4 @@
      "description": "Test to run",
    },
  ]
-}
+}
@@ -363,7 +363,7 @@
          "${workspaceFolder}/src/DataStreamSuite/DataStreamQueryAnalyzer/Configurations/DataStreamQueryAnalyzer.json",
        ],
        "options": {
-          "cwd": "${workspaceFolder}/build/${input:buildConfiguration}/bin/net8.0-windows"
+          "cwd": "${workspaceFolder}/build/${input:buildConfiguration}/bin/net7.0-windows"
        },
      },
      "icon": {
@@ -383,7 +383,7 @@
          "${workspaceFolder}/src/DataStreamSuite/DataStreamQueryAnalyzer/Configurations/DataStreamQueryAnalyzer.json",
        ],
        "options": {
-          "cwd": "${workspaceFolder}/build/Debug/bin/net8.0-windows"
+          "cwd": "${workspaceFolder}/build/Debug/bin/net7.0-windows"
        },
      },
      "icon": {
@@ -403,7 +403,7 @@
          "${workspaceFolder}/src/DataStreamSuite/DataStreamQueryAnalyzer/Configurations/DataStreamQueryAnalyzer.json",
        ],
        "options": {
-          "cwd": "${workspaceFolder}/build/Release/bin/net8.0-windows"
+          "cwd": "${workspaceFolder}/build/Release/bin/net7.0-windows"
        },
      },
      "icon": {
@@ -427,7 +427,7 @@
        "${workspaceFolder}/src/DataStreamSuite/DataStreamPublisher/Configurations/${input:dataStreamPublisherConfig}.json",
      ],
      "options": {
-        "cwd": "${workspaceFolder}/build/${input:buildConfiguration}/bin/net8.0"
+        "cwd": "${workspaceFolder}/build/${input:buildConfiguration}/bin/net7.0"
      },
      "icon": {
        "id": "run",
@@ -450,7 +450,7 @@
        "${workspaceFolder}/src/DataStreamSuite/DataStreamPublisher/Configurations/${input:dataStreamPublisherConfig}.json",
      ],
      "options": {
-        "cwd": "${workspaceFolder}/build/Debug/bin/net8.0"
+        "cwd": "${workspaceFolder}/build/Debug/bin/net7.0"
      },
      "icon": {
        "id": "run",
@@ -473,7 +473,7 @@
        "${workspaceFolder}/src/DataStreamSuite/DataStreamPublisher/Configurations/${input:dataStreamPublisherConfig}.json",
      ],
      "options": {
-        "cwd": "${workspaceFolder}/build/Release/bin/net8.0"
+        "cwd": "${workspaceFolder}/build/Release/bin/net7.0"
      },
      "icon": {
        "id": "run",
@@ -496,7 +496,7 @@
        "${workspaceFolder}/src/DataStreamSuite/PubSubServer/Configurations/${input:pubSubServerConfig}.json",
      ],
      "options": {
-        "cwd": "${workspaceFolder}/build/${input:buildConfiguration}/bin/net8.0"
+        "cwd": "${workspaceFolder}/build/${input:buildConfiguration}/bin/net7.0"
      },
      "icon": {
        "id": "run",
@@ -519,7 +519,7 @@
        "${workspaceFolder}/src/DataStreamSuite/PubSubServer/Configurations/${input:pubSubServerConfig}.json",
      ],
      "options": {
-        "cwd": "${workspaceFolder}/build/Debug/bin/net8.0"
+        "cwd": "${workspaceFolder}/build/Debug/bin/net7.0"
      },
      "icon": {
        "id": "run",
@@ -541,7 +541,7 @@
        "${workspaceFolder}/src/DataStreamSuite/PubSubServer/Configurations/${input:pubSubServerConfig}.json",
      ],
      "options": {
-        "cwd": "${workspaceFolder}/build/Debug/bin/net8.0"
+        "cwd": "${workspaceFolder}/build/Debug/bin/net7.0"
      },
      "icon": {
        "id": "run",
@@ -564,7 +564,7 @@
        "${workspaceFolder}/src/DataStreamSuite/PubSubServer/Configurations/${input:pubSubServerConfig}.json",
      ],
      "options": {
-        "cwd": "${workspaceFolder}/build/Release/bin/net8.0"
+        "cwd": "${workspaceFolder}/build/Release/bin/net7.0"
      },
      "icon": {
        "id": "run",
@@ -586,7 +586,7 @@
        "${workspaceFolder}/src/DataStreamSuite/PubSubServer/Configurations/${input:pubSubServerConfig}.json",
      ],
      "options": {
-        "cwd": "${workspaceFolder}/build/Release/bin/net8.0"
+        "cwd": "${workspaceFolder}/build/Release/bin/net7.0"
      },
      "icon": {
        "id": "run",
@@ -601,7 +601,7 @@
      "windows": {
        "command": "Spark.exe",
        "options": {
-          "cwd": "${workspaceFolder}/build/${input:buildConfiguration}/bin/net8.0-windows"
+          "cwd": "${workspaceFolder}/build/${input:buildConfiguration}/bin/net7.0-windows"
        },
      },
      "icon": {
@@ -617,7 +617,7 @@
      "windows": {
        "command": "Spark.exe",
        "options": {
-          "cwd": "${workspaceFolder}/build/Debug/bin/net8.0-windows"
+          "cwd": "${workspaceFolder}/build/Debug/bin/net7.0-windows"
        },
      },
      "icon": {
@@ -633,7 +633,7 @@
      "windows": {
        "command": "Spark.exe",
        "options": {
-          "cwd": "${workspaceFolder}/build/Release/bin/net8.0-windows"
+          "cwd": "${workspaceFolder}/build/Release/bin/net7.0-windows"
        },
      },
      "icon": {
@@ -648,7 +648,7 @@
      "windows": {
        "command": "Spark.exe",
        "options": {
-          "cwd": "${workspaceFolder}/build/${input:buildConfiguration}/bin/net8.0-windows"
+          "cwd": "${workspaceFolder}/build/${input:buildConfiguration}/bin/net7.0-windows"
        },
      },
      "icon": {
@@ -673,7 +673,7 @@
        "${input:sparkConfigQuery}",
      ],
      "options": {
-        "cwd": "${workspaceFolder}/build/${input:buildConfiguration}/bin/net8.0"
+        "cwd": "${workspaceFolder}/build/${input:buildConfiguration}/bin/net7.0"
      },
      "icon": {
        "id": "run",
@@ -754,7 +754,7 @@
          "${input:sparkTest}"
        ],
        "options": {
-          "cwd": "${workspaceFolder}/build/Debug/bin/net8.0",
+          "cwd": "${workspaceFolder}/build/Debug/bin/net7.0",
        }
      },
    },
@@ -22,9 +22,9 @@ set(
 # can run simultaneously without fighting over the same shm files (e.g., sparkd,
 # api server, etc.).
 set(SHARED_MEMORY_PREFIX "bts" CACHE STRING "Prefix to use when creating shared memory files")
-set(EXTERNALS_BIN "${EXTERNALS_ROOT}/bin")
-set(EXTERNALS_INCLUDE "${EXTERNALS_ROOT}/include")
-set(EXTERNALS_LIB "${EXTERNALS_ROOT}/lib/$<CONFIG>")
+set(SPARK_BIN "${EXTERNALS_ROOT}/bin")
+set(SPARK_INCLUDE "${EXTERNALS_ROOT}/include")
+set(SPARK_LIB "${EXTERNALS_ROOT}/lib/$<CONFIG>")
 set(BUILD_DOCS OFF CACHE BOOL "Build documentation")
 set(BUILD_SDK OFF CACHE BOOL "Build the sdk")
 set(EXPORT_MD_CONSUMER OFF CACHE BOOL "Export (extern) C functions for MD consumer interop")
@@ -36,8 +36,8 @@ else()
  set(BUILD_ADMIN_WEB_APP ON CACHE BOOL "Build the Admin WebApp")
 endif()

-include_directories(BEFORE SYSTEM ${EXTERNALS_INCLUDE} ${EXTERNALS_ROOT}/include/zlib)
-link_directories(${EXTERNALS_LIB})
+include_directories(BEFORE SYSTEM ${SPARK_INCLUDE} ${EXTERNALS_ROOT}/include/zlib)
+link_directories(${SPARK_LIB})

 add_compile_definitions(SHARED_MEMORY_PREFIX="${SHARED_MEMORY_PREFIX}")
 add_compile_definitions(APP_DATA_DIR="${APP_DATA_DIR}")
@@ -47,8 +47,12 @@ endif()
 add_compile_definitions(EXPORT_MD_CONSUMER="${EXPORT_MD_CONSUMER}")
 if(WIN32)
  # subdirs
-  add_subdirectory(src/lib_spark)
+  add_subdirectory(src/lib)
 else()
+  # make sparkd look for libspark.so in its directory
+  set(CMAKE_BUILD_WITH_INSTALL_RPATH TRUE)
+  set(CMAKE_INSTALL_RPATH ".")
+
  if(BUILD_SPARKD)
    # Note: All macros that begin w/ 'CCAPI' are ccapi's macros (defined and/or
    # read by their code). We override some below.
@@ -113,15 +117,12 @@ else()
  endif()

  # subdirs
-  add_subdirectory(src/lib_spark)
  if (BUILD_SPARKD)
-    add_subdirectory(src/lib_python_interop)
-    add_subdirectory(src/lib_onixs_fix)
    add_subdirectory(src/lib_binance_execution)
-    add_subdirectory(src/lib_okx_execution)
-    add_subdirectory(src/app_sparkd)
+    add_subdirectory(src/server/sparkd)
    add_subdirectory(src/data_files)
  endif()
+  add_subdirectory(src/lib)
  if (BUILD_ADMIN_WEB_APP)
    add_subdirectory(src/Spark.Admin.WebApp)
  endif()
@@ -131,4 +132,37 @@ else()
  if(BUILD_SDK)
    add_subdirectory(src/sdk)
  endif()
+
+  if(BUILD_SPARKD)
+    add_custom_command(
+      OUTPUT ${CMAKE_BINARY_DIR}/server
+      COMMAND mkdir -p ${CMAKE_BINARY_DIR}/server
+      COMMAND ln ARGS -sf ${CMAKE_BINARY_DIR}/src/server/sparkd/sparkd ${CMAKE_BINARY_DIR}/server
+      COMMAND ln ARGS -sf ${CMAKE_BINARY_DIR}/data_files ${CMAKE_BINARY_DIR}/server
+      COMMAND ln ARGS -sf ${CMAKE_BINARY_DIR}/src/lib/libspark.so ${CMAKE_BINARY_DIR}/server
+      COMMAND ln ARGS -sf ${EXTERNALS_ROOT}/lib/${CMAKE_BUILD_TYPE}/libonixs-fix-engine.so.4 ${CMAKE_BINARY_DIR}/server
+      
+      COMMAND
+        ln ARGS -sf
+        ${EXTERNALS_ROOT}/lib/Release/libOnixS.IceImpactMulticastPriceFeedHandlerCpp.so.3.16.0.0
+        ${CMAKE_BINARY_DIR}/server
+      COMMAND ln ARGS -sf ${EXTERNALS_ROOT}/lib/Release/libACE-5.7.5.so ${CMAKE_BINARY_DIR}/server
+      COMMAND mkdir ARGS -p ${CMAKE_BINARY_DIR}/server/plugins
+      COMMENT "Creating server runtime layout"
+      DEPENDS sparkd data_files_dir)
+
+    add_custom_target(server_layout ALL DEPENDS ${CMAKE_BINARY_DIR}/server)
+
+    install(
+      DIRECTORY "${CMAKE_BINARY_DIR}/server/plugins"
+      DESTINATION "${CMAKE_INSTALL_BINDIR}")
+    install(
+      FILES
+      "${SPARK_LIB}/libACE-5.7.5.so"
+      "${SPARK_LIB}/libOnixS.IceImpactMulticastPriceFeedHandlerCpp.so.3.16.0.0"
+      "${SPARK_LIB}/libonixs-fix-engine.so.4.10.1"
+      "${SPARK_LIB}/libonixs-fix-engine.so.4"
+      "${SPARK_LIB}/libonixs-fix-engine.so"
+      DESTINATION "${CMAKE_INSTALL_BINDIR}")
+  endif()
 endif()
@@ -31,8 +31,9 @@
        "CMAKE_SHARED_LINKER_FLAGS_RELEASE": "/OPT:REF /OPT:ICF /LTCG",
        "CMAKE_TOOLCHAIN_FILE": "${sourceDir}/vcpkg/scripts/buildsystems/vcpkg.cmake",
        "EXTERNALS_ROOT": "${sourceDir}/externals/windows",
-        "CONFIG_DEFAULTS_PATH": "spark/config/profiles/default/*.conf",
-        "DOTNET_BINARY_DIR": "${sourceDir}/build/$<CONFIG>/bin/net8.0;${sourceDir}/build/$<CONFIG>/bin/net8.0-windows",
+        "CONFIG_DEFAULTS_PATH": "config/profiles/default/*.conf",
+        "DOTNET_BINARY_DIR": "${sourceDir}/build/$<CONFIG>/bin/net7.0",
+        "DOTNET_WINDOWS_BINARY_DIR": "${sourceDir}/build/$<CONFIG>/bin/net7.0-windows",
        "EXPORT_MD_CONSUMER": {
          "type": "BOOL",
          "value": "ON"
@@ -57,18 +58,12 @@
        "APP_DATA_DIR_OVERRIDE": "${dollar}{SPARKD_DATA_DIR}",
        "CMAKE_BUILD_WITH_INSTALL_RPATH": {
          "type": "BOOL",
-          "value": "OFF"
+          "value": "ON"
        },
        "CMAKE_CXX_STANDARD": "20",
        "CMAKE_INSTALL_RPATH": ".",
        "CMAKE_INSTALL_PREFIX": "${sourceDir}/install/${presetName}",
-        "CMAKE_PREFIX_PATH": "${sourceDir}/cmake/;${sourceDir}/.venv/lib/python3.10/site-packages/pybind11/share/cmake/pybind11",
        "CMAKE_EXE_LINKER_FLAGS": "-Wno-undef",
-        "CMAKE_TOOLCHAIN_FILE": "${sourceDir}/vcpkg/scripts/buildsystems/vcpkg.cmake",
-        "DATABENTO_USE_EXTERNAL_JSON": {
-          "type": "BOOL",
-          "value": "ON"
-        },
        "EXTERNALS_ROOT": "${sourceDir}/externals/linux",
        "BUILD_SDK": {
          "type": "BOOL",
@@ -76,7 +71,7 @@
        },
        "BUILD_ADMIN_WEB_APP": {
          "type": "BOOL",
-          "value": "OFF"
+          "value": "ON"
        },
        "SHARED_MEMORY_PREFIX": "sparkd"
      }
@@ -93,7 +88,7 @@
          "value": "ON"
        },
        "CCAPI_LOG_LEVEL": "Info",
-        "CONFIG_DEFAULTS_PATH": "spark/config/profiles/md_dev/*.conf;spark/config/profiles/default/*.conf"
+        "CONFIG_DEFAULTS_PATH": "config/profiles/md_dev/*.conf;config/profiles/default/*.conf"
      }
    },

@@ -104,7 +99,7 @@
      "cacheVariables": {
        "CMAKE_BUILD_TYPE": "Release",
        "CCAPI_LOG_LEVEL": "Warn",
-        "CONFIG_DEFAULTS_PATH": "spark/config/profiles/md_prod/*.conf;spark/config/profiles/default/*.conf"
+        "CONFIG_DEFAULTS_PATH": "config/profiles/md_prod/*.conf;config/profiles/default/*.conf"
      }
    },

@@ -186,12 +181,8 @@
        "CMAKE_CXX_FLAGS_DEBUG": "-g -fPIC -Wall -pedantic -Wextra -Werror -Wno-missing-braces -Wno-ignored-qualifiers --std=c++20 -fdiagnostics-color -DDEBUG",
        "CMAKE_CXX_FLAGS_RELEASE": "-fPIC -Wall -pedantic -Wextra -Werror -Wno-missing-braces -Wno-ignored-qualifiers --std=c++20 -fdiagnostics-color -fno-omit-frame-pointer -DNDEBUG -O3",
        "CMAKE_INSTALL_PREFIX": "${sourceDir}/install/${presetName}",
-        "CMAKE_EXPORT_COMPILE_COMMANDS": {
-          "type": "BOOL",
-          "value": "ON"
-        },
-        "CONFIG_DEFAULTS_PATH": "spark/config/profiles/default/*.conf",
-        "DOTNET_BINARY_DIR": "${sourceDir}/build/$<CONFIG>/bin/net8.0",
+        "CONFIG_DEFAULTS_PATH": "config/profiles/default/*.conf",
+        "DOTNET_BINARY_DIR": "${sourceDir}/build/$<CONFIG>/bin/net7.0",
        "EXPORT_MD_CONSUMER": {
          "type": "BOOL",
          "value": "ON"
@@ -231,7 +222,7 @@
      "cacheVariables": {
        "APP_DATA_DIR": "${dollar}{APPDATA}/bts/DataStream",
        "APP_DATA_DIR_OVERRIDE": "${dollar}{DATA_STREAM_DATA_DIR}",
-        "CONFIG_DEFAULTS_PATH": "spark/config/profiles/default/*.conf;spark/config/profiles/data_stream/*.conf",
+        "CONFIG_DEFAULTS_PATH": "config/profiles/default/*.conf;config/profiles/data_stream/*.conf",
        "SHARED_MEMORY_PREFIX": "data_stream"
      }
    },
@@ -266,7 +257,8 @@
        "CMAKE_CXX_COMPILER": "clang++",
        "CMAKE_CXX_FLAGS_DEBUG": "-g -fPIC -Wall -pedantic -Wextra -Werror -Wno-missing-braces -Wno-ignored-qualifiers --std=c++20 -fdiagnostics-color -DDEBUG",
        "CMAKE_CXX_FLAGS_RELEASE": "-fPIC -Wall -pedantic -Wextra -Werror -Wno-missing-braces -Wno-ignored-qualifiers --std=c++20 -fdiagnostics-color -fno-omit-frame-pointer -DNDEBUG -O3",
-        "CONFIG_DEFAULTS_PATH": "spark/config/profiles/default/*.conf;spark/config/profiles/data_stream/*.conf",
+        "CMAKE_INSTALL_PREFIX": "${sourceDir}/install/${presetName}",
+        "CONFIG_DEFAULTS_PATH": "config/profiles/default/*.conf;config/profiles/data_stream/*.conf",
        "DOTNET_BINARY_DIR": "${sourceDir}/build/$<CONFIG>/bin/net7.0",
        "EXPORT_MD_CONSUMER": {
          "type": "BOOL",
@@ -1,7 +0,0 @@
-include_guard(GLOBAL)
-
-macro(export_to_parent)
-  foreach(var IN ITEMS ${ARGN})
-    set(${var} ${${var}} PARENT_SCOPE)
-  endforeach()
-endmacro()
@@ -1,142 +0,0 @@
-include_guard(DIRECTORY)
-include(${CMAKE_SOURCE_DIR}/cmake/ExportToParent.cmake)
-include(${CMAKE_SOURCE_DIR}/cmake/InstallBasic.cmake)
-include(${CMAKE_SOURCE_DIR}/cmake/ProtobufGenerateCpp.cmake)
-include(${CMAKE_SOURCE_DIR}/cmake/SetFirstDefined.cmake)
-include(${CMAKE_SOURCE_DIR}/cmake/safe_dep_glob.cmake)
-
-function(init_library)
-  set(_options SHARED STATIC)
-  set(_one_value_args
-    NAME
-    SOURCE_DIR
-    INCLUDE_DIR
-  )
-  set(_multi_value_args)
-  cmake_parse_arguments(_ARGS "${_options}" "${_one_value_args}" "${_multi_value_args}" ${ARGN})
-  if (_ARGS_SHARED AND _ARGS_STATIC)
-    message(FATAL_ERROR "Invalid args passed to init_library: cannot specify STATIC and SHARED")
-  endif()
-  set_first_defined(_name _ARGS_NAME PROJECT_NAME)
-  set_first_defined(_source_dir _ARGS_SOURCE_DIR _name)
-  set_first_defined(_include_dir _ARGS_INCLUDE_DIR _source_dir)
-
-  # Cpp sources
-  file(
-    GLOB_RECURSE ${_name}_SOURCES
-    RELATIVE ${CMAKE_CURRENT_SOURCE_DIR}
-    ${_source_dir}/*.cpp
-  )
-  # Note: cpp files inside 'tests' dir(s) are considered tests.
-  # Remove tests from main sources, and add them to test sources
-  foreach(item ${${_name}_SOURCES})
-    if(${item} MATCHES "tests/.*\.cpp$")
-      list(APPEND ${_name}_TEST_SOURCES ${item})
-    endif()
-  endforeach()
-  foreach(item ${${_name}_TEST_SOURCES})
-    list(REMOVE_ITEM ${_name}_SOURCES ${item})
-  endforeach()
-
-  # Proto sources 
-  file(
-    GLOB_RECURSE ${_name}_PROTO_SOURCES
-    RELATIVE ${CMAKE_CURRENT_SOURCE_DIR}
-    ${_source_dir}/protobufs/*.proto
-  )
-  update_deps_file("${${_name}_PROTO_SOURCES};")
-  protobuf_generate_cpp(
-    CXX_SOURCES ${_name}_CXX_PROTO_SOURCES
-    CXX_HEADERS ${_name}_CXX_PROTO_HEADERS
-    CXX_OUT_DIR ${CMAKE_CURRENT_BINARY_DIR}
-    PROTO_INCLUDE_DIRS
-      ${EXTERNALS_INCLUDE}
-      ${CMAKE_CURRENT_SOURCE_DIR}
-    PROTO_SOURCES
-      ${${_name}_PROTO_SOURCES}
-  )
-
-  export_to_parent(
-    ${_name}_SOURCES
-    ${_name}_TEST_SOURCES
-    ${_name}_PROTO_SOURCES
-    ${_name}_CXX_PROTO_SOURCES
-    ${_name}_CXX_PROTO_HEADERS
-  )
-
-  # Add main lib
-  if (_ARGS_STATIC)
-    add_library(${_name} STATIC ${${_name}_SOURCES} ${${_name}_CXX_PROTO_SOURCES})
-  elseif(_ARGS_SHARED)
-    add_library(${_name} SHARED ${${_name}_SOURCES} ${${_name}_CXX_PROTO_SOURCES})
-  else()
-    add_library(${_name} ${${_name}_SOURCES} ${${_name}_CXX_PROTO_SOURCES})
-  endif()
-  target_include_directories(
-    ${_name}
-    SYSTEM
-    BEFORE
-    PUBLIC
-      $<BUILD_INTERFACE:${EXTERNALS_ROOT}/include>
-      $<BUILD_INTERFACE:${EXTERNALS_ROOT}/include/zlib>
-  )
-  target_include_directories(
-    ${_name}
-    PUBLIC
-      $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}>
-      $<INSTALL_INTERFACE:${CMAKE_INSTALL_INCLUDEDIR}>
-  )
-  # Include protos (if any)
-  if (${_name}_PROTO_SOURCES)
-    target_include_directories(
-      ${_name}
-      PUBLIC
-        $<BUILD_INTERFACE:${CMAKE_CURRENT_BINARY_DIR}>
-    )
-  endif()
-  target_link_directories(
-    ${_name}
-    PUBLIC
-      $<BUILD_INTERFACE:${CMAKE_CURRENT_BINARY_DIR}>
-      $<BUILD_INTERFACE:${EXTERNALS_ROOT}/lib/$<CONFIG>>
-      $<INSTALL_INTERFACE:${CMAKE_INSTALL_LIBDIR}>
-  )
-  # Install main lib
-  foreach(_hdr ${${_name}_CXX_PROTO_HEADERS})
-    file(RELATIVE_PATH _relative_path ${CMAKE_CURRENT_BINARY_DIR} ${_hdr})
-    install(FILES ${_hdr} DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}/${_relative_path})
-  endforeach()
-  install_basic(
-    NAME ${_name}
-    INCLUDE_DIR ${_include_dir}
-  )
-
-  # Add tests executable (if there are any)
-  if (${_name}_TEST_SOURCES AND NOT WIN32)
-    add_executable(${_name}-tests ${${_name}_TEST_SOURCES})
-
-    get_target_property(_lib_type ${_name} TYPE)
-    if (_lib_type STREQUAL "SHARED_LIBRARY")
-      # Set RPATH so the tests exe knows where to find the shared lib.
-      set_target_properties(
-        ${_name}-tests
-        PROPERTIES
-          BUILD_RPATH "$ORIGIN"
-          INSTALL_RPATH "$ORIGIN/../${CMAKE_INSTALL_LIBDIR}"
-      )
-    endif()
-
-    # Link main lib
-    target_link_directories(
-      ${_name}-tests
-      PRIVATE
-        $<BUILD_INTERFACE:${CMAKE_CURRENT_BINARY_DIR}>
-        $<INSTALL_INTERFACE:${CMAKE_INSTALL_LIBDIR}>
-    )
-    target_link_libraries(${_name}-tests PRIVATE ${_name})
-
-    # Catch2
-    find_package(Catch2 CONFIG REQUIRED COMPONENTS Catch2WithMain)
-    target_link_libraries(${_name}-tests PRIVATE Catch2::Catch2WithMain)
-  endif()
-endfunction()
@@ -1,71 +0,0 @@
-include_guard(DIRECTORY)
-include(GNUInstallDirs)
-include(${CMAKE_SOURCE_DIR}/cmake/ExportToParent.cmake)
-include(${CMAKE_SOURCE_DIR}/cmake/SetFirstDefined.cmake)
-
-function(init_python_module)
-  set(_options)
-  set(_one_value_args
-    NAME
-  )
-  set(_multi_value_args
-    SOURCES
-  )
-  cmake_parse_arguments(_ARGS "${_options}" "${_one_value_args}" "${_multi_value_args}" ${ARGN})
-  set_first_defined(_name _ARGS_NAME PROJECT_NAME)
-  set_first_defined(_sources _ARGS_SOURCES ${_name}_SOURCES)
-
-  set(module_proj ${_name}_module)
-  set(${module_proj}_SOURCES ${_sources})
-  pybind11_add_module(${module_proj} SHARED ${_sources})
-  target_link_libraries(
-    ${module_proj}
-    PUBLIC
-      pybind11::embed
-      spark
-      ${_name}
-  )
-  target_include_directories(
-    ${module_proj}
-    PRIVATE
-      $<BUILD_INTERFACE:${pybind11_INCLUDE_DIRS}>
-  )
-  target_include_directories(
-    ${module_proj}
-    PUBLIC
-      ${Python3_INCLUDE_DIRS}
-      $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}>
-      $<INSTALL_INTERFACE:${CMAKE_INSTALL_INCLUDEDIR}>
-  )
-  add_custom_command(
-    OUTPUT ${CMAKE_CURRENT_BINARY_DIR}/stubs/${_name}.pyi
-    DEPENDS ${_name} ${module_proj}
-    COMMAND PYTHONPATH=${CMAKE_CURRENT_BINARY_DIR} pybind11-stubgen --numpy-array-use-type-var --ignore-invalid-expressions '.*' ${_name}_module
-    COMMENT "Generating py stubs"
-  )
-  add_custom_target(
-    "${_name}_py_stubs"
-    ALL
-    DEPENDS
-      ${CMAKE_CURRENT_BINARY_DIR}/stubs/${_name}.pyi
-  )
-  install(
-    TARGETS ${module_proj}
-    EXPORT ${module_proj}Targets
-    RUNTIME
-      DESTINATION ${CMAKE_INSTALL_BINDIR}
-    LIBRARY
-      DESTINATION ${CMAKE_INSTALL_LIBDIR}
-    ARCHIVE
-      DESTINATION ${CMAKE_INSTALL_LIBDIR}
-  )
-  foreach(dir ${pybind11_INCLUDE_DIRS})
-    install(DIRECTORY ${dir} DESTINATION ${CMAKE_INSTALL_INCLUDEDIR})
-  endforeach()
-  install(
-    EXPORT ${module_proj}Targets
-    FILE ${module_proj}Targets.cmake
-    NAMESPACE ${_name}::
-    DESTINATION ${CMAKE_INSTALL_LIBDIR}/cmake/${_name}
-  )
-endfunction()
@@ -1,65 +0,0 @@
-include_guard(DIRECTORY)
-include(GNUInstallDirs)
-
-function(init_python_plugin)
-  set(_options)
-  set(_one_value_args NAME PLUGIN_NAME)
-  set(_multi_value_args SOURCES)
-  cmake_parse_arguments(_ARGS "${_options}" "${_one_value_args}" "${_multi_value_args}" ${ARGN})
-  set_first_defined(_name _ARGS_NAME PROJECT_NAME)
-  set(_plugin_name ${_ARGS_PLUGIN_NAME})
-  if (NOT DEFINED _ARGS_PLUGIN_NAME)
-    set(_plugin_name ${_name}_plugins)
-  endif()
-  set(${_plugin_name}_SOURCES ${_ARGS_SOURCES})
-  if (NOT DEFINED _ARGS_SOURCES)
-    file(
-      GLOB_RECURSE ${_plugin_name}_SOURCES
-      RELATIVE ${CMAKE_CURRENT_SOURCE_DIR}
-      ${_plugin_name}/*.py)
-  endif()
-  set(src_outs)
-  foreach(_src ${${_plugin_name}_SOURCES})
-    set(_src_in ${CMAKE_CURRENT_SOURCE_DIR}/${_src})
-    set(_src_out ${CMAKE_CURRENT_BINARY_DIR}/${_src})
-    add_custom_command(
-      OUTPUT ${_src_out}
-      COMMAND ${CMAKE_COMMAND} -E copy ${_src_in} ${_src_out}
-      DEPENDS ${_src_in}
-      COMMENT "Copying ${_plugin_name} plugin ${_src} to ${_src_out}"
-    )
-    list(APPEND src_outs ${_src_out})
-  endforeach()
-  
-  add_custom_target(copy_${_plugin_name} ALL DEPENDS ${src_outs})
-  
-  add_library(${_plugin_name} INTERFACE)
-  add_library(${_name}::${_plugin_name} ALIAS ${_plugin_name})
-  
-  set_target_properties(
-    ${_plugin_name}
-    PROPERTIES
-      INTERFACE_PYTHON_SCRIPTS_DIR
-        $<BUILD_INTERFACE:${CMAKE_CURRENT_BINARY_DIR}/${_plugin_name}>$<INSTALL_INTERFACE:${CMAKE_INSTALL_BINDIR}/${_plugin_name}>
-  )
-  
-  set_target_properties(
-    ${_plugin_name}
-    PROPERTIES
-    INTERFACE_PYTHON_SCRIPTS $<BUILD_INTERFACE:${CMAKE_CURRENT_BINARY_DIR}/${${_plugin_name}_SOURCES}>$<INSTALL_INTERFACE:${CMAKE_INSTALL_BINDIR}/${${_plugin_name}_SOURCES}>
-  )
-  
-  install(
-    TARGETS ${_plugin_name}
-    EXPORT ${_plugin_name}Targets
-    DESTINATION ${CMAKE_INSTALL_BINDIR}
-  )
-  
-  install(
-    EXPORT ${_plugin_name}Targets
-    FILE ${_plugin_name}Targets.cmake
-    NAMESPACE ${_name}::
-    DESTINATION ${CMAKE_INSTALL_LIBDIR}/cmake/${_name}
-  )
-  install(DIRECTORY ${_plugin_name} DESTINATION ${CMAKE_INSTALL_BINDIR})
-endfunction()
@@ -1,39 +0,0 @@
-include_guard(DIRECTORY)
-include(GNUInstallDirs)
-include(${CMAKE_SOURCE_DIR}/cmake/PackBasic.cmake)
-include(${CMAKE_SOURCE_DIR}/cmake/SetFirstDefined.cmake)
-
-function(install_basic)
-  set(_options)
-  set(_one_value_args NAME INCLUDE_DIR)
-  set(_multi_value_args)
-  cmake_parse_arguments(_ARGS "${_options}" "${_one_value_args}" "${_multi_value_args}" ${ARGN})
-  set_first_defined(_name _ARGS_NAME PROJECT_NAME)
-  set_first_defined(_include_dir _ARGS_INCLUDE_DIR _name)
-
-  file(
-    GLOB_RECURSE ${_name}_HEADERS
-    RELATIVE ${CMAKE_CURRENT_SOURCE_DIR}
-    ${_include_dir}/*.h)
-
-  target_sources(${_name}
-    PUBLIC
-      FILE_SET HEADERS FILES ${${_name}_HEADERS})
-
-  install(
-    TARGETS ${_name}
-    EXPORT ${_name}Targets
-    RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR}
-    LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR}
-    ARCHIVE DESTINATION ${CMAKE_INSTALL_LIBDIR}
-    FILE_SET HEADERS DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}
-  )
-
-  install(
-    EXPORT ${_name}Targets
-    FILE ${_name}Targets.cmake
-    NAMESPACE ${_name}::
-    DESTINATION ${CMAKE_INSTALL_LIBDIR}/cmake/${_name})
-
-  pack_basic()
-endfunction()
@@ -1,44 +0,0 @@
-include_guard(DIRECTORY)
-include(CMakePackageConfigHelpers)
-include(${CMAKE_SOURCE_DIR}/cmake/SetFirstDefined.cmake)
-
-function(pack_basic)
-  set(_options)
-  set(_one_value_args NAME)
-  set(_multi_value_args)
-  cmake_parse_arguments(_ARGS "${_options}" "${_one_value_args}" "${_multi_value_args}" ${ARGN})
-  set_first_defined(_name _ARGS_NAME PROJECT_NAME)
-
-  find_package(Git QUIET)
-
-  if(GIT_FOUND)
-    execute_process(
-      COMMAND ${GIT_EXECUTABLE} describe --tags --always --dirty
-      WORKING_DIRECTORY ${CMAKE_SOURCE_DIR}
-      OUTPUT_VARIABLE PROJECT_VERSION
-      OUTPUT_STRIP_TRAILING_WHITESPACE
-    )
-  endif()
-
-  if(NOT GIT_FOUND OR PROJECT_VERSION STREQUAL "")
-    set(PROJECT_VERSION "0.0.0")
-  endif()
-
-  write_basic_package_version_file(
-    "${CMAKE_CURRENT_BINARY_DIR}/${_name}ConfigVersion.cmake"
-    VERSION ${PROJECT_VERSION}
-    COMPATIBILITY AnyNewerVersion
-  )
-
-  configure_package_config_file(
-    "${CMAKE_CURRENT_SOURCE_DIR}/Config.cmake.in"
-    "${CMAKE_CURRENT_BINARY_DIR}/${_name}Config.cmake"
-    INSTALL_DESTINATION ${CMAKE_INSTALL_LIBDIR}/cmake/${_name}
-  )
-
-  install(
-    FILES
-      "${CMAKE_CURRENT_BINARY_DIR}/${_name}Config.cmake"
-      "${CMAKE_CURRENT_BINARY_DIR}/${_name}ConfigVersion.cmake"
-    DESTINATION ${CMAKE_INSTALL_LIBDIR}/cmake/${_name})
-endfunction()
@@ -1,23 +1,12 @@
-set(Protobuf_PROTOC_EXECUTABLE "${EXTERNALS_BIN}/protoc")
-set(PROTOBUF_INCLUDE_DIRS "${EXTERNALS_INCLUDE}")
+set(Protobuf_PROTOC_EXECUTABLE "${SPARK_BIN}/protoc")
+set(PROTOBUF_INCLUDE_DIRS "${SPARK_INCLUDE}")
 set(PROTOBUF_STATIC_LIB "libprotobuf")
-set(_protobuf_src_dir ${CMAKE_SOURCE_DIR}/src/lib_spark)
-set(_protobuf_bin_dir ${CMAKE_BINARY_DIR}/src/lib_spark)

-function(Protobuf_Generate_Cpp)
-  set(_options)
-  set(_one_value_args CXX_SOURCES CXX_HEADERS CXX_OUT_DIR)
-  set(_multi_value_args PROTO_INCLUDE_DIRS PROTO_SOURCES)
-  cmake_parse_arguments(_ARGS "${_options}" "${_one_value_args}" "${_multi_value_args}" ${ARGN})
-  set(${_ARGS_CXX_SOURCES})
-  set(${_ARGS_CXX_HEADERS})
+function(Protobuf_Generate_Cpp SRCS HDRS)
+  set(${SRCS})
+  set(${HDRS})

-  set(_proto_include_cli_args)
-  foreach(_dir ${_ARGS_PROTO_INCLUDE_DIRS})
-    list(APPEND _proto_include_cli_args -I ${_dir})
-  endforeach()
-
-  foreach(FIL ${_ARGS_PROTO_SOURCES})
+  foreach(FIL ${ARGN})
    get_filename_component(ABS_FIL ${FIL} ABSOLUTE)
    get_filename_component(FIL_WE ${FIL} NAME_WE)
    get_filename_component(FIL_DIR ${FIL} DIRECTORY)
@@ -25,56 +14,48 @@ function(Protobuf_Generate_Cpp)
      set(FIL_WE "${FIL_DIR}/${FIL_WE}")
    endif()

-    list(APPEND ${_ARGS_CXX_SOURCES} "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.pb.cc")
-    list(APPEND ${_ARGS_CXX_HEADERS} "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.pb.h")
+    list(APPEND ${SRCS} "${CMAKE_CURRENT_BINARY_DIR}/${FIL_WE}.pb.cc")
+    list(APPEND ${HDRS} "${CMAKE_CURRENT_BINARY_DIR}/${FIL_WE}.pb.h")

    add_custom_command(
-      OUTPUT "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.pb.cc"
-             "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.pb.h"
+      OUTPUT "${CMAKE_CURRENT_BINARY_DIR}/${FIL_WE}.pb.cc"
+             "${CMAKE_CURRENT_BINARY_DIR}/${FIL_WE}.pb.h"
      COMMAND
-        ${Protobuf_PROTOC_EXECUTABLE} ARGS --cpp_out=${_ARGS_CXX_OUT_DIR} ${_proto_include_cli_args} ${ABS_FIL}
+        ${Protobuf_PROTOC_EXECUTABLE} ARGS --cpp_out=${CMAKE_BINARY_DIR}/src -I
+        ${PROTOBUF_INCLUDE_DIRS} -I ${CMAKE_SOURCE_DIR}/src ${ABS_FIL}
      DEPENDS ${ABS_FIL}
      COMMENT "Running C++ protocol buffer compiler on ${FIL}"
      VERBATIM)
  endforeach()

-  set_source_files_properties(${${_ARGS_CXX_SOURCES}} ${${_ARGS_CXX_HEADERS}} PROPERTIES GENERATED TRUE)
+  set_source_files_properties(${${SRCS}} ${${HDRS}} PROPERTIES GENERATED TRUE)

  # remove some warnings from protobuf generated files
  if(WIN32)
-    set_source_files_properties(${${_ARGS_CXX_SOURCES}} PROPERTIES COMPILE_FLAGS
+    set_source_files_properties(${${SRCS}} PROPERTIES COMPILE_FLAGS
                                                      "/wd4309 /wd4838")
  else()
    set_source_files_properties(
-      ${${_ARGS_CXX_SOURCES}}
+      ${${SRCS}}
      PROPERTIES
        COMPILE_FLAGS
        "-Wno-unused-parameter -Wno-effc++ -Wno-pedantic -Wno-sign-compare -Wno-uninitialized"
    )
  endif()

-  set(${_ARGS_CXX_SOURCES}
-      ${${_ARGS_CXX_SOURCES}}
+  set(${SRCS}
+      ${${SRCS}}
      PARENT_SCOPE)
-  set(${_ARGS_CXX_HEADERS}
-      ${${_ARGS_CXX_HEADERS}}
+  set(${HDRS}
+      ${${HDRS}}
      PARENT_SCOPE)
 endfunction()

-function(Service_Compiler_Generate_Cpp)
-  set(_options)
-  set(_one_value_args CXX_SOURCES CXX_HEADERS CXX_OUT_DIR)
-  set(_multi_value_args PROTO_INCLUDE_DIRS PROTO_SOURCES)
-  cmake_parse_arguments(_ARGS "${_options}" "${_one_value_args}" "${_multi_value_args}" ${ARGN})
-  set(${_ARGS_CXX_SOURCES})
-  set(${_ARGS_CXX_HEADERS})
+function(Service_Compiler_Generate_Cpp SRCS HDRS)
+  set(${SRCS})
+  set(${HDRS})

-  set(_proto_include_cli_args)
-  foreach(_dir ${_ARGS_PROTO_INCLUDE_DIRS})
-    list(APPEND _proto_include_cli_args -I ${_dir})
-  endforeach()
-
-  foreach(FIL ${_ARGS_PROTO_SOURCES})
+  foreach(FIL ${ARGN})
    get_filename_component(ABS_FIL ${FIL} ABSOLUTE)
    get_filename_component(FIL_WE ${FIL} NAME_WE)
    get_filename_component(FIL_DIR ${FIL} DIRECTORY)
@@ -82,58 +63,58 @@ function(Service_Compiler_Generate_Cpp)
      set(FIL_WE "${FIL_DIR}/${FIL_WE}")
    endif()

-    list(APPEND ${_ARGS_CXX_SOURCES} "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.pb.cc")
-    list(APPEND ${_ARGS_CXX_HEADERS} "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.pb.h")
+    list(APPEND ${SRCS} "${CMAKE_CURRENT_BINARY_DIR}/${FIL_WE}.pb.cc")
+    list(APPEND ${HDRS} "${CMAKE_CURRENT_BINARY_DIR}/${FIL_WE}.pb.h")

-    list(APPEND ${_ARGS_CXX_SOURCES} "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.client.pb.cc")
-    list(APPEND ${_ARGS_CXX_HEADERS} "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.client.pb.h")
+    list(APPEND ${SRCS} "${CMAKE_CURRENT_BINARY_DIR}/${FIL_WE}.client.pb.cc")
+    list(APPEND ${HDRS} "${CMAKE_CURRENT_BINARY_DIR}/${FIL_WE}.client.pb.h")

-    list(APPEND ${_ARGS_CXX_SOURCES} "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.server.pb.cc")
-    list(APPEND ${_ARGS_CXX_HEADERS} "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.server.pb.h")
+    list(APPEND ${SRCS} "${CMAKE_CURRENT_BINARY_DIR}/${FIL_WE}.server.pb.cc")
+    list(APPEND ${HDRS} "${CMAKE_CURRENT_BINARY_DIR}/${FIL_WE}.server.pb.h")

    set(TEMPLATES
-        ${CMAKE_SOURCE_DIR}/src/lib_spark/spark/api/service_compiler/templates/cpp/client.pb.cc
-        ${CMAKE_SOURCE_DIR}/src/lib_spark/spark/api/service_compiler/templates/cpp/client.pb.h
-        ${CMAKE_SOURCE_DIR}/src/lib_spark/spark/api/service_compiler/templates/cpp/server.pb.cc
-        ${CMAKE_SOURCE_DIR}/src/lib_spark/spark/api/service_compiler/templates/cpp/server.pb.h
+        ${CMAKE_CURRENT_SOURCE_DIR}/api/service_compiler/templates/cpp/client.pb.cc
+        ${CMAKE_CURRENT_SOURCE_DIR}/api/service_compiler/templates/cpp/client.pb.h
+        ${CMAKE_CURRENT_SOURCE_DIR}/api/service_compiler/templates/cpp/server.pb.cc
+        ${CMAKE_CURRENT_SOURCE_DIR}/api/service_compiler/templates/cpp/server.pb.h
    )

-    set(_plugin_py ${CMAKE_SOURCE_DIR}/src/lib_spark/spark/api/service_compiler/plugin.py)
    if(WIN32)
-      set(_plugin_cmd ${CMAKE_SOURCE_DIR}/src/lib_spark/spark/api/service_compiler/plugin.cmd)
      add_custom_command(
-        OUTPUT "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.pb.cc"
-               "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.pb.h"
-               "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.client.pb.cc"
-               "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.client.pb.h"
-               "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.server.pb.cc"
-               "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.server.pb.h"
+        OUTPUT "${CMAKE_CURRENT_BINARY_DIR}/${FIL_WE}.pb.cc"
+               "${CMAKE_CURRENT_BINARY_DIR}/${FIL_WE}.pb.h"
+               "${CMAKE_CURRENT_BINARY_DIR}/${FIL_WE}.client.pb.cc"
+               "${CMAKE_CURRENT_BINARY_DIR}/${FIL_WE}.client.pb.h"
+               "${CMAKE_CURRENT_BINARY_DIR}/${FIL_WE}.server.pb.cc"
+               "${CMAKE_CURRENT_BINARY_DIR}/${FIL_WE}.server.pb.h"
        COMMAND SET PYTHONPATH=${SPARK_PYTHON}
        COMMAND ${CMAKE_SOURCE_DIR}/.venv/Scripts/activate.bat
        COMMAND
-          ${Protobuf_PROTOC_EXECUTABLE} ARGS --cpp_out=${_ARGS_CXX_OUT_DIR}
-          --plugin=protoc-gen-service=${_plugin_cmd}
-          --service_out=lang=cpp:${_ARGS_CXX_OUT_DIR} -I
-          ${PROTOBUF_INCLUDE_DIRS} -I ${_protobuf_src_dir} ${ABS_FIL}
-        DEPENDS ${ABS_FIL} ${TEMPLATES} ${_plugin_cmd} ${_plugin_py}
+          ${Protobuf_PROTOC_EXECUTABLE} ARGS --cpp_out=${CMAKE_BINARY_DIR}/src
+          --plugin=protoc-gen-service=${CMAKE_SOURCE_DIR}/src/lib/api/service_compiler/plugin.cmd
+          --service_out=lang=cpp:${CMAKE_BINARY_DIR}/src -I
+          ${PROTOBUF_INCLUDE_DIRS} -I ${CMAKE_SOURCE_DIR}/src ${ABS_FIL}
+        DEPENDS ${ABS_FIL} ${TEMPLATES}
+                ${CMAKE_CURRENT_SOURCE_DIR}/api/service_compiler/plugin.py
        COMMENT "Running C++ service compiler on ${FIL}"
        VERBATIM)
    else()
      add_custom_command(
-        OUTPUT "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.pb.cc"
-               "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.pb.h"
-               "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.client.pb.cc"
-               "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.client.pb.h"
-               "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.server.pb.cc"
-               "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.server.pb.h"
+        OUTPUT "${CMAKE_CURRENT_BINARY_DIR}/${FIL_WE}.pb.cc"
+               "${CMAKE_CURRENT_BINARY_DIR}/${FIL_WE}.pb.h"
+               "${CMAKE_CURRENT_BINARY_DIR}/${FIL_WE}.client.pb.cc"
+               "${CMAKE_CURRENT_BINARY_DIR}/${FIL_WE}.client.pb.h"
+               "${CMAKE_CURRENT_BINARY_DIR}/${FIL_WE}.server.pb.cc"
+               "${CMAKE_CURRENT_BINARY_DIR}/${FIL_WE}.server.pb.h"
        COMMAND ${CMAKE_SOURCE_DIR}/tools/venv_exec.sh
        ARGS
          ${Protobuf_PROTOC_EXECUTABLE}
-          --cpp_out=${_ARGS_CXX_OUT_DIR}
-          --plugin=protoc-gen-service=${_plugin_py}
-          --service_out=lang=cpp,make_file_name=${ABS_FIL}:${_ARGS_CXX_OUT_DIR}
-          ${_proto_include_cli_args} ${ABS_FIL}
-        DEPENDS ${ABS_FIL} ${TEMPLATES} ${_plugin_py}
+          --cpp_out=${CMAKE_BINARY_DIR}/src
+          --plugin=protoc-gen-service=${CMAKE_SOURCE_DIR}/src/lib/api/service_compiler/plugin.py
+          --service_out=lang=cpp,make_file_name=${ABS_FIL}:${CMAKE_BINARY_DIR}/src
+          -I ${PROTOBUF_INCLUDE_DIRS} -I ${CMAKE_SOURCE_DIR}/src ${ABS_FIL}
+        DEPENDS ${ABS_FIL} ${TEMPLATES}
+                ${CMAKE_CURRENT_SOURCE_DIR}/api/service_compiler/plugin.py
        COMMENT "Running C++ service compiler on ${FIL}"
        VERBATIM)
    endif()
@@ -142,22 +123,22 @@ function(Service_Compiler_Generate_Cpp)

  # remove some warnings from protobuf generated files
  if(WIN32)
-    set_source_files_properties(${${_ARGS_CXX_SOURCES}} PROPERTIES COMPILE_FLAGS
+    set_source_files_properties(${${SRCS}} PROPERTIES COMPILE_FLAGS
                                                      "/wd4309 /wd4838")
  else()
    set_source_files_properties(
-      ${${_ARGS_CXX_SOURCES}}
+      ${${SRCS}}
      PROPERTIES
        COMPILE_FLAGS
        "-Wno-unused-parameter -Wno-effc++ -Wno-pedantic -Wno-sign-compare -Wno-uninitialized"
    )
  endif()

-  set(${_ARGS_CXX_SOURCES}
-      ${${_ARGS_CXX_SOURCES}}
+  set(${SRCS}
+      ${${SRCS}}
      PARENT_SCOPE)
-  set(${_ARGS_CXX_HEADERS}
-      ${${_ARGS_CXX_HEADERS}}
+  set(${HDRS}
+      ${${HDRS}}
      PARENT_SCOPE)

 endfunction()
@@ -1,83 +0,0 @@
-include_guard(GLOBAL)
-
-function(protobuf_generate_cpp)
-  set(_options)
-  set(_one_value_args
-    CXX_SOURCES
-    CXX_HEADERS
-    CXX_OUT_DIR
-    PROTOC
-  )
-  set(_multi_value_args
-    PROTO_INCLUDE_DIRS
-    PROTO_SOURCES
-  )
-  cmake_parse_arguments(_ARGS "${_options}" "${_one_value_args}" "${_multi_value_args}" ${ARGN})
-
-  if (NOT DEFINED _ARGS_PROTOC)
-    if (WIN32)
-      set(_ARGS_PROTOC "${EXTERNALS_BIN}/protoc.exe")
-    else()
-      set(_ARGS_PROTOC "${EXTERNALS_BIN}/protoc")
-    endif()
-  endif()
-  set(${_ARGS_CXX_SOURCES})
-  set(${_ARGS_CXX_HEADERS})
-
-  set(_proto_include_cli_args)
-  foreach(_dir ${_ARGS_PROTO_INCLUDE_DIRS})
-    list(APPEND _proto_include_cli_args -I ${_dir})
-  endforeach()
-
-  foreach(_fil ${_ARGS_PROTO_SOURCES})
-    get_filename_component(_abs_fil ${_fil} ABSOLUTE)
-    get_filename_component(_fil_we ${_fil} NAME_WE)
-    get_filename_component(_fil_dir ${_fil} DIRECTORY)
-    if(_fil_dir)
-      set(_fil_we "${_fil_dir}/${_fil_we}")
-    endif()
-    list(APPEND ${_ARGS_CXX_SOURCES} "${_ARGS_CXX_OUT_DIR}/${_fil_we}.pb.cc")
-    list(APPEND ${_ARGS_CXX_HEADERS} "${_ARGS_CXX_OUT_DIR}/${_fil_we}.pb.h")
-    message(STATUS "add_custom_command for ${_fil} -> ${_ARGS_CXX_OUT_DIR}/${_fil_we}.pb.h")
-    # TODO: Pass proto sources by relative path (so this fn can be called from
-    # arbitrary subdirs).
-    add_custom_command(
-      OUTPUT
-        "${_ARGS_CXX_OUT_DIR}/${_fil_we}.pb.cc"
-        "${_ARGS_CXX_OUT_DIR}/${_fil_we}.pb.h"
-      COMMAND
-        ${_ARGS_PROTOC}
-      ARGS
-        --cpp_out=${_ARGS_CXX_OUT_DIR} ${_proto_include_cli_args} ${_abs_fil}
-      DEPENDS ${_abs_fil}
-      COMMENT "Running C++ protocol buffer compiler on ${_fil}"
-      VERBATIM
-    )
-  endforeach()
-
-  set_source_files_properties(
-    ${${_ARGS_CXX_SOURCES}}
-    ${${_ARGS_CXX_HEADERS}}
-    PROPERTIES
-      GENERATED TRUE
-  )
-
-  # remove some warnings from protobuf generated files
-  if(WIN32)
-    set_source_files_properties(
-      ${${_ARGS_CXX_SOURCES}}
-      PROPERTIES
-        COMPILE_FLAGS "/wd4309 /wd4838"
-    )
-  else()
-    set_source_files_properties(
-      ${${_ARGS_CXX_SOURCES}}
-      PROPERTIES
-        COMPILE_FLAGS "-Wno-unused-parameter -Wno-effc++ -Wno-pedantic -Wno-sign-compare -Wno-uninitialized"
-    )
-  endif()
-
-  set(${_ARGS_CXX_SOURCES} ${${_ARGS_CXX_SOURCES}} PARENT_SCOPE)
-  set(${_ARGS_CXX_HEADERS} ${${_ARGS_CXX_HEADERS}} PARENT_SCOPE)
-endfunction()
-
@@ -1,61 +0,0 @@
-include_guard(GLOBAL)
-
-function(protobuf_generate_cpp_config)
-  set(_options)
-  set(_one_value_args
-    PROTO_SOURCES_HEADER
-    PROTO_DEFAULTS_HEADER
-  )
-  set(_multi_value_args
-    PROTO_SOURCES
-    PROTO_DEFAULTS
-  )
-  cmake_parse_arguments(_ARGS "${_options}" "${_one_value_args}" "${_multi_value_args}" ${ARGN})
-
-  # Configure the 'config_proto_files' header. It provides an array of strings populated with the
-  # files in PROTO_SOURCES. At runtime, we use this array to generate a special 'config' proto
-  # message with fields for each message defined in said files. A global instance of this 'config'
-  # proto is then constructed, and populated with [compile-time] defaults, and [runtime] overrides
-  # (i.e., in spark.conf). Calls to `bts::config::get<T>()` retrieve the value from the field
-  # associated with type `T` in this global instance.
-  message(STATUS "PROTO_SOURCES: ${_ARGS_PROTO_SOURCES}")
-  message(STATUS "PROTO_SOURCES_HEADER: ${_ARGS_PROTO_SOURCES_HEADER}")
-  message(STATUS "PROTO_DEFAULTS_HEADER: ${_ARGS_PROTO_DEFAULTS_HEADER}")
-  if (_ARGS_PROTO_SOURCES_HEADER)
-    set(PROTO_SOURCES)
-    set(PROTO_SOURCES_CXX_ARRAY_ITEMS "")
-    foreach(_proto_path ${_ARGS_PROTO_SOURCES})
-      list(APPEND PROTO_SOURCES ${_proto_path})
-      string(APPEND PROTO_SOURCES_CXX_ARRAY_ITEMS "\"${_proto_path}\",\n")
-    endforeach()
-    list(LENGTH PROTO_SOURCES PROTO_SOURCES_LENGTH)
-    message(STATUS "CONFIGURING PROTO_SOURCES_HEADER): ${_ARGS_PROTO_SOURCES_HEADER}")
-    configure_file(${CMAKE_SOURCE_DIR}/cmake/config_proto_files.h.in ${_ARGS_PROTO_SOURCES_HEADER})
-  endif()
-
-  # Generate static defaults. Ideally, this would be a single string, but we use an array because
-  # the compiler complains if the string is longer than 65k.
-  set(_cfg_defaults_str "inline const char * config_static_defaults[] = { ")
-
-  foreach(_cfg_file_path ${_ARGS_PROTO_DEFAULTS})
-    file(READ ${_cfg_file_path} _cfg_file_content)
-    string(REPLACE "\"" "\\\"" _cfg_file_content_escaped ${_cfg_file_content})
-    string(REPLACE "\n" ";" _cfg_lines ${_cfg_file_content_escaped})
-
-    foreach(_line ${_cfg_lines})
-      string(APPEND _cfg_defaults_str "\n  \"${_line}\",")
-    endforeach(_line)
-  endforeach(_cfg_file_path)
-
-  string(APPEND _cfg_defaults_str "};")
-  set(_old_cfg_defaults_str "")
-
-  if(EXISTS ${_ARGS_PROTO_DEFAULTS_HEADER})
-    file(READ ${_ARGS_PROTO_DEFAULTS_HEADER} _old_cfg_defaults_str)
-  endif()
-
-  if(NOT _cfg_defaults_str STREQUAL _old_cfg_defaults_str)
-    file(WRITE "${_ARGS_PROTO_DEFAULTS_HEADER}" "${_cfg_defaults_str}")
-  endif()
-endfunction()
-
@@ -10,7 +10,7 @@ function(protobuf_generate_fs)
    set(_ARGS_TARGET_NAME "${PROJECT_NAME}_generate_fs_protos")
  endif()
  if (NOT DEFINED _ARGS_PROTOC)
-    set(_ARGS_PROTOC "${EXTERNALS_BIN}/protoc")
+    set(_ARGS_PROTOC "${SPARK_BIN}/protoc")
  endif()

  set(_proto_include_cli_args)
@@ -27,8 +27,8 @@ function(protobuf_generate_fs)
  endforeach()

  set(_venv_exec ${CMAKE_SOURCE_DIR}/tools/venv_exec.sh)
-  set(_plugin_py ${CMAKE_SOURCE_DIR}/src/lib_spark/spark/api/service_compiler/plugin.py)
-  set(_template_fs ${CMAKE_SOURCE_DIR}/src/lib_spark/spark/api/service_compiler/templates/fs/fable.fs)
+  set(_plugin_py ${CMAKE_SOURCE_DIR}/src/lib/api/service_compiler/plugin.py)
+  set(_template_fs ${CMAKE_SOURCE_DIR}/src/lib/api/service_compiler/templates/fs/fable.fs)
  add_custom_command(
    OUTPUT ${_outputs}
    WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}
@@ -1,10 +0,0 @@
-include_guard(GLOBAL)
-
-macro(set_first_defined RESULT)
-  foreach(_arg ${ARGN})
-    if (DEFINED ${_arg})
-      set(${RESULT} ${${_arg}})
-      break()
-    endif()
-  endforeach()
-endmacro()
@@ -1,39 +0,0 @@
-include_guard(DIRECTORY)
-include(${CMAKE_SOURCE_DIR}/cmake/SymLink.cmake)
-
-function(sym_link_python_module)
-  set(_options)
-  set(_one_value_args LIB_NAME MODULE_NAME PLUGIN_NAME)
-  set(_multi_value_args)
-  cmake_parse_arguments(_ARGS "${_options}" "${_one_value_args}"
-                        "${_multi_value_args}" ${ARGN})
-  if(NOT DEFINED _ARGS_LIB_NAME)
-    message(
-      FATAL_ERROR "LIB_NAME must be specified for sym_link_python_module()")
-  endif()
-  set(_lib_name ${_ARGS_LIB_NAME})
-  set(_module_name ${_ARGS_MODULE_NAME})
-  if(NOT DEFINED _ARGS_MODULE_NAME)
-    set(_module_name ${_lib_name}_module)
-  endif()
-  set(_plugin_name ${_ARGS_PLUGIN_NAME})
-  set(_deps ${_lib_name} ${_module_name})
-  if(DEFINED _ARGS_PLUGIN_NAME)
-    list(APPEND _deps ${_plugin_name} copy_${_plugin_name})
-  endif()
-  add_dependencies(${PROJECT_NAME} ${_deps})
-  sym_link_local_lib(${_lib_name})
-  set(_module_suffix cpython-310-x86_64-linux-gnu.so)
-  sym_link_file(
-    SOURCE
-    ${CMAKE_BINARY_DIR}/src/lib_${_lib_name}/${_module_name}.${_module_suffix}
-    DEST
-    ${CMAKE_CURRENT_BINARY_DIR}/${_module_name}.${_module_suffix}
-    INSTALL_DEST
-    ${CMAKE_INSTALL_BINDIR}/)
-  if(DEFINED _ARGS_PLUGIN_NAME)
-    get_target_property(plugins ${_plugin_name} INTERFACE_PYTHON_SCRIPTS_DIR)
-    sym_link_dir(TARGET_NAME ${PROJECT_NAME}_link_${_plugin_name} SOURCE
-                 ${plugins} DEST ${CMAKE_CURRENT_BINARY_DIR}/${_plugin_name})
-  endif()
-endfunction()
@@ -1,11 +0,0 @@
-#pragma once
-
-#include <array>
-
-namespace bts::@PROJECT_NAME@::protobufs {
-
-inline std::array<const char *, @PROTO_SOURCES_LENGTH@> config_proto_files = {
-@PROTO_SOURCES_CXX_ARRAY_ITEMS@
-};
-
-} // namespace bts::@PROJECT_NAME@::protobufs
@@ -0,0 +1,24 @@
+cmake_minimum_required(VERSION 3.5)
+
+# Explicitly set policy to convert relative paths to absolute (although this is a default policy,
+# setting it explicitly prevents a compile-warning)
+cmake_policy(SET CMP0076 NEW)
+
+# add protos to the library to facilitate source_group
+target_sources(libspark PUBLIC ${Config_Protos} ${NonConfig_Protos} ${Service_Protos})
+
+function(assign_source_group)
+  foreach(_source IN ITEMS ${ARGN})
+    if(IS_ABSOLUTE "${_source}")
+      file(RELATIVE_PATH _source_rel "${CMAKE_CURRENT_SOURCE_DIR}" "${_source}")
+    else()
+      set(_source_rel "${_source}")
+    endif()
+    get_filename_component(_source_path "${_source_rel}" PATH)
+    string(REPLACE "/" "\\" _source_path_msvc "${_source_path}")
+    source_group("${_source_path_msvc}" FILES "${_source}")
+    set(SOURCE_FILES "${_source}")
+  endforeach()
+endfunction(assign_source_group)
+
+assign_source_group(${SparkLib_Sources} ${Config_Protos} ${NonConfig_Protos} ${Service_Protos})
@@ -3,12 +3,11 @@ services:
    volumes:
      - ${SPARKD__DATA_DIR:-sparkd_data}:/home/bts/sparkdata/data
    build:
-      dockerfile: src/app_sparkd/Dockerfile
+      dockerfile: src/server/sparkd/Dockerfile
      args:
        - IMAGE=${SPARKD__IMAGE}
        - SOURCE_DIR=${SPARKD__SOURCE_DIR}
        - CONFIG_DIR=${SPARKD__CONFIG_DIR}
-        - PYTHON_REQUIREMENTS=${SPARKD__PYTHON_REQUIREMENTS}
    shm_size: "${SPARKD__SHM_SIZE:-512mb}"
    command: >
      sh -c "
@@ -34,22 +33,20 @@ services:
      resources:
        limits:
          memory: "${SPARKD__MEMORY:-4gb}"
-
  sparkd-symbols:
    volumes:
      - ${SPARKD__DATA_DIR:-sparkd_data}:/home/bts/sparkdata/data
      - ${SPARKD__RUNTIME_CONF_DIR:-sparkd_conf}:/home/bts/sparkdata/conf
    build:
-      dockerfile: src/app_sparkd/Dockerfile
+      dockerfile: src/server/sparkd/Dockerfile
      args:
        - IMAGE=${SPARKD__IMAGE}
        - SOURCE_DIR=${SPARKD__SOURCE_DIR}
        - CONFIG_DIR=${SPARKD__CONFIG_DIR}
-        - PYTHON_REQUIREMENTS=${SPARKD__PYTHON_REQUIREMENTS}
    shm_size: "${SPARKD__SHM_SIZE:-512mb}"
    command: >
      sh -c "
-        echo bts | sudo -S chown -R bts:bts /home/bts/sparkdata
+        echo bts | sudo -S chown -R bts:bts /home/bts/sparkdata/data
        if [ -e "/home/bts/sparkdata/data/logging.sparkd/log" ]; then
          mv "/home/bts/sparkdata/data/logging.sparkd/log" "/home/bts/sparkdata/data/logging.sparkd/log_$(date '+%Y%m%d-%H%M')"
        fi
@@ -70,9 +67,6 @@ services:
      - "${SPARKD__QUICKBUS_PORT}:${SPARKD__QUICKBUS_PORT}"
      - "${SPARKD__SIM_FIX_PORT}:${SPARKD__SIM_FIX_PORT}"
    deploy:
-      restart_policy:
-        condition: any
-        delay: 3s
      resources:
        limits:
          memory: "${SPARKD__MEMORY:-4gb}"
@@ -124,8 +124,8 @@ service OrderService {
 ```
 ### C++ Example
 ```cpp
-#include "spark/api/public/ServiceContext.h"
-#include "spark/api/services/OrderService.client.pb.h"
+#include "lib/api/public/ServiceContext.h"
+#include "lib/api/services/OrderService.client.pb.h"
 using namespace bts::api;
 ServiceContext ctx;
 auto orders = ctx.locate<orderman::OrderService>();
@@ -29,9 +29,9 @@ namespace sample {
 ```

 ```cpp
-#include "spark/api/public/ServiceContext.h"
-#include "spark/domain_model/Security.h"
-#include "spark/api/services/MarketDataService.client.pb.h"
+#include "lib/api/public/ServiceContext.h"
+#include "spark/lib/domain_model/Security.h"
+#include "lib/api/services/MarketDataService.client.pb.h"
 #include <thread>

 using namespace bts::api;
@@ -16,16 +16,11 @@ jmespath==1.0.1
 lazy-object-proxy==1.6.0
 MarkupSafe==3.0.2
 mccabe==0.6.1
-numpy==2.2.6
 platformdirs==2.3.0
 protobuf==4.21.12
 pybars3==0.9.7
-pybind11==2.13.6
-pybind11-stubgen==2.5.3
-pybit==5.11.0
 pycodestyle==2.7.0
 pycparser==2.20
-pycryptodome==3.23.0
 pylint==2.10.2
 PyMeta3==0.5.1
 python-dateutil==2.9.0.post0
@@ -36,7 +31,6 @@ six==1.17.0
 termcolor==1.1.0
 toml==0.10.2
 urllib3==2.0.3
-websocket-client==1.8.0
 Werkzeug==3.1.3
 wrapt==1.12.1
 yapf==0.31.0
@@ -1,7 +1,7 @@
 <Project Sdk="Microsoft.NET.Sdk">

  <PropertyGroup>
-    <TargetFramework>net8.0</TargetFramework>
+    <TargetFramework>net7.0</TargetFramework>
    <ImplicitUsings>enable</ImplicitUsings>
    <Nullable>enable</Nullable>
  </PropertyGroup>
@@ -1,7 +1,7 @@
 <Project Sdk="Microsoft.NET.Sdk">

  <PropertyGroup>
-    <TargetFramework>net8.0</TargetFramework>
+    <TargetFramework>net7.0</TargetFramework>
    <ImplicitUsings>enable</ImplicitUsings>
    <Nullable>enable</Nullable>
  </PropertyGroup>
@@ -1,7 +1,7 @@
 <Project Sdk="Microsoft.NET.Sdk">

  <PropertyGroup>
-    <TargetFramework>net8.0</TargetFramework>
+    <TargetFramework>net7.0</TargetFramework>
    <ImplicitUsings>enable</ImplicitUsings>
    <Nullable>enable</Nullable>
  </PropertyGroup>
@@ -1,14 +1,14 @@
 <Project Sdk="Microsoft.NET.Sdk">

  <PropertyGroup>
-    <TargetFramework>net8.0</TargetFramework>
+    <TargetFramework>net7.0</TargetFramework>
    <ImplicitUsings>enable</ImplicitUsings>
    <Nullable>enable</Nullable>
  </PropertyGroup>

  <ItemGroup>
    <PackageReference Include="Google.Protobuf" Version="3.24.3" />
-    <PackageReference Include="YaccLexTools" Version="1.2.3" />
+    <PackageReference Include="YaccLexTools" Version="1.2.0" />
  </ItemGroup>

  <ItemGroup>
@@ -24,6 +24,8 @@ namespace QUT.GplexBuffers
        public BufferException(string message) : base(message) { }
        public BufferException(string message, Exception innerException)
            : base(message, innerException) { }
+        protected BufferException(SerializationInfo info, StreamingContext context)
+            : base(info, context) { }
    }

    internal abstract class ScanBuff
@@ -58,10 +60,12 @@ namespace QUT.GplexBuffers
            return new BuildBuffer(source);
        }

+#if (!BYTEMODE)
        public static ScanBuff GetBuffer(Stream source, int fallbackCodePage)
        {
            return new BuildBuffer(source, fallbackCodePage);
        }
+#endif // !BYTEMODE
 #endif // !NOFILES
    }

@@ -349,15 +353,17 @@ namespace QUT.GplexBuffers
        {
            FileStream fStrm = (stream as FileStream);
            if (fStrm != null) FileName = fStrm.Name;
-            NextBlk = BlockReaderFactory.Get(stream);
+            NextBlk = BlockReaderFactory.Raw(stream);
        }

+#if (!BYTEMODE)
        public BuildBuffer(Stream stream, int fallbackCodePage)
        {
            FileStream fStrm = (stream as FileStream);
            if (fStrm != null) FileName = fStrm.Name;
            NextBlk = BlockReaderFactory.Get(stream, fallbackCodePage);
        }
+#endif

        /// <summary>
        /// Marks a conservative lower bound for the buffer,
@@ -428,19 +434,43 @@ namespace QUT.GplexBuffers
    //
    internal static class BlockReaderFactory
    {
+        public static BlockReader Raw(Stream stream)
+        {
+            return delegate(char[] block, int index, int number)
+            {
+                byte[] b = new byte[number];
+                int count = stream.Read(b, 0, number);
+                int i = 0;
+                int j = index;
+                for (; i < count; i++, j++)
+                    block[j] = (char)b[i];
+                return count;
+            };
+        }

-        public static BlockReader Get(Stream stream, int fallbackCodePage = -2)
+#if (!BYTEMODE)
+        public static BlockReader Get(Stream stream, int fallbackCodePage)
        {
            Encoding encoding;
            int preamble = Preamble(stream);

            if (preamble != 0)  // There is a valid BOM here!
                encoding = Encoding.GetEncoding(preamble);
-            else if (fallbackCodePage >= 0) // Anything but "guess"
+            else if (fallbackCodePage == -1) // Fallback is "raw" bytes
+                return Raw(stream);
+            else if (fallbackCodePage != -2) // Anything but "guess"
                encoding = Encoding.GetEncoding(fallbackCodePage);
            else // This is the "guess" option
-                encoding = GuessEncoding(stream);
-
+            {
+                int guess = new Guesser(stream).GuessCodePage();
+                stream.Seek(0, SeekOrigin.Begin);
+                if (guess == -1) // ==> this is a 7-bit file
+                    encoding = Encoding.ASCII;
+                else if (guess == 65001)
+                    encoding = Encoding.UTF8;
+                else             // ==> use the machine default
+                    encoding = Encoding.Default;
+            }
            StreamReader reader = new StreamReader(stream, encoding);
            return reader.Read;
        }
@@ -458,14 +488,6 @@ namespace QUT.GplexBuffers
            int b2 = stream.ReadByte();
            if (b0 == 0xef && b1 == 0xbb && b2 == 0xbf)
                return 65001; // UTF8
-
-            int b3 = stream.ReadByte();
-            // Check for UTF-32 BOM (Byte Order Mark)
-            if (b0 == 0x00 && b1 == 0x00 && b2 == 0xFE && b3 == 0xFF)
-                return 12001; // UTF-32BE
-            if (b0 == 0xFF && b1 == 0xFE && b2 == 0x00 && b3 == 0x00)
-                return 12000; // UTF-32LE
-
            //
            // There is no unicode preamble, so we
            // return denoter for the machine default.
@@ -473,150 +495,7 @@ namespace QUT.GplexBuffers
            stream.Seek(0, SeekOrigin.Begin);
            return 0;
        }
-
-        private static Encoding GuessEncoding(Stream stream)
-        {
-            stream.Seek(0, SeekOrigin.Begin);
-            if (IsUtf16(stream, out Encoding encoding))
-            {
-                stream.Seek(0, SeekOrigin.Begin);
-                return encoding;
-            }
-
-            stream.Seek(0, SeekOrigin.Begin);
-            if (IsUtf8(stream))
-            {
-                stream.Seek(0, SeekOrigin.Begin);
-                return Encoding.UTF8;
-            }
-
-            stream.Seek(0, SeekOrigin.Begin);
-            if (IsAscii(stream))
-            {
-                stream.Seek(0, SeekOrigin.Begin);
-                return Encoding.ASCII;
-            }
-
-#if !NET462_OR_GREATER
-            stream.Seek(0, SeekOrigin.Begin);
-            if (IsLatin1(stream))
-            {
-                stream.Seek(0, SeekOrigin.Begin);
-                return Encoding.Latin1;
-            }
-#endif
-            stream.Seek(0, SeekOrigin.Begin);
-            return Encoding.Default;
-        }
-
-        private static bool IsUtf8(Stream stream)
-        {
-            int b;
-            while ((b = stream.ReadByte()) != -1)
-            {
-                if ((b & 0x80) == 0) // 0xxxxxxx
-                {
-                    continue;
-                }
-                else if ((b & 0xE0) == 0xC0) // 110xxxxx 10xxxxxx
-                {
-                    if (!IsValidUtf8ContinuationByte(stream)) return false;
-                }
-                else if ((b & 0xF0) == 0xE0) // 1110xxxx 10xxxxxx 10xxxxxx
-                {
-                    if (!IsValidUtf8ContinuationByte(stream) || !IsValidUtf8ContinuationByte(stream)) return false;
-                }
-                else if ((b & 0xF8) == 0xF0) // 11110xxx 10xxxxxx 10xxxxxx 10xxxxxx
-                {
-                    if (!IsValidUtf8ContinuationByte(stream) || !IsValidUtf8ContinuationByte(stream) || !IsValidUtf8ContinuationByte(stream)) return false;
-                }
-                else
-                {
-                    return false;
-                }
-            }
-            return true;
-        }
-
-        private static bool IsValidUtf8ContinuationByte(Stream stream)
-        {
-            int b = stream.ReadByte();
-            return b != -1 && (b & 0xC0) == 0x80; // 10xxxxxx
-        }
-
-        private static bool IsUtf16(Stream stream, out Encoding encoding)
-        {
-            encoding = Encoding.Default;
-
-            if (stream.Length >= 2)
-            {
-                // Read the first two bytes
-                int b0 = stream.ReadByte();
-                int b1 = stream.ReadByte();
-
-                // Check for UTF-16 BOM (Byte Order Mark)
-                if (b0 == 0xFF && b1 == 0xFE)
-                {
-                    encoding = Encoding.Unicode;
-                    return true; // UTF-16LE
-                }
-                if (b0 == 0xFE && b1 == 0xFF)
-                {
-                    encoding = Encoding.BigEndianUnicode;
-                    return true; // UTF-16BE
-                }
-
-                // Reset the stream position
-                stream.Seek(0, SeekOrigin.Begin);
-
-                // Check for common UTF-16 patterns
-                byte[] buffer = new byte[2];
-                while (stream.Read(buffer, 0, 2) == 2)
-                {
-                    if (buffer[0] == 0x00 && buffer[1] != 0x00)
-                    {
-                        encoding = Encoding.BigEndianUnicode;
-                        return true; // Likely UTF-16BE
-                    }
-                    if (buffer[0] != 0x00 && buffer[1] == 0x00)
-                    {
-                        encoding = Encoding.Unicode;
-                        return true; // Likely UTF-16LE
-                    }
-                }
-            }
-
-            return false;
-        }
-
-        private static bool IsAscii(Stream stream)
-        {
-            int b;
-            while ((b = stream.ReadByte()) != -1)
-            {
-                // Latin1 includes byte values from 0x00 to 0xFF
-                if (b > 0x7F)
-                {
-                    return false;
-                }
-            }
-            return true;
-        }
-
-        private static bool IsLatin1(Stream stream)
-        {
-            int b;
-            while ((b = stream.ReadByte()) != -1)
-            {
-                // Latin1 includes byte values from 0x00 to 0xFF
-                if (b > 0xFF)
-                {
-                    return false;
-                }
-            }
-            return true;
-        }
-
+#endif // !BYTEMODE
    }
 #endif // !NOFILES
    #endregion Buffer classes
@@ -661,6 +540,180 @@ namespace QUT.GplexBuffers
            return 0;
        }
    }
+#region guesser
+#if (!BYTEMODE)
+    // ==============================================================
+    // ============          Encoding Guesser           =============
+    // ==============================================================
+
+    /// <summary>
+    /// This class provides a simple finite state automaton that
+    /// scans the file looking for (1) valid UTF-8 byte patterns,
+    /// (2) bytes >= 0x80 which are not part of a UTF-8 sequence.
+    /// The method then guesses whether it is UTF-8 or maybe some 
+    /// local machine default encoding.  This works well for the
+    /// various Latin encodings.
+    /// </summary>
+    internal class Guesser
+    {
+        ScanBuff buffer;
+
+        public int GuessCodePage() { return Scan(); }
+
+        const int maxAccept = 10;
+        const int initial = 0;
+        const int eofNum = 0;
+        const int goStart = -1;
+        const int INITIAL = 0;
+        const int EndToken = 0;
+
+        #region user code
+        /* 
+         *  Reads the bytes of a file to determine if it is 
+         *  UTF-8 or a single-byte code page file.
+         */
+        public long utfX;
+        public long uppr;
+        #endregion user code
+
+        int state;
+        int currentStart = startState[0];
+        int code;
+
+        #region ScannerTables
+        static int[] startState = new int[] { 11, 0 };
+
+        #region CharacterMap
+        static sbyte[] map = new sbyte[256] {
+/*     '\0' */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 
+/*   '\x10' */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 
+/*   '\x20' */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 
+/*      '0' */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 
+/*      '@' */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 
+/*      'P' */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 
+/*      '`' */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 
+/*      'p' */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 
+/*   '\x80' */ 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 
+/*   '\x90' */ 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 
+/*   '\xA0' */ 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 
+/*   '\xB0' */ 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 
+/*   '\xC0' */ 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 
+/*   '\xD0' */ 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 
+/*   '\xE0' */ 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 
+/*   '\xF0' */ 4, 4, 4, 4, 4, 4, 4, 4, 5, 5, 5, 5, 5, 5, 5, 5 };
+        #endregion
+
+        static sbyte[][] nextState = new sbyte[][] {
+            new sbyte[] {0, 0, 0, 0, 0, 0},
+            new sbyte[] {-1, -1, 10, -1, -1, -1},
+            new sbyte[] {-1, -1, -1, -1, -1, -1},
+            new sbyte[] {-1, -1, 8, -1, -1, -1},
+            new sbyte[] {-1, -1, 5, -1, -1, -1},
+            new sbyte[] {-1, -1, 6, -1, -1, -1},
+            new sbyte[] {-1, -1, 7, -1, -1, -1},
+            null,
+            new sbyte[] {-1, -1, 9, -1, -1, -1},
+            null,
+            null,
+            new sbyte[] {-1, 1, 2, 3, 4, 2}
+        };
+
+
+        [SuppressMessage("Microsoft.Performance", "CA1810:InitializeReferenceTypeStaticFieldsInline")]
+        // Reason for suppression: cannot have self-reference in array initializer.
+        static Guesser()
+        {
+            nextState[7] = nextState[2];
+            nextState[9] = nextState[2];
+            nextState[10] = nextState[2];
+        }
+
+        int NextState()
+        {
+            if (code == ScanBuff.EndOfFile)
+                return eofNum;
+            else
+                return nextState[state][map[code]];
+        }
+        #endregion
+
+        public Guesser(System.IO.Stream file) { SetSource(file); }
+
+        public void SetSource(System.IO.Stream source)
+        {
+            this.buffer = new BuildBuffer(source);
+            code = buffer.Read();
+        }
+
+        int Scan()
+        {
+            for (; ; )
+            {
+                int next;
+                state = currentStart;
+                while ((next = NextState()) == goStart)
+                    code = buffer.Read();
+
+                state = next;
+                code = buffer.Read();
+
+                while ((next = NextState()) > eofNum)
+                {
+                    state = next;
+                    code = buffer.Read();
+                }
+                if (state <= maxAccept)
+                {
+                    #region ActionSwitch
+#pragma warning disable 162
+                    switch (state)
+                    {
+                        case eofNum:
+                            switch (currentStart)
+                            {
+                                case 11:
+                                    if (utfX == 0 && uppr == 0) return -1; /* raw ascii */
+                                    else if (uppr * 10 > utfX) return 0;   /* default code page */
+                                    else return 65001;                     /* UTF-8 encoding */
+                                    break;
+                            }
+                            return EndToken;
+                        case 1: // Recognized '{Upper128}',	Shortest string "\xC0"
+                        case 2: // Recognized '{Upper128}',	Shortest string "\x80"
+                        case 3: // Recognized '{Upper128}',	Shortest string "\xE0"
+                        case 4: // Recognized '{Upper128}',	Shortest string "\xF0"
+                            uppr++;
+                            break;
+                        case 5: // Recognized '{Utf8pfx4}{Utf8cont}',	Shortest string "\xF0\x80"
+                            uppr += 2;
+                            break;
+                        case 6: // Recognized '{Utf8pfx4}{Utf8cont}{2}',	Shortest string "\xF0\x80\x80"
+                            uppr += 3;
+                            break;
+                        case 7: // Recognized '{Utf8pfx4}{Utf8cont}{3}',	Shortest string "\xF0\x80\x80\x80"
+                            utfX += 3;
+                            break;
+                        case 8: // Recognized '{Utf8pfx3}{Utf8cont}',	Shortest string "\xE0\x80"
+                            uppr += 2;
+                            break;
+                        case 9: // Recognized '{Utf8pfx3}{Utf8cont}{2}',	Shortest string "\xE0\x80\x80"
+                            utfX += 2;
+                            break;
+                        case 10: // Recognized '{Utf8pfx2}{Utf8cont}',	Shortest string "\xC0\x80"
+                            utfX++;
+                            break;
+                        default:
+                            break;
+                    }
+#pragma warning restore 162
+                    #endregion
+                }
+            }
+        }
+    } // end class Guesser
+    
+#endif // !BYTEMODE
+#endregion
 #endif // !NOFILES

 // End of code copied from embedded resource
@@ -1,7 +1,7 @@
 <Project Sdk="Microsoft.NET.Sdk">

  <PropertyGroup>
-    <TargetFramework>net8.0</TargetFramework>
+    <TargetFramework>net7.0</TargetFramework>
    <ImplicitUsings>enable</ImplicitUsings>
    <Nullable>enable</Nullable>
  </PropertyGroup>
@@ -1,7 +1,7 @@
 <Project Sdk="Microsoft.NET.Sdk">

  <PropertyGroup>
-    <TargetFramework>net8.0</TargetFramework>
+    <TargetFramework>net7.0</TargetFramework>
    <ImplicitUsings>enable</ImplicitUsings>
    <Nullable>enable</Nullable>
  </PropertyGroup>
@@ -1,7 +1,7 @@
 <Project Sdk="Microsoft.NET.Sdk">

  <PropertyGroup>
-    <TargetFramework>net8.0</TargetFramework>
+    <TargetFramework>net7.0</TargetFramework>
    <ImplicitUsings>enable</ImplicitUsings>
    <Nullable>enable</Nullable>
  </PropertyGroup>
@@ -1,49 +0,0 @@
-using System;
-
-namespace BCG {
-   [DataStreamUserRespresentation("count")]
-   public class CountStream : ViewStream {
-      private DataPacket myLastFiredDataPacket;
-      private IBlueTableSchema mySchema = new BlueTableSchema("count",
-         new ColumnInfo("id", 0, typeof(int)),
-         new ColumnInfo("count", 1, typeof(int)));
-      private int myId = 0;
-      private ViewStream myVS;
-
-      public CountStream(ViewStream vs) {
-         myVS = vs;
-      }
-
-      public CountStream(string tableName)
-         : this(new TableStream(tableName)) { }
-
-      protected override void myOnFirstSubscriber() {
-         myVS.onChange += new ChangeDelegate(myHandleChanges);
-      }
-      protected override void myOnLastSubscriber() {
-         myVS.onChange -= new ChangeDelegate(myHandleChanges);
-      }
-
-
-      private void myHandleChanges(object newValue) {
-         myFireEvent(
-            myLastFiredDataPacket =
-               myLastFiredDataPacket == null ?
-                  new DataPacket(myGetRows(newValue, ++myId), myGetRows(newValue, myId),
-                  new BlueTable(mySchema))
-                  :
-                  new DataPacket(myGetRows(newValue, ++myId), myGetRows(newValue, myId),
-                  myLastFiredDataPacket.newRows)
-            );
-      }
-
-      private IBlueRowCollection myGetRows(object dp, int id) {
-         BlueTable retVal = new BlueTable(mySchema);
-         retVal.addRow
-            (new BlueRow(mySchema, id, (dp as DataPacket).newRows.Count));
-         return retVal;
-      }
-
-
-   }
-}
@@ -1,38 +0,0 @@
-namespace bts.BlueOrb {
-   using bts;
-
-   using BCG;
-
-   using System;
-   [DataStreamUserRespresentation("DateTimeExpanderStream")]
-   public class DateTimeExpanstionStream : ViewStream {
-      public DateTimeExpanstionStream(ViewStream parent_stream, string inp_col) {
-         Func<IBlueTableSchema, IBlueTableSchema> get_schema = (s) => {
-            IBlueTableSchema rv = s.add_column(System.TypeCode.DateTime, $"{inp_col}_date");
-            rv = rv.add_column(System.TypeCode.String, $"{inp_col}_day_of_week");
-            rv = rv.add_column(System.TypeCode.Int32, $"{inp_col}_day_of_month");
-            rv = rv.add_column(System.TypeCode.Int32, $"{inp_col}_month");
-            rv = rv.add_column(System.TypeCode.Int32, $"{inp_col}_hour");
-            rv = rv.add_column(System.TypeCode.Int32, $"{inp_col}_minute");
-            rv = rv.add_column(System.TypeCode.Int32, $"{inp_col}_seconds_of_day");
-            return rv;
-         };
-         ViewStream rs = new MapStream(parent_stream,
-         s => get_schema(s),
-         r => {
-            DateTime dt = r.get_datetime(inp_col);
-            IBlueTableSchema s = get_schema(r.schema);
-            MutableBlueRow rv = new MutableBlueRow(s, r);
-            rv.setValue($"{inp_col}_date", dt.Date);
-            rv.setValue($"{inp_col}_day_of_week", dt.DayOfWeek.ToString());
-            rv.setValue($"{inp_col}_day_of_month", dt.Day);
-            rv.setValue($"{inp_col}_month", dt.Date.Month);
-            rv.setValue($"{inp_col}_hour", dt.Hour);
-            rv.setValue($"{inp_col}_minute", dt.Minute);
-            rv.setValue($"{inp_col}_seconds_of_day", (dt - dt.Date).TotalSeconds);
-            return rv;
-         });
-         mySetRealSource(rs);
-      }
-   }
-}
@@ -1,158 +0,0 @@
-namespace bts.BlueOrb {
-   using BCG;
-   using System;
-   using System.Text.RegularExpressions;
-
-
-   public class RegExpFilter : IStreamFilter {
-      private Regex[] myRegExps;
-      private string[] myCols;
-
-      public RegExpFilter(string col, string regExp)
-         : this(new string[] { col }, new string[] { regExp }) { }
-
-      public RegExpFilter(string[] cols, string[] regExps) {
-         myCols = cols;
-         myRegExps = new Regex[regExps.Length];
-         for (int i = 0; i < regExps.Length; i++)
-            myRegExps[i] = new Regex(regExps[i]);
-      }
-
-      #region IStreamFilter Members
-      public bool filter(IBlueRow dr) {
-         bool rv = true;
-         for (int i = 0; i < myCols.Length; i++) {
-            rv &= myRegExps[i].IsMatch(dr[myCols[i]] is DBNull ? "" : dr[myCols[i]].ToString());
-            if (!rv)
-               return rv;
-         }
-         return rv;
-      }
-      #endregion
-   }
-
-   //[DataStreamUserReprsentation("FilterStream")]
-   public class FilterStream : ViewStream {
-      private BlueTable myLastFiredRows;
-      private IBlueRowCollection myLastReceivedRows;
-      private IStreamFilter myFilter;
-      private ViewStream myParentStream;
-      private string myFilterString;
-      private bool myUseNewFilter = false;// Configuration.get<DataStreamConfig>().UseNewFilter;
-
-      #region constructors
-      public delegate bool OnFilter(IBlueRow r);
-      private class OnFilterWrapper : IStreamFilter {
-         public OnFilterWrapper(OnFilter f) { this.f = f; }
-
-         public bool filter(IBlueRow dr) { return f(dr); }
-
-         private OnFilter f;
-      }
-      public FilterStream(ViewStream vs, OnFilter f)
-         : this(vs, new OnFilterWrapper(f)) { }
-
-
-      public FilterStream(ViewStream vs, IStreamFilter filter) {
-         {
-            myParentStream = vs;
-            myFilter = filter;
-            myFilterString = filter.ToString();
-            if (filter is IDynamicStreamFilter)
-               (filter as IDynamicStreamFilter).onFilterChange += new FilterChangeDelegate(myHandleFilterChange);
-         }
-      }
-
-      public FilterStream(ViewStream vs, string filterExpression)
-         : this(vs, new BooleanExpressionFilter(filterExpression)) {
-         myFilterString = filterExpression;
-      }
-
-      public FilterStream(ViewStream vs, string col, string regExp)
-         : this(vs, new RegExpFilter(col, regExp)) {
-         myFilterString = col + " , " + regExp;
-      }
-
-      public FilterStream(ViewStream vs)
-         : this(vs, new BinaryFilter(true)) {
-         myFilterString = true.ToString();
-      }
-      #endregion
-
-      #region overridden methods
-      protected override void myOnFirstSubscriber() { myParentStream.onChange += new ChangeDelegate(myHandleCallbacks); }
-
-      protected override void myOnLastSubscriber() {
-         myLastFiredRows = null;
-         myParentStream.onChange -= new ChangeDelegate(myHandleCallbacks);
-      }
-
-      protected override void myFireEvent(object o) {
-         DataPacket dp = o as DataPacket;
-         if (myLastValue == null || dp.addedRows.Count + dp.deletedRows.Count > 0)
-            base.myFireEvent(o);
-      }
-      #endregion
-
-      private void myHandleCallbacks(object newValue) {
-         DataPacket retVal;
-         DataPacket arrivedPacket = newValue as DataPacket;
-         myLastReceivedRows = arrivedPacket.newRows;
-         if (myLastFiredRows != null) {
-            retVal = new DataPacket(myLastFiredRows, new BlueTable(myLastFiredRows.schema),
-               new BlueTable(myLastFiredRows.schema));
-            foreach (IBlueRow r in arrivedPacket.deletedRows)
-               if (myLastFiredRows.containsID((int)r["id"])) {
-                  myLastFiredRows.deleteRow(r);
-                  (retVal.deletedRows as BlueTable).addRow(r);
-               }
-            foreach (IBlueRow r in arrivedPacket.addedRows)
-               if (myFilter.filter(r)) {
-                  myLastFiredRows.addRow(r);
-                  (retVal.addedRows as BlueTable).addRow(r);
-               }
-         } else {
-            myLastFiredRows = new BlueTable(arrivedPacket.schema);
-            retVal = new DataPacket(myLastFiredRows, myLastFiredRows, new BlueTable(myLastFiredRows.schema));
-            foreach (IBlueRow r in arrivedPacket.newRows)
-               if (myFilter.filter(r))
-                  myLastFiredRows.addRow(r);
-         }
-         myFireEvent(retVal);
-      }
-
-      public IStreamFilter filter {
-         set {
-            {
-               myFilter = value;
-               if (numberOfSubscribers > 0) {
-                  if (myLastReceivedRows != null) {
-                     BlueTable nRows = new BlueTable(myLastReceivedRows.schema);
-                     foreach (IBlueRow r in myLastReceivedRows)
-                        if (myFilter.filter(r))
-                           nRows.addRow(r);
-                     IBlueRowCollection dRows = new BlueTable(nRows.schema),
-                        aRows = new BlueTable(nRows.schema);
-                     BlueDataUtility.computeDifferences(nRows, myLastFiredRows, out dRows, out aRows);
-                     myLastFiredRows = nRows;
-                     myFireEvent(new DataPacket(nRows, aRows, dRows));
-                  }
-               }
-            }
-         }
-         get {            
-               return myFilter;
-         }
-      }
-
-      //public string filterString { set { filter = new BooleanExpressionFilter(value); } }
-
-      public override string ToString() {
-         return GetType().ToString() + " " + myFilterString;
-      }
-
-      private void myHandleFilterChange() {
-         filter = myFilter;
-      }
-   }
-}
@@ -1,121 +0,0 @@
-using System;
-using System.Collections.Generic;
-using System.Diagnostics;
-using System.Linq;
-using System.Text.Json;
-
-using bts;
-//using bts.SilverPine;
-using BCG;
-
-namespace bts.BlueOrb {
-   [DataStreamUserRespresentation("GroupByStream")]
-   public class GroupByStream : ViewStream {
-
-      public interface INonIncGroupByEvaluator {
-         // grouped_rows
-         IBlueRowCollection evaluate(IDictionary<int, List<IBlueRow>> grouped_rows);
-         string[] get_group_by_cols();
-      }
-
-      private class DelNonIncGroupByEvaluator : INonIncGroupByEvaluator {
-         Func<string[]> _groupby_cols_func;
-         Func<List<IBlueRow>, List<IBlueRow>> _evaluate_func;
-         public DelNonIncGroupByEvaluator(Func<string[]> groupby_cols_func,
-                                          Func<List<IBlueRow>, List<IBlueRow>> evaluate_func) {
-            _groupby_cols_func = groupby_cols_func;
-            _evaluate_func = evaluate_func;
-         }
-         public IBlueRowCollection evaluate(IDictionary<int, List<IBlueRow>> grouped_rows) {
-            BlueTable rv = null;
-            foreach (var kv in grouped_rows) {
-               List<IBlueRow> g_rows = kv.Value;
-               List<IBlueRow> out_rows = _evaluate_func.Invoke(g_rows);
-               if (out_rows != null && out_rows.Count > 0) {
-                  rv = rv ?? new BlueTable(out_rows.ElementAt(0).schema);
-               }
-               rv.addRows(out_rows);
-            }
-            return rv;
-         }
-         public string[] get_group_by_cols() {
-            return _groupby_cols_func();
-         }
-
-      }
-      private ViewStream _main_stream;
-      private INonIncGroupByEvaluator _groupby_func;
-      private BlueRowHasher _row_hasher;
-      private string[] _groupby_cols;
-      private Stopwatch _timer = new();
-      private int _n_total_fires = 0, _n_expensive_fires = 0;
-
-      public GroupByStream(ViewStream main_stream, INonIncGroupByEvaluator groupby_func) {
-         _main_stream = main_stream;
-         _groupby_func = groupby_func;
-         _groupby_cols = _groupby_func.get_group_by_cols();
-         _row_hasher = new BlueRowHasher(_groupby_cols == null ? 0 : _groupby_cols.Length);
-      }
-
-      public GroupByStream(ViewStream main_stream, Func<string[]> groupby_cols_func,
-                                 Func<List<IBlueRow>, List<IBlueRow>> evaluate_func)
-                                 : this(main_stream, new DelNonIncGroupByEvaluator(groupby_cols_func, evaluate_func)) { }
-
-      public GroupByStream(ViewStream parent_str, string groupby_col, string standard_group_by_text)
-      : this(parent_str, groupby_col, standard_group_by_text, String.Empty) { }
-
-      // this is the main constructor that does everything
-      public GroupByStream(ViewStream parent_str, string groupby_col, string standard_group_by_ops, string optional_sort_col) {
-         MultiRowOpHelper std_group_by = MultiRowOpHelper.parse_ops(standard_group_by_ops);
-         Func<string[]> groupby_cols_func = () => {
-            return String.IsNullOrEmpty(groupby_col) ? null : new string[] { groupby_col };
-         };
-         Func<List<IBlueRow>, List<IBlueRow>> evaluate_func = (group_rows) => {
-            if (!String.IsNullOrEmpty(optional_sort_col))
-               group_rows.Sort((r1, r2) => BlueDataUtility.compare_rows(r2, r1, optional_sort_col));
-            // this si where the group by aggregation happens
-            IBlueRow rv = MultiRowOpHelper.group_by_compute(group_rows, std_group_by);
-            return new IBlueRow[] { rv }.ToList();
-         };
-         ViewStream rs = new GroupByStream(parent_str, groupby_cols_func, evaluate_func);
-         mySetRealSource(rs);
-      }
-      public GroupByStream(ViewStream parent_str, string standard_group_by_text) :
-      this(parent_str, String.Empty, standard_group_by_text) {
-      }
-
-
-      protected override void myOnFirstSubscriber() { _main_stream.onChange += _main_stream_onchange; }
-
-      protected override void myOnLastSubscriber() { _main_stream.onChange -= _main_stream_onchange; }
-
-      private void _main_stream_onchange(object nv) {
-         DataPacket dp = nv as DataPacket;
-         ///grouping together
-         _timer.Restart();
-         IDictionary<int, List<IBlueRow>> grouped_rows = (_groupby_cols == null || _groupby_cols.Length == 0) ?
-         new Dictionary<int, List<IBlueRow>>() { { 0, dp.newRows.to_list() } } :
-         BlueDataUtility.group_together(dp.newRows, _groupby_cols, ref _row_hasher);
-         _timer.Stop();
-         var groupElapsedMs = _timer.ElapsedMilliseconds;
-         if (groupElapsedMs >= 50) {
-            Log.DataFlow.info($"Time to group {dp.newRows.Count} rows into {grouped_rows.Count} groups: {_timer.ElapsedMilliseconds}ms");
-         }
-         // Log.log(bts.Log.Level.Info, bts.log.Subsystem.gen, String.Format("Time to group {0} rows into {1} groups = {2}", dp.newRows.Count, grouped_rows.Count, _timer.lastIntervalTime));
-         /// reducing
-         _timer.Restart();
-         IBlueRowCollection total_state = _groupby_func.evaluate(grouped_rows);
-         _timer.Stop();
-         var applyElapsedMs = _timer.ElapsedMilliseconds;
-         _n_total_fires++;
-         if (applyElapsedMs >= 50) {
-            _n_expensive_fires++;
-            Log.DataFlow.info(
-               $"Time to evaluate {grouped_rows.Count} groups {_timer.ElapsedMilliseconds}ms, ratio {_n_expensive_fires}/{_n_total_fires}");
-         }
-         // TODO,
-         if (total_state != null)
-            myFireEvent(new DataPacket(total_state, total_state, myLastValue == null ? new BlueTable(total_state.schema) : (myLastValue as DataPacket).newRows));
-      }
-   }
-}
@@ -1,17 +0,0 @@
-using System;
-using System.Collections.Generic;
-using System.Linq;
-using System.Text;
-using System.Threading.Tasks;
-using BCG;
-
-namespace bts.BlueOrb {   
-   public interface INonIncJoinInlineMapper
-   {      
-      IBlueTableSchema get_output_schema(IBlueTableSchema schema);
-
-      IBlueRowCollection map_rows(IBlueRowCollection rows);
-      
-      event MapperChangeDelegate onMapperChange;
-   }
-}
@@ -1,152 +0,0 @@
-using bts;
-using bts.collections;
-
-using mbts;
-
-using System;
-using System.Collections.Generic;
-using System.Diagnostics;
-using System.Linq;
-using System.Security.Cryptography;
-using BCG;
-
-namespace bts.BlueOrb {   
-   internal class NonIncJoinBookkeeper : IDisposable {
-      public Action on_new_data;
-      private readonly NonIncJoinSpec _join_spec;
-      private readonly int _join_position;
-      private readonly int _start_index_in_output;
-      private readonly BlueRowHasher _row_hasher;
-      private readonly bool _is_left_most;
-      private readonly bool _is_tag_along;
-      private int _n_total_fires = 0, _n_expensive_fires = 0;
-
-      private IBlueTableSchema _schema = null;
-      private IDictionary<int, IBlueRow> _row_map = new Dictionary<int, IBlueRow>();
-      private List<Pair<int, IBlueRow>> _row_list = new List<Pair<int, IBlueRow>>();
-      private IBlueRow _solitary_row = null;
-
-      private int[] _join_col_indices = null,
-                     _data_col_indices = null;
-
-      public NonIncJoinBookkeeper(NonIncJoinSpec the_join_spec, int join_position, int start_index_in_output, BlueRowHasher row_hasher) {
-         _join_spec = the_join_spec;
-         _join_position = join_position;
-         _start_index_in_output = start_index_in_output;
-         _row_hasher = row_hasher;
-         _is_left_most = _join_position == 0;
-         _is_tag_along = _join_spec.join_col_names.Length == 0;
-      }
-
-      private bool _filled_missing_data = false;
-      private void _fill_missing_data(DataPacket dp) {
-         if (_filled_missing_data)
-            return;
-         Func<string[], bool> null_or_empty = (str_arr) => {
-            if (str_arr == null || str_arr.Length == 0)
-               return true;
-            foreach (string s in str_arr) {
-               if (!String.IsNullOrEmpty(s)) {
-                  return false;
-               }
-            }
-            return true;
-         };
-         Func<string[], string[]> remove_bad_cols = (col_names) => {
-            HashSet<string> schema_col_map = new HashSet<string>(dp.schema.columnNames);
-            List<string> cols_list = new List<string>(col_names);
-            cols_list.RemoveWhere(c => !schema_col_map.Contains(c));
-            return cols_list.ToArray();
-         };
-         if (null_or_empty(_join_spec.data_col_names)) {
-            _join_spec.data_col_names = dp.schema.columnNames;
-         }
-         _join_spec.data_col_names = remove_bad_cols(_join_spec.data_col_names);
-         if (null_or_empty(_join_spec.data_col_names)) {
-            _join_spec.data_col_names = dp.schema.columnNames;
-         }
-         if (null_or_empty(_join_spec.projected_col_names)) {
-            _join_spec.projected_col_names = _join_spec.data_col_names;
-         }
-      }
-
-      public void init() {
-         _join_spec.stream.onChange += _on_stream_cb;
-      }
-
-      public void _on_stream_cb(object dp_obj) {
-         //Log.Gen.info($"NonIncJoinBookkeeper._on_stream_cb dp_obj: {dp_obj}");
-         var timer = Stopwatch.StartNew();
-         DataPacket dp = (DataPacket)dp_obj;
-         _fill_missing_data(dp);
-         if (_schema == null) {
-            _schema = dp.schema;
-            _join_col_indices = BlueDataUtility.get_column_indices(_schema, _join_spec.join_col_names);
-            _data_col_indices = BlueDataUtility.get_column_indices(_schema, _join_spec.data_col_names);
-         }
-
-         if (_is_left_most) {
-            _row_list.Clear();
-            foreach (IBlueRow r in dp.newRows)
-               _row_list.Add(new Pair<int, IBlueRow>(_row_hasher.get_hash(r, _join_col_indices), r));
-         } else {
-            if (_is_tag_along)
-               _solitary_row = BlueDataUtility.extractRow(dp.newRows, 0);
-            else {
-               _row_map.Clear();
-               foreach (IBlueRow r in dp.newRows)
-                  _row_map[_row_hasher.get_hash(r, _join_col_indices)] = r;
-            }
-         }
-         timer.Stop();
-         _n_total_fires++;
-         if (timer.ElapsedMilliseconds >= 50) {
-            _n_expensive_fires++;
-            Log.Gen.info($"NonIncJoinBookkeepingData._on_stream_cb {dp.newRows.Count} rows {_join_col_indices.Length} "
-               + $"key cols {_join_position} join pos time {timer.Elapsed} ratio {_n_expensive_fires}/{_n_total_fires}");
-         }
-         if (on_new_data != null)
-            on_new_data();
-      }
-
-      public bool is_ready { get { return _schema != null; } }
-
-      public void constribute_cols_to_schema(List<ColumnInfo> cis) {
-         for (int i = 0; i < _join_spec.data_col_names.Length; i++)
-            cis.Add(new ColumnInfo(_join_spec.projected_col_names[i], cis.Count, _schema[_join_spec.data_col_names[i]].colType));
-      }
-
-      public List<Pair<int, IBlueRow>> get_list() { return _row_list; }
-
-      public void copy_master_row_to_output_row(int row_index, object[] output_row) {
-         IBlueRow this_row = _row_list[row_index].second;
-         BlueDataUtility.copy_to_row(this_row, _data_col_indices, output_row, _start_index_in_output);
-      }
-
-      public void copy_matching_row_to_output_row(int row_hash, object[] output_row) {
-         if (_is_tag_along) {
-            if (_solitary_row != null)
-               BlueDataUtility.copy_to_row(_solitary_row, _data_col_indices, output_row, _start_index_in_output);
-            else
-               copy_dbnull_to_output_row(output_row);
-         } else {
-            IBlueRow this_row = null;
-            if (_row_map.TryGetValue(row_hash, out this_row))
-               BlueDataUtility.copy_to_row(this_row, _data_col_indices, output_row, _start_index_in_output);
-            else
-               copy_dbnull_to_output_row(output_row);
-         }
-      }
-
-      private void copy_dbnull_to_output_row(object[] output_row) {
-         for (int i = 0; i < _data_col_indices.Length; i++)
-            output_row[i + _start_index_in_output] = DBNull.Value;
-      }
-
-      #region IDisposable
-
-      public void Dispose() { _join_spec.stream.onChange -= _on_stream_cb; }
-
-      #endregion
-   }
-}
@@ -1,67 +0,0 @@
-using System;
-using System.Collections.Generic;
-using System.Linq;
-using System.Text;
-using System.Threading.Tasks;
-
-using BCG;
-
-using mbts;
-
-namespace bts.BlueOrb {   
-   // this a collection of spec, theres no joining logic in here
-   public class NonIncJoinSpec {
-      public ViewStream stream;
-      // join col names are the names on which the streams are joined (for equality),
-      // data col names are the names in the original stream
-      // projected col names are their new names, ie if its x:l_x, then x is in the 
-      // data_col_names and l_x in projected cols
-      public string[] join_col_names, data_col_names, projected_col_names;
-
-      public NonIncJoinSpec() { }
-
-      public NonIncJoinSpec(ViewStream the_stream, string[] the_join_col_names, string[] the_data_col_names)
-         : this(the_stream, the_join_col_names, the_data_col_names, the_data_col_names) { }
-
-      public NonIncJoinSpec(ViewStream the_stream, string[] the_join_col_names, string[] the_data_col_names, string[] the_projected_col_names) {
-         stream = the_stream;
-         join_col_names = the_join_col_names;
-         data_col_names = the_data_col_names;
-         projected_col_names = the_projected_col_names;
-      }
-
-      public static NonIncJoinSpec[] from_triplet(params Triplet<ViewStream, string, string>[] join_specs) {
-         List<NonIncJoinSpec> rv = new List<NonIncJoinSpec>();
-         foreach (Triplet<ViewStream, string, string> r in join_specs) {
-            string[] keys = String.IsNullOrEmpty(r.second) ? new string[] { } : r.second.Split(' ');
-            string[] visible_cols = r.third.Split(' ');
-            List<string> data_col_names = new List<string>(), mapped_col_names = new List<string>();
-            foreach (string s in visible_cols) {
-               string[] toks = s.Split(':');
-               if (toks.Length == 1) {
-                  data_col_names.Add(toks[0]);
-                  mapped_col_names.Add(toks[0]);
-               } else if (toks.Length == 2) {
-                  data_col_names.Add(toks[0]);
-                  mapped_col_names.Add(toks[1]);
-               }
-            }
-            rv.Add(new NonIncJoinSpec(r.first, keys, data_col_names.ToArray(), mapped_col_names.ToArray()));
-         }
-         return rv.ToArray();
-      }
-
-      public override string ToString() {
-         Func<string[], string> arr_txt = (a) => {
-            string rv = "";
-            foreach (string s in a) {
-               rv += "{s} ";
-            }
-            rv = rv.Trim();
-            return rv;
-         };
-         string rv = $"stream: {stream} join_col_names: {arr_txt(join_col_names)} data_col_names: {arr_txt(data_col_names)} projected_col_names: {arr_txt(projected_col_names)}";
-         return rv;
-      }
-   }
-}
@@ -1,165 +0,0 @@
-using bts;
-
-using mbts;
-
-using System;
-using System.Collections.Generic;
-using System.Diagnostics;
-using BCG;
-
-namespace bts.BlueOrb {   
-   [DataStreamUserRespresentation("JoinStream")]
-   public class NonIncJoinStream : ViewStream, BCG.Utilities.General.IWorker {
-      private readonly NonIncJoinSpec[] _join_specs;
-      private readonly INonIncJoinInlineMapper _output_mapper;
-      private readonly BlueRowHasher _row_hasher;
-      private readonly TimeSpan _throttle_period;
-      private readonly ViewStreamDatabase _vsd;
-
-      private List<NonIncJoinBookkeeper> _book_keeping_data = new List<NonIncJoinBookkeeper>();
-      private IBlueTableSchema _output_schema = null;
-      private DateTime _last_fire_time = DateTime.MinValue;
-      private bool _is_dirty = false;
-
-      public NonIncJoinStream(ViewStream left, string left_join_cols, string left_proj,
-                              ViewStream right, string right_join_cols, string right_proj)
-         : this(new Triplet<ViewStream, string, string>[]
-               {
-                  new Triplet<ViewStream, string, string> (left, left_join_cols, left_proj),
-                  new Triplet<ViewStream, string, string> (right, right_join_cols, right_proj)
-               }) { }
-
-      public NonIncJoinStream(params Triplet<ViewStream, string, string>[] join_specs)
-         : this(NonIncJoinSpec.from_triplet(join_specs), null, 0) { }
-
-      public NonIncJoinStream(ViewStreamDatabase vsd, INonIncJoinInlineMapper output_mapper, int throttles_ms,
-                              params Triplet<ViewStream, string, string>[] join_specs)
-         : this(vsd, NonIncJoinSpec.from_triplet(join_specs), output_mapper, throttles_ms) { 
-      }
-
-      public NonIncJoinStream(NonIncJoinSpec[] join_specs, INonIncJoinInlineMapper output_mapper, int throttles_ms = 0)
-         : this(ViewStreamDatabase.instance, join_specs, output_mapper, throttles_ms) { }
-
-      public NonIncJoinStream(ViewStreamDatabase vsd, NonIncJoinSpec[] join_specs, INonIncJoinInlineMapper output_mapper, int throttles_ms = 0) {
-         _join_specs = join_specs;
-         _output_mapper = output_mapper;
-         _row_hasher = new BlueRowHasher(_join_specs[0].join_col_names.Length);
-         _throttle_period = throttles_ms == 0 ? TimeSpan.Zero : new TimeSpan(0, 0, 0, 0, throttles_ms);
-         _vsd = vsd;
-      }
-
-      protected override void myOnFirstSubscriber() {
-         int data_col_no = 1;
-         for (int jp = 0; jp < _join_specs.Length; jp++) {
-            NonIncJoinBookkeeper s = new NonIncJoinBookkeeper(_join_specs[jp], jp, data_col_no, _row_hasher);
-            //Log.Gen.info($"NonIncJoinStream.myOnFirstSubscriber jp: {jp} spec: {_join_specs[jp]}");
-            data_col_no += _join_specs[jp].data_col_names.Length;
-            s.on_new_data += _stream_callback;
-            _book_keeping_data.Add(s);
-         }
-         if (_output_mapper != null)
-            _output_mapper.onMapperChange += _on_mapper_change;
-         _vsd.addWorker(this);
-         foreach (var bkd in _book_keeping_data)
-            bkd.init();
-      }
-
-      protected override void myOnLastSubscriber() {
-         foreach (NonIncJoinBookkeeper d in _book_keeping_data)
-            d.Dispose();
-         if (_output_mapper != null)
-            _output_mapper.onMapperChange -= _on_mapper_change;
-         _vsd.removeWorker(this);
-      }
-
-      private void _stream_callback() {
-         //Log.Gen.info("NonIncJoinStream._stream_callback received callback");
-         _is_dirty = true;
-         doWork();
-      }
-
-      private void _on_mapper_change() {
-         _is_dirty = true;
-         doWork();
-      }
-
-      private bool _is_ready() {
-         bool rv = true;
-         foreach (NonIncJoinBookkeeper d in _book_keeping_data)
-            rv &= d.is_ready;
-         return rv;
-      }
-
-      private void _construct_schema() {
-         if (_output_schema == null) {
-            List<ColumnInfo> cis = new List<ColumnInfo>(new ColumnInfo[] { new ColumnInfo("id", 0, typeof(int)) });
-            foreach (NonIncJoinBookkeeper d in _book_keeping_data)
-               d.constribute_cols_to_schema(cis);
-            _output_schema = new BlueTableSchema("NonIncJoinStream", cis.ToArray());
-         }
-      }
-
-      private int _n_total_fires = 0, _n_expensive_fires = 0;
-      private void _form_and_fire() {
-         //Log.Gen.info($"NonIncJoinStream._form_and_fire _is_ready(): {_is_ready()}");
-         if (_is_ready()) {
-            _construct_schema();
-            var timer = Stopwatch.StartNew();
-            IList<Pair<int, IBlueRow>> left_row_list = _book_keeping_data[0].get_list();
-            List<object[]> output_rows = new List<object[]>();
-
-            for (int i = 0; i < left_row_list.Count; i++) {
-               object[] or = new object[_output_schema.columnInfo.Length];
-               or[0] = i;
-               output_rows.Add(or);
-            }
-
-            for (int ji = 0; ji < _book_keeping_data.Count; ji++) {
-               NonIncJoinBookkeeper bk_data = _book_keeping_data[ji];
-               for (int i = 0; i < left_row_list.Count; i++) {
-                  if (ji == 0)
-                     bk_data.copy_master_row_to_output_row(i, output_rows[i]);
-                  else
-                     bk_data.copy_matching_row_to_output_row(left_row_list[i].first, output_rows[i]);
-               }
-            }
-            IBlueRowCollection dp_output_rows = _apply_map(new SequentialIDBlueTable(_output_schema, output_rows));
-            timer.Stop();
-            _n_total_fires++;
-            //if (timer.ElapsedMilliseconds >= 250) {
-            _n_expensive_fires++;
-            //string log_msg = $"Join and map time for {_book_keeping_data.Count} "
-            //   + $"tables {timer.Elapsed.TotalMilliseconds} msecs, ratio {_n_expensive_fires}/{_n_total_fires}";
-            //Log.Gen.info(log_msg);
-            //}
-            var dp_out = BlueDataUtility.build_dp_inc(dp_output_rows, myLastValue as DataPacket, true);
-            myFireEvent(dp_out);
-            //myFireEvent(new DataPacket(dp_output_rows, dp_output_rows, myLastValue == null ? new BlueTable(_output_schema) : (myLastValue as DataPacket).newRows));
-         }
-      }
-
-      private IBlueRowCollection _apply_map(IBlueRowCollection unmapped_rows) {
-         if (_output_mapper != null) {
-            IBlueRowCollection output_rows;
-            var timer = Stopwatch.StartNew();
-            output_rows = _output_mapper.map_rows(unmapped_rows);
-            timer.Stop();
-            if (timer.ElapsedMilliseconds >= 50) {
-               Log.Gen.info($"Time to map {unmapped_rows.Count} rows into {output_rows.Count} rows {timer.Elapsed}");
-            }
-            return output_rows;
-         } else {
-            return unmapped_rows;
-         }
-      }
-
-      public void doWork() {
-         //Log.Gen.info($"NonIncJoinStream.doWork _last_fire_time: {_last_fire_time} _is_dirty: {_is_dirty}");
-         if (DateTime.Now - _last_fire_time >= _throttle_period && _is_dirty) {
-            _is_dirty = false;
-            _last_fire_time = DateTime.Now;
-            _form_and_fire();
-         }
-      }
-   }
-}
@@ -1,28 +0,0 @@
-//namespace bts.Blueorb {
-//   using BCG;
-
-//   public class JsonSaverStream : ViewStream {
-//      private ViewStream _stream;
-//      private string _file_name;
-//      public JsonSaverStream(ViewStream vs, string file_name) {
-//         _stream = vs;
-//         _file_name = file_name;
-//      }
-
-//      protected override void myOnFirstSubscriber() {
-//         _stream.onChange += _on_data;
-//      }
-
-//      private void _on_data(object dp_obj) {
-//         if (dp_obj == null) {
-//            return;
-//         }
-//         DataPacket dp = dp_obj as DataPacket;
-//         dp.newRows.write_to_json_file(dp.newRows, _file_name);
-//      }
-
-//      protected override void myOnLastSubscriber() {
-//         _stream.onChange -= _on_data;
-//      }
-//   }
-//}
@@ -1,91 +0,0 @@
-using System;
-using System.Collections;
-using System.Collections.Generic;
-
-using Amazon.Runtime.Internal.Util;
-
-using BCG;
-using BCG.Utilities;
-
-using bts;
-
-namespace bts.BlueOrb{
-   public class LiteralStream : ViewStream {
-
-      public static List<Tuple<ColumnInfo, object>> parse_columns(string key_value_pairs) {
-         List<Tuple<ColumnInfo, object>> rv = new List<Tuple<ColumnInfo, object>>();
-         string[] pairs = key_value_pairs.Split(' ');
-         foreach (string p in pairs) {
-            string[] kvp = p.Split(':');
-            string col_name = kvp[0];
-            string val = kvp[1];
-            var tup = BlueDataUtility.guess_type(val);
-            var ci = new ColumnInfo(col_name, rv.Count, tup.Item1);
-            rv.Add(Tuple.Create(ci, tup.Item2));
-         }
-         return rv;
-      }
-
-      //literal("table1", "x:123, y:1.001, z:abc;x:123, y:1.001, z:abc")
-      // col_name: col_value ... ; col_name: col_value
-      public LiteralStream(string table_name, string rows) {
-         Func<string, int, IBlueRow> get_row = (json_row, id) => {
-            Dictionary<string, object> map = new Dictionary<string, object>();
-            string[] toks = json_row.Split(',', StringSplitOptions.TrimEntries);
-            //Logger.assert(col_vals.Length == 2);
-            foreach (string t in toks) {
-               var col_vals = t.Split(':', StringSplitOptions.TrimEntries);
-               string col = col_vals[0];
-               string val = col_vals[1];
-               map.Add(col, val);
-            }
-            map["id"] = id;
-            return new BlueRow(map);
-         };
-         string[] row_arr = rows.Split(';', StringSplitOptions.TrimEntries);
-         BlueTable table = null;
-         foreach (string json_string in row_arr) {
-            IBlueRow r = //BlueDataUtility.convert_json_to_row(json_string, table == null ? 0 : table.Count);
-            get_row(json_string, table == null ? 0 : table.Count);
-            table = table ?? new BlueTable(r.schema);
-            table.addRow(r);
-         }
-         ViewStream str = new BlueTableStream(table);
-         str = new SchemaGuesserStream(str);
-         mySetRealSource(str);
-      }
-
-      public LiteralStream(string tableName, string schema, string value) {
-         string[] columns = schema.Split(' ');
-
-
-         ColumnInfo[] ci = new ColumnInfo[columns.Length + 1];
-         ci[0] = new ColumnInfo("id", 0, typeof(int));
-         int n = 1;
-         foreach (string c in columns) {
-            string[] columnAndType = c.Split('/');
-            if (columnAndType.Length == 1)
-               ci[n] = new ColumnInfo(c, n, typeof(double));
-            else
-               ci[n] = new ColumnInfo(columnAndType[0], n, Type.GetType("System." + columnAndType[1], true));
-            n++;
-         }
-
-
-         BlueTable t = new BlueTable(new BlueTableSchema(tableName, ci));
-         string[] rows = value.Split(';');
-         n = 0;
-         foreach (string r in rows)
-            try {
-               if (r == string.Empty)
-                  continue;
-               t.addRow(
-                  BlueDataUtility.createRowFromCSVFileRow(r, t.schema, ++n));
-            } catch (Exception e) {
-               throw (new Exception("Error occured at line: " + n.ToString() + " " + e.Message));
-            }
-
-         mySetRealSource(new BlueRowCollectionStaticStream(t));
-      }
-   }
-}
@@ -1,89 +0,0 @@
-using System;
-
-using BCG.Utilities.General;
-using BCG;
-using bts;
-
-namespace bts.BlueOrb {
-   [DataStreamUserRespresentation("MillisecondStream")]
-   public class MillisecondStream : ViewStream, IWorker {
-      private DateTime _last_fire_time = DateTime.MinValue;
-      private readonly TimeSpan _period;
-      private readonly int _size;
-      private readonly ViewStreamDatabase _vsd;
-      private IBlueTableSchema _schema;
-      private bts.CircularBufferTable _table;
-
-      public MillisecondStream(int interval_msecs, int size)
-      : this(ViewStreamDatabase.instance, interval_msecs, size) {
-      }
-
-      public MillisecondStream(ViewStreamDatabase vsd, int interval_msecs, int size) {
-         _vsd = vsd;
-         _period = TimeSpan.FromMilliseconds(interval_msecs);
-         _size = size;
-         _schema = BlueDataUtility.form_schema("Milliseconds",
-               Tuple.Create("id", TypeCode.Int32),
-               Tuple.Create("time", TypeCode.DateTime),
-               Tuple.Create("display_time", TypeCode.String));
-         _table = new CircularBufferTable(_schema, _size);
-      }
-
-      protected override void myOnFirstSubscriber() {
-         _vsd.addWorker(this);
-      }
-
-      protected override void myOnLastSubscriber() {
-         _vsd.removeWorker(this);
-      }
-
-      private int _id = 0;
-      public void doWork() {
-         if (DateTime.Now - _last_fire_time >= _period) {
-            _last_fire_time = DateTime.Now;
-            IBlueRow nr = new BlueRow(_schema, _id++, _last_fire_time,
-            _last_fire_time.ToString("yyyy-MM-dd HH:mm:ss.fff"));
-            _table.add_row(nr);
-            var dp = BlueDataUtility.build_dp_inc(_table, myLastValue as DataPacket);
-            myFireEvent(dp);
-         }
-      }
-   }
-   public class MSecondStream : TimeUnitStream {
-      public MSecondStream(int intervalInms, int size)
-         : base(DateTime.Now, new TimeSpan(0, 0, 0, 0, intervalInms), size, ViewStreamDatabase.instance) { }
-
-      protected override DateTime myRoundTime(DateTime dt) { return dt; }
-   }
-
-   public class DisplayableMillisecondStream : ViewStream, IStreamMapper {
-      private IBlueTableSchema mySchema = new BlueTableSchema("ms",
-                                                               new ColumnInfo("id", 0, typeof(int)),
-                                                               new ColumnInfo("time", 1, typeof(string)));
-      public DisplayableMillisecondStream(int intervalInMs, int sz) {
-         mySetRealSource(new MapStream(new MSecondStream(intervalInMs, sz), this));
-      }
-      #region IStreamMapper Members
-
-      public IBlueTableSchema getOutputSchema(IBlueTableSchema inputSchema) {
-         return mySchema;
-      }
-
-      #endregion
-
-      #region IDynamicStreamMapper Members
-
-      public IBlueRow map(IBlueRow row) {
-         DateTime time = row.get_datetime("time");
-         var time_txt = time.ToString("yyyy-MM-dd HH:mm:ss.fff");
-         return new BlueRow(mySchema, row["id"], time_txt);
-      }
-
-      #endregion
-
-      private string myToString(DateTime dt) {
-         string sep = ":";
-         return dt.Hour + sep + dt.Minute + sep + dt.Second + sep + dt.Millisecond;
-      }
-   }
-}
@@ -1,127 +0,0 @@
-namespace bts.BlueOrb {
-   using BCG;
-
-   using System;
-   using System.Collections.Generic;
-   using System.Linq;
-   //using MathNet.Numerics.Statistics;
-
-
-   public enum MultiRowOp {
-      Mean,
-      Median,
-      Mode,
-      Range,
-      StdDev,
-      Max,
-      Min,
-      First,
-      Last,
-      Sum,
-      EdgeDiff,
-      EdgeDiffPercentage,
-      //IsEdgeDiff,
-   }
-   public class MultiRowOpHelper {
-      public readonly List<Tuple<string, MultiRowOp>> operations;
-
-      public static MultiRowOpHelper parse_ops(string text) {
-         // text is inp_col: op, inp_col: op, inp_col:op
-         List<Tuple<string, MultiRowOp>> rv = new List<Tuple<string, MultiRowOp>>();
-         string[] calcs = text.Split(',');
-         foreach (string c in calcs) {
-            string[] toks = c.Split(':');
-            string inp_col = toks[0].Trim();
-            string op_txt = toks[1].Trim();
-            MultiRowOp op = Enum.Parse<MultiRowOp>(op_txt, true);
-            rv.Add(Tuple.Create(inp_col, op));
-         }
-         return new MultiRowOpHelper(rv);
-      }
-
-      public MultiRowOpHelper(List<Tuple<string, MultiRowOp>> operations_) {
-         operations = operations_;
-      }
-
-      // aggregates multiple rows into one - ie does group_by compute
-      public static IBlueRow group_by_compute(List<IBlueRow> rows, MultiRowOpHelper parameters) {
-         if (rows.Count() == 0) {
-            return null;
-         }
-
-         IBlueRow ir = rows.ElementAt(0);
-         IBlueTableSchema in_schema = ir.schema;
-         IBlueTableSchema out_schema = in_schema;
-         IDictionary<string, Tuple<string, MultiRowOp>> out_col_map =
-            new Dictionary<string, Tuple<string, MultiRowOp>>();
-         foreach (var t in parameters.operations) {
-            string out_col_name = $"{t.Item1}_{t.Item2}".ToLower();
-            out_schema = out_schema.add_column(TypeCode.Double, out_col_name);
-            out_col_map[out_col_name] = t;
-         }
-         MutableBlueRow or = new MutableBlueRow(out_schema, ir);
-         foreach (var kv in out_col_map) {
-            string out_col_name = kv.Key;
-            string in_col_name = kv.Value.Item1;
-            MultiRowOp op = kv.Value.Item2;
-            IEnumerable<double> col_vals = BlueDataUtility.get_colvalues<double>(rows, in_col_name);
-            double comp_res = compute_op(col_vals, op);
-            or.setValue(out_col_name, comp_res);
-         }
-         return or;
-      }
-
-      // rolling_rows are the rolling window of rows. the 0th row is the one thats gettin
-      // calculated. rows 1 - n-1 are its window rows. the schema is based on the operations.
-
-      public IBlueRow rolling_calc(List<IBlueRow> rolling_rows) {
-         if (rolling_rows.Count() == 0) {
-            return null;
-         }
-         IBlueRow ir = rolling_rows.ElementAt(0);
-         IBlueTableSchema in_schema = ir.schema;
-         IBlueTableSchema out_schema = in_schema;
-         IDictionary<string, Tuple<string, MultiRowOp>> out_col_map =
-            new Dictionary<string, Tuple<string, MultiRowOp>>();
-         foreach (var t in operations) {
-            string out_col_name = $"{t.Item1}_{t.Item2}".ToLower();
-            out_schema = out_schema.add_column(TypeCode.Double, out_col_name);
-            out_col_map[out_col_name] = t;
-         }
-         MutableBlueRow or = new MutableBlueRow(out_schema, ir);
-         foreach (var kv in out_col_map) {
-            string out_col_name = kv.Key;
-            string in_col_name = kv.Value.Item1;
-            MultiRowOp op = kv.Value.Item2;
-            IEnumerable<double> col_vals = BlueDataUtility.get_colvalues<double>(rolling_rows, in_col_name);
-            double comp_res = compute_op(col_vals, op);
-            or.setValue(out_col_name, comp_res);
-         }
-         return or;
-      }
-
-      public static double compute_op(IEnumerable<double> col_vals, MultiRowOp op) {
-         switch (op) {
-            case MultiRowOp.Max:
-               return col_vals.Max<double>();
-            case MultiRowOp.Min:
-               return col_vals.Min<double>();
-            case MultiRowOp.Mean:
-               return col_vals.Sum() / col_vals.Count();
-            case MultiRowOp.Sum:
-               return col_vals.Sum();
-            case MultiRowOp.First:
-               return col_vals.First();
-            case MultiRowOp.Last:
-               return col_vals.Last();
-            case MultiRowOp.StdDev:
-               return MathNet.Numerics.Statistics.Statistics.StandardDeviation(col_vals);
-            default:
-               // TODO
-               Log.assert(false);
-               return Double.NaN;
-         }
-      }
-
-   }
-}
@@ -1,207 +0,0 @@
-namespace bts.BlueOrb {
-   using BCG;
-
-   using ProtoBuf;
-
-   using System;
-   using System.Collections;
-   using System.Collections.Generic;
-   using System.Linq;
-
-   #region mappings
-   internal class ProjectData {
-      // schema from the base stream
-      public readonly IBlueTableSchema in_schema;
-      // this is the projected schema
-      public readonly IBlueTableSchema out_schema;
-      // projected cols to the input cols, for cols who's names are changed
-      public readonly IDictionary<string, string> mapped_to_orig_col_map = new Dictionary<string, string>();
-      public ProjectData(IBlueTableSchema in_schema_, IBlueTableSchema out_schema_,
-                  IDictionary<string, string> mapped_to_orig_col_map_) {
-         in_schema = in_schema_;
-         out_schema = out_schema_;
-         mapped_to_orig_col_map = mapped_to_orig_col_map_;
-      }
-   }
-   #endregion
-
-
-#region Row with mapipings build in
-   internal class NewProjectRow : IBlueRow {
-      #region IBlueRow members
-      private ProjectData _project_data;
-      private IBlueRow _row;
-      public NewProjectRow(ProjectData project_data, IBlueRow row) {
-         _project_data = project_data;
-         _row = row;
-      }
-      public IBlueTableSchema schema { get { return _project_data.out_schema; } }
-      #endregion
-
-      #region ICollection members
-      public int Count { get { return schema.columnNames.Length; } }
-
-      public bool IsSynchronized { get { return false; } }
-
-      public object SyncRoot { get { return this; } }
-
-      public void CopyTo(Array array, int index) {
-         int i = index;
-         foreach (object o in this as IEnumerable)
-            array.SetValue(o, i++);
-      }
-      #endregion
-
-      #region IDictionary members
-      public bool IsFixedSize { get { return true; } }
-
-      public bool IsReadOnly { get { return true; } }
-
-      public object this[object key] {
-         get {
-            if (key is string)
-               return this[(string)key];
-            return this[(int)key];
-         }
-         set { throw new NotSupportedException("The IBlueRow is readonly"); }
-      }
-
-      public object this[int index] {
-         get {
-            return this[schema.columnInfo[index].colName];
-         }
-      }
-
-      public object this[string col_name] {
-         get {
-            string mapped_to_col = null;
-            if (_project_data.mapped_to_orig_col_map.TryGetValue(col_name, out mapped_to_col)) {
-               return _row[mapped_to_col];
-            }
-            return _row[col_name];
-         }
-      }
-
-      public ICollection Keys { get { return schema.columnNames; } }
-
-      public ICollection Values {
-         get {
-            object[] retVal = new object[schema.columnNames.Length];
-            CopyTo(retVal, 0);
-            return retVal;
-         }
-      }
-
-      public void Add(object key, object value) { throw new NotSupportedException("The IBlueRow is readonly"); }
-
-      public void Clear() { throw new NotSupportedException("The IBlueRow is readonly"); }
-
-      public bool Contains(object key) { return schema[(string)key] != null; }
-
-      /// <summary>
-      /// Note, the default enumerator is the IDictionary's enumerator, so if user needs to
-      /// iterate through the values, typecast this into an IEnumerable
-      /// </summary>
-      /// <returns></returns>
-      public IDictionaryEnumerator GetEnumerator() { return new BlueRowDictionaryEnumerator(this); }
-
-      IEnumerator IEnumerable.GetEnumerator() { return new BlueRowEnumerator(this); }
-
-      public void Remove(object key) { throw new NotSupportedException("The IBlueRow is readonly"); }
-      #endregion
-   }
-   #endregion
-
-   [DataStreamUserRespresentation("ProjectStream")]
-   public class NewProjectStream : ViewStream, IStreamMapper {
-
-      private ViewStream _stream;
-      private string _project_expression;
-      private ProjectData _project_data = null;
-      //public ProjectData 
-      public NewProjectStream(ViewStream vs, string project_expression) {
-         _stream = vs;
-         _project_expression = project_expression;
-         _stream = new MapStream(vs, this);
-         mySetRealSource(_stream);
-      }
-
-      public IBlueTableSchema getOutputSchema(IBlueTableSchema input_schema) {
-         _project_data = _create_project_data(input_schema, _project_expression);
-         return _project_data.out_schema;
-      }
-
-      public IBlueRow map(IBlueRow row) {
-         return new NewProjectRow(_project_data, row);
-      }
-
-      private static ProjectData _create_project_data(IBlueTableSchema input_schema,
-                                                      string project_expression) {
-         project_expression = project_expression.Trim();
-         string[] project_toks = project_expression.Split(' ');
-         HashSet<string> removed_cols = new HashSet<string>();
-         Dictionary<string, string> mapped_to_orig_map = new Dictionary<string, string>();
-         Dictionary<string, string> orig_to_mapped = new Dictionary<string, string>();
-         // by default every col will show up, so if we have the projection as (a:a1)
-         // it will convert a to a1, 
-         // examples: if a schema has cols a b c
-         // project(parent_str, "a:aa ") -> aa b c
-         // project(parent_str, "a:aa !c") -> aa b
-         // project(parent_str, "a:aa !") -> aa
-         // project(parent_str, "a:aa b")) -> aa b
-         bool hide_all = false;
-         foreach (string p_tok in project_toks) {
-            if (p_tok.StartsWith("!")) {
-               if (p_tok.Length == 1) {
-                  // if it is a "blanket "!"=> hide all cols except the one that
-                  // are stated explicitly
-                  hide_all = true;
-               } else {
-                  string removed_col = p_tok.Substring(1);
-                  removed_cols.Add(removed_col);
-               }
-            } else if (p_tok.Contains(":")) {
-               string[] map_toks = p_tok.Split(":");
-               string orig_col = map_toks[0];
-               string mapped_col = map_toks[1];
-               mapped_to_orig_map.Add(mapped_col, orig_col);
-               orig_to_mapped.Add(orig_col, mapped_col);
-            } else {
-               if (input_schema.columnNames.Contains(p_tok)) {
-                  string orig_col = p_tok;
-                  string mapped_col = orig_col;
-                  orig_to_mapped.Add(orig_col, mapped_col);
-               }
-            }
-         }
-
-         // make the new schema
-         List<ColumnInfo> new_cols = new List<ColumnInfo>();
-         foreach (ColumnInfo ci in input_schema.columnInfo) {
-            if (ci.colName == "id") {
-               ColumnInfo nc = new ColumnInfo(ci.colName, new_cols.Count, ci.colType);
-               new_cols.Add(nc);
-               continue;
-            }
-            if (removed_cols.Contains(ci.colName)) {
-               continue;
-            } else if (orig_to_mapped.ContainsKey(ci.colName)) {
-               string mapped_name = orig_to_mapped[ci.colName];
-               ColumnInfo nc = new ColumnInfo(mapped_name, new_cols.Count, ci.colType);
-               new_cols.Add(nc);
-            } else {
-               if (hide_all)
-                  continue;
-               //if (ci.colName == "id" || removed_cols.Count > 0) {
-               // unless columns are  explicitely hidden, all columns show up
-               ColumnInfo nc = new ColumnInfo(ci.colName, new_cols.Count, ci.colType);
-               new_cols.Add(nc);
-               //}
-            }
-         }
-         IBlueTableSchema new_schema = new BlueTableSchema(input_schema.tableName, new_cols.ToArray());
-         return new ProjectData(input_schema, new_schema, mapped_to_orig_map);
-      }
-
-   }
-}
@@ -1,30 +0,0 @@
-using BCG;
-
-namespace bts.BlueOrb {
-   public class SchemaGuesserStream : ViewStream {
-
-      private IBlueRowCollection _rows = null;
-      private ViewStream _vs;
-      public SchemaGuesserStream(ViewStream vs) {
-         _vs = vs;
-      }
-
-      protected override void myOnFirstSubscriber() {
-         _vs.onChange += _on_data;
-      }
-      protected override void myOnLastSubscriber() {
-         _vs.onChange -= _on_data;
-      }
-
-      private void _on_data(object obj) {
-         DataPacket dp = obj as DataPacket;
-         if (_rows == null) {
-            _rows = BlueDataUtility.get_typed_table(dp.newRows);
-         }
-         var dp_out = BlueDataUtility.build_dp_inc(_rows, myLastValue as DataPacket);
-         myFireEvent(dp_out);
-
-      }
-
-   }
-}
@@ -1,67 +0,0 @@
-using Bts.ArithmeticExpressions;
-
-using System;
-using System.Collections.Generic;
-using System.Linq;
-using System.Text;
-using BCG;
-
-namespace bts.BlueOrb {
-
-   [DataStreamUserRespresentation("SingleRowMathStream")]
-   public class MathStream : ViewStream, IStreamMapper {
-      private IArithmeticProgram myExp;
-      private BooleanExpressionFilter _bool;
-      List<string> myExtraColumns = new List<string>();
-      BlueTableSchema mySchema;
-
-
-      public MathStream(ViewStream vs, string mathExpression)
-         : this(vs, mathExpression, string.Empty) {
-      }
-
-      public MathStream(ViewStream vs, string mathExpression, string boolean_expression) {
-         myExp = ArithmeticProgramParser.Parse(mathExpression) ??
-            throw new Exception($"Failed to parse ArithmeticExpression '{mathExpression}'");
-         myExtraColumns = myExp.OutputIdentifiers.ToList();
-         if (!string.IsNullOrEmpty(boolean_expression))
-            _bool = new BooleanExpressionFilter(boolean_expression);
-         mySetRealSource(new MapStream(vs, this));
-      }
-
-      #region IStreamMapper Members
-
-      public IBlueTableSchema getOutputSchema(IBlueTableSchema inputSchema) {
-         if (mySchema == null) {
-            Dictionary<string, ColumnInfo> colIdx = new Dictionary<string, ColumnInfo>();
-            foreach (ColumnInfo ci in inputSchema.columnInfo)
-               colIdx[ci.colName] = ci;
-
-            foreach (string ncn in myExtraColumns) {
-               ColumnInfo newC;
-               if (colIdx.ContainsKey(ncn))
-                  newC = new ColumnInfo(colIdx[ncn].colName, colIdx[ncn].colNo, typeof(double));
-               else
-                  newC = new ColumnInfo(ncn, colIdx.Count, typeof(double));
-               colIdx[ncn] = newC;
-            }
-            mySchema = new BlueTableSchema(colIdx.Values.ToArray(), "math(" + inputSchema.tableName + ")");
-         }
-         return mySchema;
-      }
-
-      #endregion
-
-      #region IDynamicStreamMapper Members
-
-      public IBlueRow map(IBlueRow row) {
-         MutableBlueRow result = new MutableBlueRow(mySchema, row);
-         if (_bool == null || _bool.filter(row)) {
-            myExp.Evaluate(result.AsDictionary());
-         }
-         return result;
-      }
-
-      #endregion
-   }
-}
@@ -1,11 +0,0 @@
-namespace bts.BlueOrb {
-   using BCG;
-   [DataStreamUserRespresentation("TopStream")]
-   public class TopStream : ViewStream { 
-      public TopStream(ViewStream vs, int n_rows, string sort_col) {
-         ViewStream rs = new BCG.TopStream (vs, n_rows, sort_col);
-         mySetRealSource(rs);
-      }
-   }
-
-}
@@ -1,261 +0,0 @@
-using System;
-using System.Collections.Generic;
-
-using mbts;
-using BCG;
-
-namespace bts.BlueOrb {
-   internal class MutableAppendTable : IBlueRowCollection {
-      private class MyRow : IBlueRow {
-         private IBlueRow myRow;
-         private int myID;
-
-         public MyRow(IBlueRow r, int id) {
-            myRow = r;
-            myID = id;
-         }
-
-         #region IBlueRow Members
-
-         public IBlueTableSchema schema { get { return myRow.schema; } }
-
-         public object this[int index] {
-            get {
-               if (myRow.schema[index].colName == "id")
-                  return myID;
-               else
-                  return myRow[index];
-            }
-         }
-
-         public object this[string colName] { get { return colName == "id" ? (object)myID : myRow[colName]; } }
-
-         #endregion
-
-         #region IDictionary Members
-
-         public void Add(object key, object value) { throw new Exception("Row is read only."); }
-
-         public void Clear() { throw new Exception("Row is read only."); }
-
-         public bool Contains(object key) { return myRow.Contains(key); }
-
-         public System.Collections.IDictionaryEnumerator GetEnumerator() { return new BlueRowDictionaryEnumerator(this); }
-
-         public bool IsFixedSize { get { return true; } }
-
-         public bool IsReadOnly { get { return true; } }
-
-         public System.Collections.ICollection Keys { get { return myRow.Keys; } }
-
-         public void Remove(object key) { throw new Exception("The method or operation is not implemented."); }
-
-         public System.Collections.ICollection Values {
-            get {
-               object[] vals = new object[myRow.Count];
-               CopyTo(vals, 0);
-               return vals;
-            }
-         }
-
-         public object this[object key] {
-            get { return key is string ? this[key as string] : this[(int)key]; }
-            set { throw new Exception("The method or operation is not implemented."); }
-         }
-
-         #endregion
-
-         #region ICollection Members
-
-         public void CopyTo(Array array, int index) {
-            myRow.CopyTo(array, index);
-            array.SetValue(myID, index + myRow.schema["id"].colNo);
-         }
-
-         public int Count { get { return myRow.Count; } }
-
-         public bool IsSynchronized { get { return myRow.IsSynchronized; } }
-
-         public object SyncRoot { get { return myRow.SyncRoot; } }
-
-         #endregion
-
-         #region IEnumerable Members
-
-         System.Collections.IEnumerator System.Collections.IEnumerable.GetEnumerator() { return new BlueRowEnumerator(this); }
-
-         #endregion
-      }
-
-      private BlueTable myRows;
-      private int myID = 0;
-      /// <summary>
-      /// maps the table no to a dictionary containing the
-      /// local IDs to a row in the final big table
-      /// </summary>
-      private Dictionary<int, Dictionary<int, IBlueRow>> myIDMap = new Dictionary<int, Dictionary<int, IBlueRow>>();
-
-      public MutableAppendTable(IBlueTableSchema schema) { myRows = new BlueTable(schema); }
-
-      public IBlueRow addRow(IBlueRow r, int tableNo) {
-         if (!myIDMap.ContainsKey(tableNo))
-            myIDMap.Add(tableNo, new Dictionary<int, IBlueRow>());
-         IBlueRow nr = new MyRow(r, myID++); // BlueDataUtility.getNewReplacedRow(r, myID++, "id");
-         myRows.addRow(nr);
-         myIDMap[tableNo].Add((int)r["id"], nr);
-         return nr;
-      }
-
-      public IBlueRow removeRow(IBlueRow r, int tableNo) {
-         IBlueRow dr = myIDMap[tableNo][(int)r["id"]];
-         myRows.deleteRow(dr);
-         myIDMap[tableNo].Remove((int)r["id"]);
-         return dr;
-      }
-
-      public IBlueRowCollection removeTable(int tableNo) {
-         if (!myIDMap.ContainsKey(tableNo))
-            return null;
-         else {
-            BlueTable rv = new BlueTable(myRows.schema);
-            foreach (IBlueRow r in myIDMap[tableNo].Values)
-               rv.addRow(r);
-            myRows.deleteRows(rv);
-            myIDMap.Remove(tableNo);
-            return rv;
-         }
-      }
-
-      public void clear() {
-         myRows.clear();
-         myIDMap.Clear();
-      }
-
-      #region IBlueRowCollection Members
-
-      public IBlueRow this[int id] { get { return myRows[id]; } }
-
-      public IBlueTableSchema schema { get { return myRows.schema; } }
-
-      public bool containsID(int id) { return myRows.containsID(id); }
-
-      #endregion
-
-      #region ICollection Members
-
-      public void CopyTo(Array array, int index) { myRows.CopyTo(array, index); }
-
-      public int Count { get { return myRows.Count; } }
-
-      public bool IsSynchronized { get { return myRows.IsSynchronized; } }
-
-      public object SyncRoot { get { return myRows.SyncRoot; } }
-
-      #endregion
-
-      #region IEnumerable Members
-
-      public System.Collections.IEnumerator GetEnumerator() { return myRows.GetEnumerator(); }
-
-      #endregion
-   }
-   /// <summary>
-   /// MutableStream can append multiple streams
-   /// as well as give the user the flexibility
-   /// to append (or remove) more streams at
-   /// runtime
-   /// </summary>
-   }
-namespace bts.BlueOrb { 
-   [DataStreamUserRespresentation("UnionStream")]
-   public class MutableAppendStream : ViewStream {
-      private bool mySubscriptionActive = false;
-      private MutableAppendTable myTable;
-      private int myMovingIndex;
-      private Dictionary<int, Pair<ViewStream, ViewStream>> myStreamIndex = new Dictionary<int, Pair<ViewStream, ViewStream>>();
-      private bool _allign_col_orders = true;
-
-      public MutableAppendStream(params ViewStream[] streams)
-         : this(new List<ViewStream>(streams)) { }
-
-      public MutableAppendStream(bool allign_col_order, params ViewStream[] streams)
-         : this(allign_col_order, new List<ViewStream>(streams)) { }
-
-      public MutableAppendStream(ICollection<ViewStream> streams)
-         : this(true, streams) { }
-
-      public MutableAppendStream(bool allign_col_orders, ICollection<ViewStream> streams) {
-         _allign_col_orders = allign_col_orders;
-         foreach (ViewStream vs in streams) {
-            myMovingIndex++;
-            ViewStream pj = new ChangeTableNameStream(vs, myMovingIndex.ToString());
-            if (_allign_col_orders)
-               pj = new ProjectStream(pj, "!");
-            myStreamIndex.Add(myMovingIndex, new Pair<ViewStream, ViewStream>(vs, pj));
-         }
-      }
-
-      public void addStream(ViewStream vs) {
-         myMovingIndex++;
-         ///the change name is done to identify the callbacks, and the project to
-         ///make all the schemas col orders the same
-         ViewStream pj = new ChangeTableNameStream(vs, myMovingIndex.ToString());
-         if (_allign_col_orders)
-            pj = new ProjectStream(pj, "!");
-         myStreamIndex.Add(myMovingIndex, new Pair<ViewStream, ViewStream>(vs, pj));
-         if (mySubscriptionActive)
-            pj.onChange += new ChangeDelegate(myHandleStreamChanges);
-      }
-
-      public void removeStream(ViewStream vs) {
-         int index = -1;
-         foreach (KeyValuePair<int, Pair<ViewStream, ViewStream>> p in myStreamIndex) {
-            if (p.Value.first == vs) {
-               index = p.Key;
-               if (mySubscriptionActive) {
-                  p.Value.second.onChange -= new ChangeDelegate(myHandleStreamChanges);
-                  if (myTable != null) {
-                     IBlueRowCollection delRows = myTable.removeTable(p.Key);
-                     if (delRows != null)
-                        myFireEvent(new DataPacket(myTable, new BlueTable(myTable.schema), delRows));
-                  }
-               }
-               break;
-            }
-            if (index > 0)
-               myStreamIndex.Remove(index);
-         }
-      }
-
-      protected override void myOnFirstSubscriber() {
-         mySubscriptionActive = true;
-         foreach (KeyValuePair<int, Pair<ViewStream, ViewStream>> p in myStreamIndex)
-            p.Value.second.onChange += new ChangeDelegate(myHandleStreamChanges);
-      }
-
-      protected override void myOnLastSubscriber() {
-         mySubscriptionActive = false;
-         foreach (KeyValuePair<int, Pair<ViewStream, ViewStream>> p in myStreamIndex)
-            p.Value.second.onChange -= new ChangeDelegate(myHandleStreamChanges);
-         if (myTable != null)
-            myTable.clear();
-      }
-
-      private void myHandleStreamChanges(object newValue) {
-         DataPacket dp = newValue as DataPacket;
-         myTable = myTable ?? new MutableAppendTable(new BlueTableSchema("MutableAppend", dp.schema.columnInfo));
-         BlueTable adds = new BlueTable(myTable.schema),
-                     dels = new BlueTable(myTable.schema);
-         int index = Int32.Parse(dp.schema.tableName);
-         foreach (IBlueRow r in dp.deletedRows) {
-            IBlueRow dr = myTable.removeRow(r, index);
-            dels.addRow(dr);
-         }
-         foreach (IBlueRow r in dp.addedRows) {
-            IBlueRow nr = myTable.addRow(r, index);
-            adds.addRow(nr);
-         }
-         myFireEvent(new DataPacket(myTable, adds, dels));
-      }
-   }
-}
@@ -1,7 +1,7 @@
 <Project Sdk="Microsoft.NET.Sdk">

  <PropertyGroup>
-    <TargetFramework>net8.0</TargetFramework>
+    <TargetFramework>net7.0</TargetFramework>
    <Nullable>enable</Nullable>
  </PropertyGroup>

@@ -18,13 +18,11 @@

  <ItemGroup>
    <PackageReference Include="HtmlAgilityPack" Version="1.11.57" />
-    <PackageReference Include="IronPython" Version="3.4.2" />
    <PackageReference Include="Microsoft.Data.Analysis" Version="0.21.1" />
    <PackageReference Include="MongoDB.Bson" Version="2.23.1" />
    <PackageReference Include="MongoDB.Driver" Version="2.23.1" />
    <PackageReference Include="Parquet.Net" Version="5.0.1" />
    <PackageReference Include="System.Data.SQLite" Version="1.0.118" />
-    <PackageReference Include="MathNet.Numerics" Version="5.0.0" />
  </ItemGroup>

 </Project>
@@ -1,47 +1,52 @@
 using System;

-namespace bts.BlueOrb {
-   using BCG;
-   [DataStreamUserRespresentation("CountStream")]
-   public class CountStream : ViewStream {
+namespace BCG
+{
+   [DataStreamUserRespresentation ("count")]
+	public class CountStream : ViewStream
+	{
      private DataPacket myLastFiredDataPacket;
-      private IBlueTableSchema mySchema = new BlueTableSchema("count",
-         new ColumnInfo("id", 0, typeof(int)),
-         new ColumnInfo("count", 1, typeof(int)));
+      private IBlueTableSchema mySchema = new BlueTableSchema ("count", 
+         new ColumnInfo ("id", 0, typeof(int)),
+         new ColumnInfo ("count", 1, typeof(int)));
      private int myId = 0;
      private ViewStream myVS;

-      public CountStream(ViewStream vs) {
+      public CountStream (ViewStream vs)
+      {
         myVS = vs;
      }

-      public CountStream(string tableName)
-         : this(new TableStream(tableName)) { }
+      public CountStream (string tableName)
+         : this (new TableStream (tableName))
+      {}

-      protected override void myOnFirstSubscriber() {
+      protected override void myOnFirstSubscriber () {
         myVS.onChange += new ChangeDelegate(myHandleChanges);
      }
-      protected override void myOnLastSubscriber() {
+      protected override void myOnLastSubscriber () {
         myVS.onChange -= new ChangeDelegate(myHandleChanges);
      }


-      private void myHandleChanges(object newValue) {
-         myFireEvent(
+      private void myHandleChanges(object newValue)
+      {
+         myFireEvent (
            myLastFiredDataPacket =
               myLastFiredDataPacket == null ?
-                  new DataPacket(myGetRows(newValue, ++myId), myGetRows(newValue, myId),
-                  new BlueTable(mySchema))
+                  new DataPacket (myGetRows (newValue, ++myId), myGetRows (newValue, myId), 
+                  new BlueTable (mySchema))
                  :
-                  new DataPacket(myGetRows(newValue, ++myId), myGetRows(newValue, myId),
+                  new DataPacket (myGetRows (newValue, ++myId), myGetRows (newValue, myId), 
                  myLastFiredDataPacket.newRows)
            );
      }

-      private IBlueRowCollection myGetRows(object dp, int id) {
-         BlueTable retVal = new BlueTable(mySchema);
+      private IBlueRowCollection myGetRows (object dp, int id)
+      {
+         BlueTable retVal = new BlueTable (mySchema);
         retVal.addRow
-            (new BlueRow(mySchema, id, (dp as DataPacket).newRows.Count));
+            (new BlueRow (mySchema, id, (dp as DataPacket).newRows.Count));
         return retVal;
      }

@@ -5,20 +5,25 @@ using mbts.config;
 using System;
 using System.Text.RegularExpressions;

-namespace BCG.StreamWorld.Addition {
-   [DataStreamUserRespresentation(false)]
-   public class FilterStream : ViewStream {
+namespace BCG.StreamWorld.Addition
+{
+   [DataStreamUserRespresentation (false)]
+   public class FilterStream : ViewStream
+   {
      private ViewStream myBaseStream;
      private BlueTable myBaseTable, myStateTable;
      private IStreamFilter myFilter;

      public FilterStream(ViewStream vs)
-         : this(vs, "1=1") { }
+         : this(vs, "1=1")
+      { }

      public FilterStream(ViewStream vs, string filterExp)
-         : this(vs, new BooleanExpressionFilter(filterExp)) { }
+         : this(vs, new BooleanExpressionFilter(filterExp))
+      { }

-      public FilterStream(ViewStream vs, IStreamFilter filter) {
+      public FilterStream(ViewStream vs, IStreamFilter filter)
+      {
         myBaseStream = vs;
         myFilter = filter;
         if (filter is IDynamicStreamFilter)
@@ -26,35 +31,43 @@ namespace BCG.StreamWorld.Addition {
      }


-      protected override void myOnFirstSubscriber() {
+      protected override void myOnFirstSubscriber()
+      {
         myBaseStream.onChange += new ChangeDelegate(myHandleChanges);
      }

-      protected override void myOnLastSubscriber() {
+      protected override void myOnLastSubscriber()
+      {
         myBaseStream.onChange -= new ChangeDelegate(myHandleChanges);
         myBaseTable.clear();
         myStateTable.clear();
      }

-      private void myHandleChanges(object newValue) {
+      private void myHandleChanges(object newValue)
+      {
         DataPacket dp = newValue as DataPacket;
-         if (myStateTable == null) {
+         if (myStateTable == null)
+         {
            bts.Log.assert(myBaseTable == null);
            myStateTable = new BlueTable(dp.schema);
            myBaseTable = new BlueTable(dp.schema);
         }
         BlueTable dels = new BlueTable(dp.schema),
                     adds = new BlueTable(dp.schema);
-         foreach (IBlueRow r in dp.deletedRows) {
+         foreach (IBlueRow r in dp.deletedRows)
+         {
            myBaseTable.deleteRow(r);
-            if (myStateTable.containsID((int)r["id"])) {
+            if (myStateTable.containsID((int)r["id"]))
+            {
               dels.addRow(r);
               myStateTable.deleteRow(r);
            }
         }
-         foreach (IBlueRow r in dp.addedRows) {
+         foreach (IBlueRow r in dp.addedRows)
+         {
            myBaseTable.addRow(r);
-            if (myFilter.filter(r)) {
+            if (myFilter.filter(r))
+            {
               adds.addRow(r);
               myStateTable.addRow(r);
            }
@@ -63,10 +76,13 @@ namespace BCG.StreamWorld.Addition {
            myFireEvent(new DataPacket(myStateTable, adds, dels));
      }

-      public IStreamFilter filter {
-         set {
+      public IStreamFilter filter
+      {
+         set
+         {
            myFilter = value;
-            if (myBaseTable != null) {
+            if (myBaseTable != null)
+            {
               IBlueRowCollection oldState = new BlueTable(myStateTable);
               myStateTable.clear();
               foreach (IBlueRow r in myBaseTable)
@@ -4,46 +4,42 @@ namespace bts.SilverPine {
   using System;
   using System.Collections.Generic;
   using System.Linq;
-   //using MathNet.Numerics.Statistics;

+   using ZstdSharp.Unsafe;

-   public enum MultiRowOp {
+   public enum GroupByBundleOp {
      Mean,
      Median,
      Mode,
-      Range,
      StdDev,
      Max,
      Min,
      First,
      Last,
-      Sum,
-      EdgeDiff,
-      EdgeDiffPercentage,
-      //IsEdgeDiff,
+      Sum
   }
-   public class MultiRowOpHelper {
-      public readonly List<Tuple<string, MultiRowOp>> operations;
+   public class GroupByBundle {
+      public readonly List<Tuple<string, GroupByBundleOp>> operations;

-      public static MultiRowOpHelper parse(string text) {
+      public static GroupByBundle parse(string text) {
         // text is inp_col: op, inp_col: op, inp_col:op
-         List<Tuple<string, MultiRowOp>> rv = new List<Tuple<string, MultiRowOp>>();
+         List<Tuple<string, GroupByBundleOp>> rv = new List<Tuple<string, GroupByBundleOp>>();
         string[] calcs = text.Split(',');
         foreach (string c in calcs) {
            string[] toks = c.Split(':');
            string inp_col = toks[0].Trim();
            string op_txt = toks[1].Trim();
-            MultiRowOp op = Enum.Parse<MultiRowOp>(op_txt, true);
+            GroupByBundleOp op = Enum.Parse<GroupByBundleOp>(op_txt, true);
            rv.Add(Tuple.Create(inp_col, op));
         }
-         return new MultiRowOpHelper(rv);
+         return new GroupByBundle(rv);
      }

-      public MultiRowOpHelper(List<Tuple<string, MultiRowOp>> operations_) {
+      public GroupByBundle(List<Tuple<string, GroupByBundleOp>> operations_) {
         operations = operations_;
      }

-      public static IBlueRow group_by(List<IBlueRow> rows, MultiRowOpHelper parameters) {
+      public static IBlueRow group_by(List<IBlueRow> rows, GroupByBundle parameters) {
         if (rows.Count() == 0) {
            return null;
         }
@@ -51,8 +47,8 @@ namespace bts.SilverPine {
         IBlueRow ir = rows.ElementAt(0);
         IBlueTableSchema in_schema = ir.schema;
         IBlueTableSchema out_schema = in_schema;
-         IDictionary<string, Tuple<string, MultiRowOp>> out_col_map =
-            new Dictionary<string, Tuple<string, MultiRowOp>>();
+         IDictionary<string, Tuple<string, GroupByBundleOp>> out_col_map =
+            new Dictionary<string, Tuple<string, GroupByBundleOp>>();
         foreach (var t in parameters.operations) {
            string out_col_name = $"{t.Item1}_{t.Item2}".ToLower();
            out_schema = out_schema.add_column(TypeCode.Double, out_col_name);
@@ -62,7 +58,7 @@ namespace bts.SilverPine {
         foreach (var kv in out_col_map) {
            string out_col_name = kv.Key;
            string in_col_name = kv.Value.Item1;
-            MultiRowOp op = kv.Value.Item2;
+            GroupByBundleOp op = kv.Value.Item2;
            IEnumerable<double> col_vals = BlueDataUtility.get_colvalues<double>(rows, in_col_name);
            double comp_res = compute_op(col_vals, op);
            or.setValue(out_col_name, comp_res);
@@ -70,51 +66,20 @@ namespace bts.SilverPine {
         return or;
      }

-      // rolling_rows are the rolling window of rows. the 0th row is the one thats gettin
-      // calculated. rows 1 - n-1 are its window rows. the schema is based on the operations.
-
-      public IBlueRow rolling_calc(List<IBlueRow> rolling_rows) {
-         if (rolling_rows.Count() == 0) {
-            return null;
-         }
-         IBlueRow ir = rolling_rows.ElementAt(0);
-         IBlueTableSchema in_schema = ir.schema;
-         IBlueTableSchema out_schema = in_schema;
-         IDictionary<string, Tuple<string, MultiRowOp>> out_col_map =
-            new Dictionary<string, Tuple<string, MultiRowOp>>();
-         foreach (var t in operations) {
-            string out_col_name = $"{t.Item1}_{t.Item2}".ToLower();
-            out_schema = out_schema.add_column(TypeCode.Double, out_col_name);
-            out_col_map[out_col_name] = t;
-         }
-         MutableBlueRow or = new MutableBlueRow(out_schema, ir);
-         foreach (var kv in out_col_map) {
-            string out_col_name = kv.Key;
-            string in_col_name = kv.Value.Item1;
-            MultiRowOp op = kv.Value.Item2;
-            IEnumerable<double> col_vals = BlueDataUtility.get_colvalues<double>(rolling_rows, in_col_name);
-            double comp_res = compute_op(col_vals, op);
-            or.setValue(out_col_name, comp_res);
-         }
-         return or;
-      }
-
-      public static double compute_op(IEnumerable<double> col_vals, MultiRowOp op) {
+      public static double compute_op(IEnumerable<double> col_vals, GroupByBundleOp op) {
         switch (op) {
-            case MultiRowOp.Max:
+            case GroupByBundleOp.Max:
               return col_vals.Max<double>();
-            case MultiRowOp.Min:
+            case GroupByBundleOp.Min:
               return col_vals.Min<double>();
-            case MultiRowOp.Mean:
+            case GroupByBundleOp.Mean:
               return col_vals.Sum() / col_vals.Count();
-            case MultiRowOp.Sum:
+            case GroupByBundleOp.Sum:
               return col_vals.Sum();
-            case MultiRowOp.First:
+            case GroupByBundleOp.First:
               return col_vals.First();
-            case MultiRowOp.Last:
+            case GroupByBundleOp.Last:
               return col_vals.Last();
-            case MultiRowOp.StdDev:
-               return MathNet.Numerics.Statistics.Statistics.StandardDeviation(col_vals);
            default:
               // TODO
               Log.assert(false);
@@ -5,7 +5,7 @@ using System.Collections.Generic;
 using System.Linq;

 namespace BCG {
-   internal class GroupByStream : MethodDrivenStream {
+   public class GroupByStream : MethodDrivenStream {
      private class MyFuncGroupByEvaluator : IGroupByEvaluator {
         private string[] _groupby_cols;
         private Func<DataPacket, IBlueRow> _groupby_func;
@@ -1,20 +0,0 @@
-namespace bts.SilverPine {
-   using BCG;
-
-   using System;
-
-   [DataStreamUserRespresentation("AggregationStream")]
-   public class AgregationStream : ViewStream {
-
-      public AgregationStream(ViewStream parent_str, string operation_txt) {
-         ViewStream rs = new NonIncGroupByStream(parent_str, String.Empty, operation_txt);
-         mySetRealSource(rs);
-      }
-
-      public AgregationStream(ViewStream parent_str, string groupby_col, string operation_txt) {
-         ViewStream rs = new NonIncGroupByStream(parent_str, groupby_col, operation_txt);
-         mySetRealSource(rs);
-      }
-   }
-
-}
@@ -8,7 +8,7 @@ using bts;
 using bts.SilverPine;

 namespace BCG {
-   [DataStreamUserRespresentation("AggregateStream")]
+   [DataStreamUserRespresentation("aggregate")]
   public class NonIncGroupByStream : ViewStream {
      private class DelNonIncGroupByEvaluator : INonIncGroupByEvaluator {
         Func<string[]> _groupby_cols_func;
@@ -53,18 +53,15 @@ namespace BCG {
                                 Func<List<IBlueRow>, List<IBlueRow>> evaluate_func)
                                 : this(main_stream, new DelNonIncGroupByEvaluator(groupby_cols_func, evaluate_func)) { }

-      public NonIncGroupByStream(ViewStream parent_str, string groupby_col, string standard_group_by_text)
-      : this(parent_str, groupby_col, standard_group_by_text, String.Empty) { }
-
-      public NonIncGroupByStream(ViewStream parent_str, string groupby_col, string standard_group_by_text, string optional_sort_col) {
-         MultiRowOpHelper std_group_by = MultiRowOpHelper.parse(standard_group_by_text);
+      public NonIncGroupByStream(ViewStream parent_str, string groupby_col, string standard_group_by_text, string optional_sort_col = null) {
+         GroupByBundle std_group_by = GroupByBundle.parse(standard_group_by_text);
         Func<string[]> groupby_cols_func = () => {
            return String.IsNullOrEmpty(groupby_col) ? null : new string[] { groupby_col };
         };
         Func<List<IBlueRow>, List<IBlueRow>> evaluate_func = (group_rows) => {
            if (!String.IsNullOrEmpty(optional_sort_col))
               group_rows.Sort((r1, r2) => BlueDataUtility.compare_rows(r2, r1, optional_sort_col));
-            IBlueRow rv = MultiRowOpHelper.group_by(group_rows, std_group_by);
+            IBlueRow rv = GroupByBundle.group_by(group_rows, std_group_by);
            return new IBlueRow[] { rv }.ToList();
         };
         ViewStream rs = new NonIncGroupByStream(parent_str, groupby_cols_func, evaluate_func);
@@ -74,7 +74,7 @@ namespace BCG {
      }

      public void _on_stream_cb(object dp_obj) {
-         //Log.Gen.info($"NonIncJoinBookkeeper._on_stream_cb dp_obj: {dp_obj}");
+         Log.Gen.info($"NonIncJoinBookkeeper._on_stream_cb dp_obj: {dp_obj}");
         var timer = Stopwatch.StartNew();
         DataPacket dp = (DataPacket)dp_obj;
         _fill_missing_data(dp);
@@ -50,7 +50,7 @@ namespace BCG {
         int data_col_no = 1;
         for (int jp = 0; jp < _join_specs.Length; jp++) {
            NonIncJoinBookkeeper s = new NonIncJoinBookkeeper(_join_specs[jp], jp, data_col_no, _row_hasher);
-            //Log.Gen.info($"NonIncJoinStream.myOnFirstSubscriber jp: {jp} spec: {_join_specs[jp]}");
+            Log.Gen.info($"NonIncJoinStream.myOnFirstSubscriber jp: {jp} spec: {_join_specs[jp]}");
            data_col_no += _join_specs[jp].data_col_names.Length;
            s.on_new_data += _stream_callback;
            _book_keeping_data.Add(s);
@@ -71,7 +71,7 @@ namespace BCG {
      }

      private void _stream_callback() {
-         //Log.Gen.info("NonIncJoinStream._stream_callback received callback");
+         Log.Gen.info("NonIncJoinStream._stream_callback received callback");
         _is_dirty = true;
         doWork();
      }
@@ -99,7 +99,7 @@ namespace BCG {

      private int _n_total_fires = 0, _n_expensive_fires = 0;
      private void _form_and_fire() {
-         //Log.Gen.info($"NonIncJoinStream._form_and_fire _is_ready(): {_is_ready()}");
+         Log.Gen.info($"NonIncJoinStream._form_and_fire _is_ready(): {_is_ready()}");
         if (_is_ready()) {
            _construct_schema();
            var timer = Stopwatch.StartNew();
@@ -126,11 +126,11 @@ namespace BCG {
            _n_total_fires++;
            //if (timer.ElapsedMilliseconds >= 250) {
            _n_expensive_fires++;
-            //string log_msg = $"Join and map time for {_book_keeping_data.Count} "
-            //   + $"tables {timer.Elapsed.TotalMilliseconds} msecs, ratio {_n_expensive_fires}/{_n_total_fires}";
-            //Log.Gen.info(log_msg);
+            string log_msg = $"Join and map time for {_book_keeping_data.Count} "
+               + $"tables {timer.Elapsed.TotalMilliseconds} msecs, ratio {_n_expensive_fires}/{_n_total_fires}";
+            Log.Gen.info(log_msg);
            //}
-            var dp_out = BlueDataUtility.build_dp_inc(dp_output_rows, myLastValue as DataPacket, true);
+            var dp_out = BlueDataUtility.build_dp_inc(dp_output_rows, myLastValue as DataPacket);
            myFireEvent(dp_out);
            //myFireEvent(new DataPacket(dp_output_rows, dp_output_rows, myLastValue == null ? new BlueTable(_output_schema) : (myLastValue as DataPacket).newRows));
         }
@@ -152,7 +152,7 @@ namespace BCG {
      }

      public void doWork() {
-         //Log.Gen.info($"NonIncJoinStream.doWork _last_fire_time: {_last_fire_time} _is_dirty: {_is_dirty}");
+         Log.Gen.info($"NonIncJoinStream.doWork _last_fire_time: {_last_fire_time} _is_dirty: {_is_dirty}");
         if (DateTime.Now - _last_fire_time >= _throttle_period && _is_dirty) {
            _is_dirty = false;
            _last_fire_time = DateTime.Now;
@@ -69,6 +69,4 @@ namespace bts {
         myFireEvent(dp_out);
      }
   }
-
-
 }
@@ -13,8 +13,9 @@ namespace BCG {
   /// desired row ie newRow = f(row), where f is the mapping fucntion. Note that the id 
   /// is preserved across the rows, ie row["id"] = f(row)["id"]
   /// </summary>
-   [DataStreamUserRespresentation(false)]
-   public class MapStream : MethodDrivenStream {
+   [DataStreamUserRespresentation (false)]
+   public class MapStream : MethodDrivenStream
+   {
      protected IStreamMapper myMapper;
      protected IChangeableStreamMapper myChangeableMapper = null;
      private bool myUseNewMap = Configuration.get<DataStreamConfig>().UseNewMap;
@@ -1,14 +0,0 @@
-namespace bts.SilverPine
-{
-   using BCG;
-
-   [DataStreamUserRespresentation("RollingComputationStream")]
-   public class RollingComputationStream : ViewStream
-   {
-      private ViewStream _parent_str;
-      public RollingComputationStream(ViewStream parent_str, string operation_txt, string roll_column, uint window_sz)
-      {
-
-      }
-   }
-}
@@ -1,10 +0,0 @@
-namespace bts.SilverPine {
-   using BCG;
-
-
-   [DataStreamUserRespresentation("RollingComputationStream")]
-   public class RollingComputationStream : ViewStream {
-      public RollingComputationStream(ViewStream parent_str, string operation_txt, string roll_column, uint window_sz) {
-      }
-   }
-}
@@ -108,7 +108,6 @@ namespace bts {

   }

-   [DataStreamUserRespresentation("Project")]
   public class NewProjectStream : ViewStream, IStreamMapper {

      private ViewStream _stream;
@@ -37,42 +37,4 @@ namespace BCG
      }
         
   }
-
-   [DataStreamUserRespresentation("throttle")]
-   public class ThrottleStream : ViewStream{
-      private readonly ViewStream _main_stream;
-      private readonly TimeSpan _ts;
-      private readonly ViewStreamDatabase _vsd;
-      private DateTime _last_fire_time = DateTime.MinValue;
-
-      public ThrottleStream (ViewStream main_stream, int throttle_seconds)
-      :
-      this(main_stream, TimeSpan.FromSeconds(throttle_seconds), ViewStreamDatabase.instance)
-      {
-      }
-
-      public ThrottleStream (ViewStream main_stream, TimeSpan ts, ViewStreamDatabase vsd){
-         _main_stream = main_stream;
-         _ts = ts;
-         _vsd = vsd;
-      }
-
-       protected override void myOnFirstSubscriber() {
-         _main_stream.onChange += _handle_base_stream_update;
-      }
-
-      protected override void myOnLastSubscriber() {
-         _main_stream.onChange -= _handle_base_stream_update;
-      }
-
-      private void _handle_base_stream_update(object dp_obj){
-         if (DateTime.Now - _last_fire_time >= _ts){
-            DataPacket dp_in = dp_obj as DataPacket;
-            _last_fire_time = DateTime.Now;
-            DataPacket dp = BlueDataUtility.build_dp_inc(dp_in.newRows, myLastValue as DataPacket);
-            myFireEvent(dp);
-         }
-      }
- }
-
 }
@@ -1,40 +0,0 @@
-namespace bts.SilverPine{
-   using BGC;
-   using System;
-   [DataStreamUserRespresentation("throttle")]
-   public class ThrottleStream : ViewStream{
-      private readonly ViewStream _main_stream;
-      private readonly TimeSpan _ts;
-      private readonly ViewStreamDatabase _vsd;
-      private DateTime _last_fire_time = DateTime.MinValue;
-
-      public ThrottleStream (ViewStream main_stream, int throttle_seconds)
-      :
-      this(main_stream, TimeSpan.FromSeconds(throttle_seconds), ViewStreamDatabase.instance)
-      {
-      }
-
-      public ThrottleStream (ViewStream main_stream, TimeSpan ts, ViewStreamDatabase vsd){
-         _main_stream = main_stream;
-         _ts = ts;
-         _vsd;
-      }
-
-       protected override void myOnFirstSubscriber() {
-         _main_stream.onChange += _handle_base_stream_update;
-      }
-
-      protected override void myOnLastSubscriber() {
-         _main_stream.onChange -= _handle_base_stream_update;
-      }
-
-      private void _handle_base_stream_update(object dp_obj){
-         if (DateTime.Now - _last_fire_time >= _ts){
-            DataPacket dp_in = dp_obj;
-            _last_fire_time = DateTime.Now;
-            DataPacket dp = BlueDataUtility.build_dp_inc(dp_in.newRow, myLastValue as DataPacket);
-            myFireEvent(dp);
-         }
-      }
- }
-}
@@ -40,4 +40,4 @@ namespace BCG {
            myFireSwitch = myLastFiredPacket.addedRows.Count + myLastFiredPacket.deletedRows.Count == 0 ? false : true;
        }
    }
-}
+}
@@ -1,15 +1,18 @@
 using System;
 using System.Collections.Generic;
-
 using mbts;

-namespace BCG {
-   internal class MutableAppendTable : IBlueRowCollection {
-      private class MyRow : IBlueRow {
+namespace BCG
+{
+   internal class MutableAppendTable : IBlueRowCollection
+   {
+      private class MyRow : IBlueRow
+      {
         private IBlueRow myRow;
         private int myID;

-         public MyRow(IBlueRow r, int id) {
+         public MyRow(IBlueRow r, int id)
+         {
            myRow = r;
            myID = id;
         }
@@ -18,8 +21,10 @@ namespace BCG {

         public IBlueTableSchema schema { get { return myRow.schema; } }

-         public object this[int index] {
-            get {
+         public object this[int index]
+         {
+            get
+            {
               if (myRow.schema[index].colName == "id")
                  return myID;
               else
@@ -47,17 +52,20 @@ namespace BCG {

         public System.Collections.ICollection Keys { get { return myRow.Keys; } }

-         public void Remove(object key) { throw new Exception("The method or operation is not implemented."); }
+         public void Remove(object key) {throw new Exception("The method or operation is not implemented.");}

-         public System.Collections.ICollection Values {
-            get {
+         public System.Collections.ICollection Values
+         {
+            get
+            {
               object[] vals = new object[myRow.Count];
               CopyTo(vals, 0);
               return vals;
            }
         }

-         public object this[object key] {
+         public object this[object key]
+         {
            get { return key is string ? this[key as string] : this[(int)key]; }
            set { throw new Exception("The method or operation is not implemented."); }
         }
@@ -66,7 +74,8 @@ namespace BCG {

         #region ICollection Members

-         public void CopyTo(Array array, int index) {
+         public void CopyTo(Array array, int index)
+         {
            myRow.CopyTo(array, index);
            array.SetValue(myID, index + myRow.schema["id"].colNo);
         }
@@ -81,7 +90,8 @@ namespace BCG {

         #region IEnumerable Members

-         System.Collections.IEnumerator System.Collections.IEnumerable.GetEnumerator() { return new BlueRowEnumerator(this); }
+         System.Collections.IEnumerator System.Collections.IEnumerable.GetEnumerator()
+         { return new BlueRowEnumerator(this); }

         #endregion
      }
@@ -96,7 +106,8 @@ namespace BCG {

      public MutableAppendTable(IBlueTableSchema schema) { myRows = new BlueTable(schema); }

-      public IBlueRow addRow(IBlueRow r, int tableNo) {
+      public IBlueRow addRow(IBlueRow r, int tableNo)
+      {
         if (!myIDMap.ContainsKey(tableNo))
            myIDMap.Add(tableNo, new Dictionary<int, IBlueRow>());
         IBlueRow nr = new MyRow(r, myID++); // BlueDataUtility.getNewReplacedRow(r, myID++, "id");
@@ -105,17 +116,20 @@ namespace BCG {
         return nr;
      }

-      public IBlueRow removeRow(IBlueRow r, int tableNo) {
+      public IBlueRow removeRow(IBlueRow r, int tableNo)
+      {
         IBlueRow dr = myIDMap[tableNo][(int)r["id"]];
         myRows.deleteRow(dr);
         myIDMap[tableNo].Remove((int)r["id"]);
         return dr;
      }

-      public IBlueRowCollection removeTable(int tableNo) {
+      public IBlueRowCollection removeTable(int tableNo)
+      {
         if (!myIDMap.ContainsKey(tableNo))
            return null;
-         else {
+         else
+         {
            BlueTable rv = new BlueTable(myRows.schema);
            foreach (IBlueRow r in myIDMap[tableNo].Values)
               rv.addRow(r);
@@ -125,7 +139,8 @@ namespace BCG {
         }
      }

-      public void clear() {
+      public void clear()
+      {
         myRows.clear();
         myIDMap.Clear();
      }
@@ -136,7 +151,7 @@ namespace BCG {

      public IBlueTableSchema schema { get { return myRows.schema; } }

-      public bool containsID(int id) { return myRows.containsID(id); }
+      public bool containsID(int id) {return myRows.containsID(id);}

      #endregion

@@ -154,7 +169,7 @@ namespace BCG {

      #region IEnumerable Members

-      public System.Collections.IEnumerator GetEnumerator() { return myRows.GetEnumerator(); }
+      public System.Collections.IEnumerator GetEnumerator() {return myRows.GetEnumerator();}

      #endregion
   }
@@ -164,8 +179,8 @@ namespace BCG {
   /// to append (or remove) more streams at
   /// runtime
   /// </summary>
-   [DataStreamUserRespresentation("UnionStream")]
-   public class MutableAppendStream : ViewStream {
+   public class MutableAppendStream : ViewStream
+   {
      private bool mySubscriptionActive = false;
      private MutableAppendTable myTable;
      private int myMovingIndex;
@@ -173,17 +188,22 @@ namespace BCG {
      private bool _allign_col_orders = true;

      public MutableAppendStream(params ViewStream[] streams)
-         : this(new List<ViewStream>(streams)) { }
+         : this(new List<ViewStream>(streams))
+      { }

      public MutableAppendStream(bool allign_col_order, params ViewStream[] streams)
-         : this(allign_col_order, new List<ViewStream>(streams)) { }
+         : this(allign_col_order, new List<ViewStream>(streams))
+      { }

      public MutableAppendStream(ICollection<ViewStream> streams)
-         : this(true, streams) { }
+         : this(true, streams)
+      { }

-      public MutableAppendStream(bool allign_col_orders, ICollection<ViewStream> streams) {
+      public MutableAppendStream(bool allign_col_orders, ICollection<ViewStream> streams)
+      {
         _allign_col_orders = allign_col_orders;
-         foreach (ViewStream vs in streams) {
+         foreach (ViewStream vs in streams)
+         {
            myMovingIndex++;
            ViewStream pj = new ChangeTableNameStream(vs, myMovingIndex.ToString());
            if (_allign_col_orders)
@@ -192,7 +212,8 @@ namespace BCG {
         }
      }

-      public void addStream(ViewStream vs) {
+      public void addStream(ViewStream vs)
+      {
         myMovingIndex++;
         ///the change name is done to identify the callbacks, and the project to
         ///make all the schemas col orders the same
@@ -204,14 +225,19 @@ namespace BCG {
            pj.onChange += new ChangeDelegate(myHandleStreamChanges);
      }

-      public void removeStream(ViewStream vs) {
+      public void removeStream(ViewStream vs)
+      {
         int index = -1;
-         foreach (KeyValuePair<int, Pair<ViewStream, ViewStream>> p in myStreamIndex) {
-            if (p.Value.first == vs) {
+         foreach (KeyValuePair<int, Pair<ViewStream, ViewStream>> p in myStreamIndex)
+         {
+            if (p.Value.first == vs)
+            {
               index = p.Key;
-               if (mySubscriptionActive) {
+               if (mySubscriptionActive)
+               {
                  p.Value.second.onChange -= new ChangeDelegate(myHandleStreamChanges);
-                  if (myTable != null) {
+                  if (myTable != null)
+                  {
                     IBlueRowCollection delRows = myTable.removeTable(p.Key);
                     if (delRows != null)
                        myFireEvent(new DataPacket(myTable, new BlueTable(myTable.schema), delRows));
@@ -224,13 +250,15 @@ namespace BCG {
         }
      }

-      protected override void myOnFirstSubscriber() {
+      protected override void myOnFirstSubscriber()
+      {
         mySubscriptionActive = true;
         foreach (KeyValuePair<int, Pair<ViewStream, ViewStream>> p in myStreamIndex)
            p.Value.second.onChange += new ChangeDelegate(myHandleStreamChanges);
      }

-      protected override void myOnLastSubscriber() {
+      protected override void myOnLastSubscriber()
+      {
         mySubscriptionActive = false;
         foreach (KeyValuePair<int, Pair<ViewStream, ViewStream>> p in myStreamIndex)
            p.Value.second.onChange -= new ChangeDelegate(myHandleStreamChanges);
@@ -238,17 +266,20 @@ namespace BCG {
            myTable.clear();
      }

-      private void myHandleStreamChanges(object newValue) {
+      private void myHandleStreamChanges(object newValue)
+      {         
         DataPacket dp = newValue as DataPacket;
         myTable = myTable ?? new MutableAppendTable(new BlueTableSchema("MutableAppend", dp.schema.columnInfo));
         BlueTable adds = new BlueTable(myTable.schema),
                     dels = new BlueTable(myTable.schema);
         int index = Int32.Parse(dp.schema.tableName);
-         foreach (IBlueRow r in dp.deletedRows) {
+         foreach (IBlueRow r in dp.deletedRows)
+         {
            IBlueRow dr = myTable.removeRow(r, index);
            dels.addRow(dr);
         }
-         foreach (IBlueRow r in dp.addedRows) {
+         foreach (IBlueRow r in dp.addedRows)
+         {
            IBlueRow nr = myTable.addRow(r, index);
            adds.addRow(nr);
         }
@@ -280,28 +280,6 @@ namespace BCG {

      #region json

-      public static void write_to_json_file(this IBlueRowCollection table, string file_path) { 
-         write_dict_to_json_file(file_path, table.to_namevalue_pairs());
-      }
-
-      public static void write_dict_to_json_file(string file_path, List<Dictionary<string, string>> data) {
-         var options = new JsonSerializerOptions { WriteIndented = true };
-         string json = System.Text.Json.JsonSerializer.Serialize(data, options);
-         File.WriteAllText(file_path, json);
-      }
-
-      public static List<Dictionary<string, string>> ReadFromFile(string filePath) {
-         string json = File.ReadAllText(filePath);
-         return System.Text.Json.JsonSerializer.Deserialize<List<Dictionary<string, string>>>(json);
-      }
-
-      public static void write_table_to_json(IBlueRowCollection table) {
-         List<Dictionary<string, string>> rows = new List<Dictionary<string, string>>();
-         foreach (IBlueRow row in table) {
-            row.to_json();
-         }
-      }
-
      public static Dictionary<string, string> convert_json_to_dictionary(string json_string) {
         Dictionary<string, string> dictionary = JsonConvert.DeserializeObject<Dictionary<string, string>>(json_string);
         return dictionary;
@@ -370,14 +348,6 @@ namespace BCG {
         }
         return rv;
      }
-
-      public static List<Dictionary<string,string>> to_namevalue_pairs(this IBlueRowCollection table) {
-         List<Dictionary<string, string>> rv = new List<Dictionary<string, string>>();
-         foreach (IBlueRow r in table) {
-            rv.Add(r.to_dict());
-         }
-         return rv;
-      }
      #region Schema types
      // these are the only types allowed in schemas
      private static IReadOnlyDictionary<TypeCode, Type> _typecode_to_type = new Dictionary<TypeCode, Type>
@@ -1255,20 +1225,6 @@ namespace BCG {
         return rv;
      }

-      public static Dictionary<string, string> to_dict(this IBlueRow r) {
-         Dictionary<string, string> map = new Dictionary<string, string>();
-         foreach (ColumnInfo c in r.schema.columnInfo) {
-            map.Add(c.colName, r[c.colName].ToString());
-         }
-         return map;
-      }
-
-      public static string to_json(this IBlueRow r) {
-         Dictionary<string, string> map = r.to_dict();
-         string json = System.Text.Json.JsonSerializer.Serialize(map);
-         return json;
-      }
-
      public static void compute_differences_fast(IBlueRowCollection new_rows, IBlueRowCollection old_rows,
                                                  out IBlueRowCollection out_d_rows, out IBlueRowCollection out_a_rows) {
         if (old_rows == null || old_rows.Count == 0) {
@@ -2,7 +2,7 @@

  <PropertyGroup>
    <AllowUnsafeBlocks>true</AllowUnsafeBlocks>
-    <TargetFramework>net8.0</TargetFramework>
+    <TargetFramework>net7.0</TargetFramework>
    <Nullable>disable</Nullable>
  </PropertyGroup>

@@ -19,8 +19,8 @@
  </ItemGroup>

  <ItemGroup>
-    <Proto Include="../../lib_spark/spark/config/protos/data_stream_config.proto" CSName="DataStreamConfig">
-      <OutDir>$(ProtoOutputPath)/lib_spark/spark/config/protos</OutDir>
+    <Proto Include="../../lib/config/protos/data_stream_config.proto" CSName="DataStreamConfig">
+      <OutDir>$(ProtoOutputPath)/lib/config/protos</OutDir>
    </Proto>
  </ItemGroup>

@@ -47,10 +47,10 @@ namespace BCG {
      /// </summary>
      /// <param name="assemblyName"></param>
      public void loadAssembly(string assemblyName) {
-         Func<Type, bool> is_blue_orb = (t) => {
-            //if (!assemblyName.Contains("SharedStreams.dll"))
-            //   return true;
-            return t.Namespace == "bts.BlueOrb";// || t.Namespace == "BCG";
+         Func<Type, bool> is_silverpine_only = (t) => {
+            if (!assemblyName.Contains("SharedStreams.dll"))
+               return true;
+            return t.Namespace == "bts.SilverPine";
         };
         bts.Log.Gen.info($"PrimitiveManager.loadAssembly assemblyName: {assemblyName}");
         if (myAssemblyNames.Contains(assemblyName))
@@ -61,7 +61,7 @@ namespace BCG {
            Assembly.LoadFrom(assemblyName);

         foreach (Type type in loadedAssembly.GetTypes())
-            if (!type.IsInterface && is_blue_orb(type)
+            if (!type.IsInterface && is_silverpine_only(type)
               && !type.IsAbstract
               && type.IsSubclassOf(typeof(ViewStream))
               && type.IsPublic) {
@@ -77,7 +77,7 @@ namespace BCG {
                     break;
                  }
               if (isVisible) {
-                 // Log.assert_d(myTypeLookupTable[userRep] == null, type.FullName);
+                  Log.assert_d(myTypeLookupTable[userRep] == null, type.FullName);
                  myTypeLookupTable[userRep] = type;
               }
            }
@@ -1,7 +1,7 @@
 <Project Sdk="Microsoft.NET.Sdk">

  <PropertyGroup>
-    <TargetFramework>net8.0</TargetFramework>
+    <TargetFramework>net7.0</TargetFramework>
    <ImplicitUsings>enable</ImplicitUsings>
    <Nullable>enable</Nullable>
  </PropertyGroup>
@@ -2,7 +2,7 @@

  <PropertyGroup>
    <OutputType>Exe</OutputType>
-    <TargetFramework>net8.0</TargetFramework>
+    <TargetFramework>net7.0</TargetFramework>
    <Nullable>enable</Nullable>
    <AllowUnsafeBlocks>true</AllowUnsafeBlocks>
  </PropertyGroup>
@@ -2,7 +2,7 @@

  <PropertyGroup>
    <OutputType>Exe</OutputType>
-    <TargetFramework>net8.0-windows</TargetFramework>
+    <TargetFramework>net7.0-windows</TargetFramework>
    <Nullable>enable</Nullable>
    <UseWindowsForms>true</UseWindowsForms>
  </PropertyGroup>
@@ -1,7 +1,7 @@
 <Project Sdk="Microsoft.NET.Sdk">

  <PropertyGroup>
-    <TargetFramework>net8.0</TargetFramework>
+    <TargetFramework>net7.0</TargetFramework>
    <ImplicitUsings>enable</ImplicitUsings>
    <Nullable>enable</Nullable>

@@ -1,7 +1,7 @@
 <Project Sdk="Microsoft.NET.Sdk">

  <PropertyGroup>
-    <TargetFramework>net8.0</TargetFramework>
+    <TargetFramework>net7.0</TargetFramework>
    <ImplicitUsings>enable</ImplicitUsings>
    <Nullable>enable</Nullable>
  </PropertyGroup>
@@ -1,7 +1,7 @@
 <Project Sdk="Microsoft.NET.Sdk">

  <PropertyGroup>
-    <TargetFramework>net8.0-windows</TargetFramework>
+    <TargetFramework>net7.0-windows</TargetFramework>
    <ImplicitUsings>enable</ImplicitUsings>
    <Nullable>enable</Nullable>
    <UseWindowsForms>true</UseWindowsForms>
@@ -6,18 +6,12 @@ namespace BCG
 	public interface IPublisher
 	{
      void publish( IPubItem pi );
-#pragma warning disable SYSLIB0011 // Type or member is obsolete
      void publish( IPubItem pi, IFormatter f );
-#pragma warning restore SYSLIB0011 // Type or member is obsolete
      void publish( IPubItem[] pi );
-#pragma warning disable SYSLIB0011 // Type or member is obsolete
      void publish( IPubItem[] pi, IFormatter f );
-#pragma warning restore SYSLIB0011 // Type or member is obsolete
   }

-#pragma warning disable SYSLIB0011 // Type or member is obsolete
   public interface IPublisher<T> where T : IFormatter
-#pragma warning restore SYSLIB0011 // Type or member is obsolete
   {
      void publish (IPubItem pi);
      void publish (IPubItem[] pi);
@@ -5,21 +5,23 @@ using System.Runtime.Serialization.Formatters.Binary;

 namespace BCG
 {
-#pragma warning disable SYSLIB0011 // Type or member is obsolete
   public class GenericFormatter<F> : IGenericFormatter where F : IFormatter, new ()
   {
      IFormatter myFormatter = new F ();

      public T Deserialize<T> (Stream serializationStream)
      {
+         #pragma warning disable 
         return (T) myFormatter.Deserialize (serializationStream);
+         #pragma warning restore SYSLIB0011
      }
      public void Serialize<T> (Stream serializationStream, T graph)
      {
+         #pragma warning disable 
         myFormatter.Serialize (serializationStream, graph);
+         #pragma warning restore SYSLIB0011
      }
   }

   public class GenericBinaryFormatter : GenericFormatter<BinaryFormatter> { }
-#pragma warning restore SYSLIB0011 // Type or member is obsolete
 }
@@ -6,7 +6,7 @@

  <PropertyGroup>
      <AllowUnsafeBlocks>true</AllowUnsafeBlocks>
-    <TargetFramework>net8.0</TargetFramework>
+    <TargetFramework>net7.0</TargetFramework>
    <Nullable>enable</Nullable>
  </PropertyGroup>

@@ -2,7 +2,7 @@

  <PropertyGroup>
    <OutputType>Exe</OutputType>
-    <TargetFramework>net8.0</TargetFramework>
+    <TargetFramework>net7.0</TargetFramework>
    <Nullable>enable</Nullable>
  </PropertyGroup>

@@ -1,18 +0,0 @@
-namespace bts.BlueOrb {
-   using bts.SilverPine;
-   using BCG;
-   using System;
-
-   [DataStreamUserRespresentation("SpatialArbScoreStream")]
-   public class SpatialArbScoreStream : ViewStream {
-      public SpatialArbScoreStream(int window_sz_secs, int sample_period_sz)
-         : this(TimeSpan.FromSeconds(window_sz_secs), TimeSpan.FromSeconds(sample_period_sz), ViewStreamDatabase.instance) { }
-
-      public SpatialArbScoreStream(TimeSpan window_sz, TimeSpan sample_period, ViewStreamDatabase vsd) {
-         ViewStream rs = new SpatialArbStatsStream(window_sz, sample_period, vsd);
-         rs = new ThrottleStream(rs, TimeSpan.FromSeconds(1), vsd);
-         rs = new SortStream(rs, "-n_trade_ticks");
-         mySetRealSource(rs);
-      }
-   }
-}
@@ -1,113 +0,0 @@
-namespace bts.SilverPine {
-   using BCG;
-
-   using SharpCompress.Common.Rar;
-
-   using System;
-   using System.Collections.Generic;
-   using System.Linq;
-   using System.Text;
-   using System.Threading.Tasks;
-
-   // For all the pairs generated by SpatialFuturePairsStream, this stream 
-   // generates the statistics, by appending together a SpreadStatsStream
-   [DataStreamUserRespresentation("SpatialArbStatsStream")]
-   public class SpatialArbStatsStream : ViewStream{
-      private ViewStream _spatial_pairs_stream;
-      private ViewStreamDatabase _vsd;
-      private ViewStream _tot_str = null;
-      private HashSet<string> _key_map = new HashSet<string>();
-      private ViewStream _bbo_stream;
-      private readonly TimeSpan _window_sz, _sampling_period_sz;
-
-      public SpatialArbStatsStream(int window_sz_seconds, int sampling_period_sz_seconds)
-         : this(TimeSpan.FromSeconds(window_sz_seconds), TimeSpan.FromSeconds(sampling_period_sz_seconds), ViewStreamDatabase.instance) { }
-
-      public SpatialArbStatsStream(TimeSpan window_sz, TimeSpan sampling_period_sz, ViewStreamDatabase vsd) {
-         _vsd = vsd;
-         _window_sz = window_sz;
-         _sampling_period_sz = sampling_period_sz;
-         _spatial_pairs_stream = new SpatialFuturePairsStream(vsd);
-         _bbo_stream = new BBOStream(vsd);
-      }
-
-      protected override void myOnFirstSubscriber() {
-         _spatial_pairs_stream.onChange += _spatial_pairs_stream_onchange;
-      }
-
-      protected override void myOnLastSubscriber() {
-         _spatial_pairs_stream.onChange -= _spatial_pairs_stream_onchange;
-      }
-
-      private ViewStream _build_mapped_stream(ViewStream parent_str) {
-         Func<IBlueTableSchema, IBlueTableSchema> build_schema = (inp_schema) => {
-            IBlueTableSchema rv = new BlueTableSchema("SpreaddStats", new ColumnInfo("id", 0, TypeCode.Int32));
-            rv = rv.add_column(TypeCode.String, "spread");
-            rv = rv.add_column(TypeCode.String, "window");
-            // bids
-            rv = rv.add_column(TypeCode.Double, "bid_mean");
-            rv = rv.add_column(TypeCode.Double, "bid_stddev");
-            rv = rv.add_column(TypeCode.Double, "bid_skewness");
-            rv = rv.add_column(TypeCode.Double, "bid_kurtosis");
-            rv = rv.add_column(TypeCode.Double, "bid_max");
-            rv = rv.add_column(TypeCode.Double, "bid_min");
-            // asks
-            rv = rv.add_column(TypeCode.Double, "ask_mean");
-            rv = rv.add_column(TypeCode.Double, "ask_stddev");
-            rv = rv.add_column(TypeCode.Double, "ask_skewness");
-            rv = rv.add_column(TypeCode.Double, "ask_kurtosis");
-            rv = rv.add_column(TypeCode.Double, "ask_max");
-            rv = rv.add_column(TypeCode.Double, "ask_min");
-            // tail distribution
-            rv = rv.add_column(TypeCode.Double, "bid Z ≥ 2σ");
-            rv = rv.add_column(TypeCode.Double, "ask Z < -2σ");
-            // trading ticks
-            rv = rv.add_column(TypeCode.Int32, "n_buy_ticks");
-            rv = rv.add_column(TypeCode.Double, "cumm_buy_price");
-            rv = rv.add_column(TypeCode.Int32, "n_sell_ticks");
-            rv = rv.add_column(TypeCode.Double, "cumm_sell_price");
-            rv = rv.add_column(TypeCode.Int32, "n_trade_ticks");
-            return rv;
-         };
-         Func<IBlueRow, IBlueRow> build_row = (r) => {
-            IBlueTableSchema s = build_schema(r.schema);
-            return new BlueRow(s, r["id"], r["spread"], r["window"],
-                           r["bid_mean"], r["bid_stddev"], r["bid_skewness"], r["bid_kurtosis"], r["bid_max"], r["bid_min"],
-                           r["ask_mean"], r["ask_stddev"], r["ask_skewness"], r["ask_kurtosis"], r["ask_max"], r["ask_min"],
-                           r["bid Z ≥ 2σ"], r["ask Z < -2σ"],
-                           r["n_buy_ticks"], r["cumm_buy_price"], r["n_sell_ticks"], r["cumm_sell_price"], r["n_trade_ticks"]);
-         };
-         MapStream rs = new MapStream(parent_str, inp_schema => build_schema(inp_schema),
-            inp_r => build_row(inp_r));
-         return rs;
-      }
-
-      private void _spatial_pairs_stream_onchange(object dp_obj) {
-         if (_tot_str != null)
-            return;
-         MutableAppendStream append_str = new MutableAppendStream();
-         _tot_str = _build_mapped_stream(append_str);
-         //_tot_str = new ThrottleStream(_tot_str, 1000);
-         //_tot_str = new SortStream(_tot_str, "-n_trade_ticks");
-         _tot_str.onChange += _handle_tot_stream;
-
-         DataPacket dp = dp_obj as DataPacket;         
-         foreach (IBlueRow r in dp.newRows) {
-            string s1 = r.get<string>("s1");
-            string s2 = r.get<string>("s2");
-            string key = SpatialFuturePairsStream.get_key(s1, s2);
-            if (!_key_map.Contains(key)) {
-               _key_map.Add(key);
-               ViewStream spread_stats_stream = new SpreadStatsStream(s1, s2, 1, 1, Convert.ToInt32(_window_sz.TotalSeconds), 
-                                                                     Convert.ToInt32(_sampling_period_sz.TotalSeconds), _bbo_stream, _vsd);
-               append_str.addStream(spread_stats_stream);
-            }
-         }
-      }
-
-      private void _handle_tot_stream(object dp_obj) {
-         myFireEvent(dp_obj);
-      }
-   }
-
-}
@@ -1,89 +0,0 @@
-namespace bts.SilverPine {
-   using BCG;
-
-   using bts.api;
-
-   using System;
-   using System.Collections.Generic;
-   using System.Linq;
-   using System.Text;
-   using System.Threading.Tasks;
-
-   [DataStreamUserRespresentation("SpatialFuturePairsStream")]
-   public class SpatialFuturePairsStream : ViewStream {
-      private ViewStream _main_stream;
-      private IBlueTableSchema _schema;
-      private HashSet<string> _key_map = new HashSet<string>();
-      public SpatialFuturePairsStream()
-         : this(ViewStreamDatabase.instance) { }
-
-      public SpatialFuturePairsStream(ViewStreamDatabase vsd) {
-         _main_stream = new InstrumentsStream(vsd);
-         _main_stream = new FilterStream(_main_stream, "sec_type='Future'");
-         _schema = new BlueTableSchema("SpatialFutures", new ColumnInfo("id", 0, TypeCode.Int32));
-         _schema = _schema.add_column(TypeCode.String, "s1");
-         _schema = _schema.add_column(TypeCode.String, "venue_1");
-         _schema = _schema.add_column(TypeCode.String, "s2");
-         _schema = _schema.add_column(TypeCode.String, "venue_2");
-
-      }
-
-      protected override void myOnFirstSubscriber() {
-         _main_stream.onChange += _main_stream_onchange;
-      }
-
-      protected override void myOnLastSubscriber() {
-         _main_stream.onChange -= _main_stream_onchange;
-      }
-
-      public static string get_key(string s1, string s2) {
-         string key = string.Compare(s1, s2, StringComparison.Ordinal) <= 0 ? s1 + s2 : s2 + s1;
-         return key;
-      }
-
-      public static string get_key(IBlueRow r1, IBlueRow r2) {
-         string s1 = r1.get<string>("sec");
-         string s2 = r2.get<string>("sec");
-         return get_key(s1, s2);
-      }
-
-      private bool _combine(IBlueRow r1, IBlueRow r2) {
-         string key = get_key(r1, r2);
-         if (_key_map.Contains(key))
-            return false;
-         if (r1.get<string>("symbol") != r2.get<string>("symbol"))
-            return false;
-         if (r1.get_enum<Venue>("venue") == r2.get_enum<Venue>("venue"))
-            return false;
-         if (r1.get<int>("sec_key") == r2.get<int>("sec_key")) {
-            return false;
-         }
-         DateTime e1 = r1.get_datetime("expiration");
-         DateTime e2 = r2.get_datetime("expiration");
-         bool exps_eq = e1 == e2 || (e1.Year >= 2100 && e2.Year >= 2100);
-         if (!exps_eq)
-            return false;         
-         return true;
-      }
-
-      private void _main_stream_onchange(object dp_obj) {
-         _key_map.Clear();
-         DataPacket dp = dp_obj as DataPacket;
-         BlueTable table = new BlueTable(_schema);
-         var rows = dp.newRows.to_list();
-         foreach (var r1 in rows) {
-            foreach (var r2 in rows) {
-               if (_combine(r1, r2)) {
-                  IBlueRow nr = new BlueRow(_schema, table.Count, r1.get<string>("sec"), r1.get<string>("venue"),
-                     r2.get<string>("sec"), r2.get<string>("venue"));
-                  table.addRow(nr);
-                  string key = get_key(r1, r2);
-                  _key_map.Add(key);
-               }
-            }
-         }
-         var dp_out = BlueDataUtility.build_dp(table, myLastValue as DataPacket);
-         myFireEvent(dp_out);
-      }
-   }
-}
@@ -1,656 +0,0 @@
-using System;
-using System.Collections.Generic;
-using System.Linq;
-using MathNet.Numerics.Statistics;
-using MathNet.Numerics.LinearRegression;
-using System.Collections.Generic;
-using System.Linq;
-using MathNet.Numerics.Statistics;
-using MathNet.Numerics.LinearAlgebra;
-using Amazon.Runtime;
-using System;
-using System.Collections.Generic;
-using System.Linq;
-using MathNet.Numerics.Statistics;
-using System;
-using System.Collections.Generic;
-using System.Linq;
-using MathNet.Numerics.Statistics;
-using System;
-using System.Collections.Generic;
-using System.Linq;
-using MathNet.Numerics.Statistics;
-// stats calculator
-using System;
-using System.Collections.Generic;
-using System.Linq;
-using MathNet.Numerics.Statistics;
-
-/// <summary>
-/// Provides evaluation metrics to assess the quality of a synthetic spread for mean-reversion trading.
-/// Assumes the input spread data is a trailing, time-causal window of recent ticks (e.g., last 120 seconds).
-/// </summary>
-public static class SpreadPerformanceEvaluator
-{
-
-   public static class SpreadStatistics {
-      public class SpreadStatsResult {
-         public Dictionary<string, double> bid_zscore_distribution { get; set; }
-         public Dictionary<string, double> ask_zscore_distribution { get; set; }
-         public Dictionary<string, double> mid_zscore_distribution { get; set; }
-
-         public DescriptiveStatistics bid_stats { get; set; }
-         public DescriptiveStatistics ask_stats { get; set; }
-         public DescriptiveStatistics mid_stats { get; set; }
-      }
-
-      private static readonly string[] _zscore_labels = new[]
-      {
-        "Z < -2σ",
-        "-2σ ≤ Z < -1σ",
-        "-1σ ≤ Z < 0σ",
-        "0σ ≤ Z < 1σ",
-        "1σ ≤ Z < 2σ",
-        "Z ≥ 2σ"
-    };
-
-      public static SpreadStatsResult analyze_spread_quotes(List<(DateTime timestamp, double bid, double bid_qty, double ask, double ask_qty)> quotes) {
-         if (quotes == null || quotes.Count == 0)
-            throw new ArgumentException("Spread quote list cannot be null or empty.");
-
-         var bid_prices = quotes.Select(q => q.bid).ToList();
-         var ask_prices = quotes.Select(q => q.ask).ToList();
-         var mid_prices = quotes.Select(q => (q.bid + q.ask) / 2.0).ToList();
-
-         var bid_stats = new DescriptiveStatistics(bid_prices);
-         var ask_stats = new DescriptiveStatistics(ask_prices);
-         var mid_stats = new DescriptiveStatistics(mid_prices);
-
-         var bid_zscores = bid_prices.Select(x => zscore(x, bid_stats.Mean, bid_stats.StandardDeviation)).ToList();
-         var ask_zscores = ask_prices.Select(x => zscore(x, ask_stats.Mean, ask_stats.StandardDeviation)).ToList();
-         var mid_zscores = mid_prices.Select(x => zscore(x, mid_stats.Mean, mid_stats.StandardDeviation)).ToList();
-
-         var bid_distribution = bin_zscores_to_percentages(bid_zscores);
-         var ask_distribution = bin_zscores_to_percentages(ask_zscores);
-         var mid_distribution = bin_zscores_to_percentages(mid_zscores);
-
-         return new SpreadStatsResult {
-            bid_zscore_distribution = bid_distribution,
-            ask_zscore_distribution = ask_distribution,
-            mid_zscore_distribution = mid_distribution,
-            bid_stats = bid_stats,
-            ask_stats = ask_stats,
-            mid_stats = mid_stats
-         };
-      }
-
-      private static double zscore(double x, double mean, double std_dev) {
-         return std_dev == 0 ? 0 : (x - mean) / std_dev;
-      }
-
-      private static Dictionary<string, double> bin_zscores_to_percentages(List<double> zscores) {
-         var bins = _zscore_labels.ToDictionary(label => label, _ => 0);
-
-         foreach (var z in zscores) {
-            if (z < -2) bins["Z < -2σ"]++;
-            else if (z < -1) bins["-2σ ≤ Z < -1σ"]++;
-            else if (z < 0) bins["-1σ ≤ Z < 0σ"]++;
-            else if (z < 1) bins["0σ ≤ Z < 1σ"]++;
-            else if (z < 2) bins["1σ ≤ Z < 2σ"]++;
-            else bins["Z ≥ 2σ"]++;
-         }
-
-         double total = zscores.Count;
-         return bins.ToDictionary(kvp => kvp.Key, kvp => total == 0 ? 0 : (kvp.Value / total) * 100.0);
-      }
-
-      public static double score_spread(SpreadStatsResult result) {
-         var ask_bins = result.ask_zscore_distribution;
-         var bid_bins = result.bid_zscore_distribution;
-         var mid_bins = result.mid_zscore_distribution;
-
-         double buy_opportunity = ask_bins["Z < -2σ"] + ask_bins["-2σ ≤ Z < -1σ"];
-         double sell_opportunity = bid_bins["Z ≥ 2σ"] + bid_bins["1σ ≤ Z < 2σ"];
-         double opportunity_score = buy_opportunity + sell_opportunity;
-
-         double below_mean = mid_bins["Z < -2σ"] + mid_bins["-2σ ≤ Z < -1σ"] + mid_bins["-1σ ≤ Z < 0σ"];
-         double above_mean = mid_bins["Z ≥ 2σ"] + mid_bins["1σ ≤ Z < 2σ"] + mid_bins["0σ ≤ Z < 1σ"];
-         double symmetry_score = 1.0 - Math.Abs(below_mean - above_mean) / (below_mean + above_mean + 1e-6);
-
-         double amplitude_score = result.mid_stats.StandardDeviation;
-         double skew_penalty = 1.0 / (1.0 + Math.Abs(result.mid_stats.Skewness));
-         double kurtosis_bonus = Math.Min(result.mid_stats.Kurtosis, 6.0) / 6.0;
-
-         double score =
-             opportunity_score *
-             symmetry_score *
-             amplitude_score *
-             skew_penalty *
-             (1.0 + kurtosis_bonus * 0.1);
-
-         return score;
-      }
-   }
-
-
-
-
-
-/// <summary>
-/// Represents a single trade action (entry or exit) taken during the simulation.
-/// </summary>
-public struct TradeAction
-    {
-        public bool IsEntry { get; set; }    // True if this action opened a position
-        public bool IsLong { get; set; }     // True = Long, False = Short
-        public double Price { get; set; }    // Executed price
-        public int Index { get; set; }       // Tick index of the action
-
-        public override string ToString()
-        {
-            string side = IsLong ? "Long" : "Short";
-            string type = IsEntry ? "Open" : "Close";
-            return $"{type} {side} @ {Price:F4} (Tick {Index})";
-        }
-    }
-
-    /// <summary>
-/// Simulates a mean-reversion strategy using rolling, causal mid-price stats.
-/// ✅ Entry:
-///   - Buy at ask when ask < rollingMean - entryThreshold × rollingStdDev
-///   - Sell at bid when bid > rollingMean + entryThreshold × rollingStdDev
-/// ✅ Exit:
-///   - Close when mid returns within rollingMean ± exitThreshold × rollingStdDev
-///   - Long exits at bid, short exits at ask
-/// This version uses only past and current data at each step (no forward-looking bias).
-/// </summary>
-public static (double TotalPnL, List<TradeAction> TradeActions, int BuyTurns, int SellTurns) EvaluateSimulatedPnL(
-    List<Tuple<double, double, double, double>> spreadData,
-    int statWindow = 30,
-    double entryThresholdMultiplier = 0.5,
-    double exitThresholdMultiplier = 0.1)
-{
-    var trades = new List<TradeAction>();
-    var mids = spreadData.Select(x => (x.Item1 + x.Item3) / 2.0).ToList();
-
-    if (spreadData.Count < statWindow)
-        return (0.0, trades, 0, 0);
-
-    bool positionOpen = false;
-    bool isLong = false;
-    double entryPrice = 0.0;
-    double pnl = 0.0;
-    int buyTurns = 0;
-    int sellTurns = 0;
-
-    for (int i = statWindow; i < spreadData.Count; i++)
-    {
-        double bid = spreadData[i].Item1;
-        double ask = spreadData[i].Item3;
-        double mid = mids[i];
-
-        // Rolling window
-        var window = mids.Skip(i - statWindow).Take(statWindow).ToList();
-        double mean = window.Average();
-        double stddev = Math.Sqrt(window.Select(x => Math.Pow(x - mean, 2)).Average());
-
-        double lowerThreshold = mean - entryThresholdMultiplier * stddev;
-        double upperThreshold = mean + entryThresholdMultiplier * stddev;
-        double exitBand = exitThresholdMultiplier * stddev;
-
-        if (!positionOpen)
-        {
-            if (ask < lowerThreshold)
-            {
-                positionOpen = true;
-                isLong = true;
-                entryPrice = ask;
-                trades.Add(new TradeAction { IsEntry = true, IsLong = true, Price = entryPrice, Index = i });
-            }
-            else if (bid > upperThreshold)
-            {
-                positionOpen = true;
-                isLong = false;
-                entryPrice = bid;
-                trades.Add(new TradeAction { IsEntry = true, IsLong = false, Price = entryPrice, Index = i });
-            }
-        }
-        else
-        {
-            if (Math.Abs(mid - mean) < exitBand)
-            {
-                double exitPrice = isLong ? bid : ask;
-                pnl += isLong ? (exitPrice - entryPrice) : (entryPrice - exitPrice);
-
-                trades.Add(new TradeAction { IsEntry = false, IsLong = isLong, Price = exitPrice, Index = i });
-
-                if (isLong) buyTurns++;
-                else sellTurns++;
-
-                positionOpen = false;
-            }
-        }
-    }
-
-    return (pnl, trades, buyTurns, sellTurns);
-}
-
-
-
-    /// <summary>
-    /// Estimates the Hurst exponent of the spread mid-prices.
-    /// Assumes the input is a trailing window of recent data.
-    /// Hurst < 0.5 implies mean-reversion; ideal is 0.35–0.45.
-    /// </summary>
-    public static double EvaluateHurstExponent(List<Tuple<double, double, double, double>> spreadData)
-    {
-        const int MinLength = 21;
-
-        if (spreadData == null || spreadData.Count < MinLength)
-            return double.NaN;
-
-        var spreadMids = spreadData.Select(x => (x.Item1 + x.Item3) / 2.0).ToArray();
-        int n = spreadMids.Length;
-
-        var lags = new List<double>();
-        var tau = new List<double>();
-
-        for (int lag = 2; lag <= 20; lag++)
-        {
-            if (n - lag <= 1)
-                continue;
-
-            double[] diffs = new double[n - lag];
-            for (int i = 0; i < diffs.Length; i++)
-                diffs[i] = spreadMids[i + lag] - spreadMids[i];
-
-            double stddev = diffs.StandardDeviation();
-
-            if (stddev > 0)
-            {
-                lags.Add(Math.Log(lag));
-                tau.Add(Math.Log(stddev));
-            }
-        }
-
-        if (lags.Count < 2)
-            return double.NaN;
-
-        var (intercept, slope) = SimpleRegression.Fit(lags.ToArray(), tau.ToArray());
-        double hurst = slope * 0.5 + 0.5;
-        return hurst;
-    }
-
-    /// <summary>
-/// Computes the mean and standard deviation of the spread mid-prices
-/// over a trailing, causal window.
-/// </summary>
-/// <param name="spreadData">List of (bid, bidQty, ask, askQty)</param>
-/// <returns>Tuple of (Mean, StdDev). Returns (NaN, NaN) if input is too short.</returns>
-public static (double Mean, double StdDev) EvaluateMeanAndStdDev(List<Tuple<double, double, double, double>> spreadData)
-{
-    if (spreadData == null || spreadData.Count < 2)
-        return (double.NaN, double.NaN);
-
-    var mids = spreadData.Select(x => (x.Item1 + x.Item3) / 2.0).ToList();
-    double mean = mids.Average();
-    double stddev = Math.Sqrt(mids.Select(x => Math.Pow(x - mean, 2)).Average());
-
-    return (mean, stddev);
-}
-
-
-   /// <summary>
-   /// Counts how many times the spread ask dropped below (mean - threshold) [buy signal]
-   /// and how many times the spread bid rose above (mean + threshold) [sell signal].
-   /// Assumes spreadData is a trailing, causal window.
-   /// </summary>
-   /// <returns>(BuyExcursions, SellExcursions)</returns>
-   public static (uint BuyExcursions, double cumm_buy_price, uint SellExcursions, double cumm_sell_price) EvaluateExcursionCount(
-       List<Tuple<double, double, double, double>> spreadData,
-       double entryThresholdMultiplier = 0.5) {
-      if (spreadData == null || spreadData.Count < 2)
-         return (0, Double.NaN, 0, Double.NaN);
-
-      var midPrices = spreadData.Select(x => (x.Item1 + x.Item3) / 2.0).ToList();
-      double mean = midPrices.Average();
-      double stddev = midPrices.StandardDeviation();
-      double buy_threshold = mean - entryThresholdMultiplier * stddev;
-      double sell_threshold = mean + entryThresholdMultiplier * stddev;
-
-      int buyCount = 0;
-      int sellCount = 0;
-      //bool in_buy_zone = false;
-      //bool in_sell_zone = false;
-
-      double cumm_buy_price = 0.0, cumm_sell_price = 0.0;
-      foreach (var (bid, _, ask, _) in spreadData) {
-         if (ask <= buy_threshold) {
-            //if (!in_buy_zone) {
-            cumm_buy_price += ask;
-            buyCount++;
-            //in_buy_zone = true;
-         }
-         //} else {
-         //   in_buy_zone = false;
-         //}
-
-         if (bid >= sell_threshold) {
-            //if (!in_sell_zone) {
-            cumm_sell_price += bid;
-            sellCount++;
-            //in_sell_zone = true;
-         }
-         //} else {
-         //   in_sell_zone = false;
-         //}
-      }
-
-      return ((uint)buyCount, cumm_buy_price / buyCount, (uint)sellCount, cumm_sell_price / sellCount);
-   }
-
-/// <summary>
-/// Counts how many times the spread mid-price crosses its mean level.
-/// Each crossing indicates a potential oscillation turn (cycle midpoint).
-/// Assumes the input is a trailing, causal window of spread ticks.
-/// </summary>
-/// <param name="spreadData">List of (bid, bidQty, ask, askQty)</param>
-/// <returns>Number of zero crossings in the mid-price series</returns>
-public static UInt32 EvaluateZeroCrossingCount(List<Tuple<double, double, double, double>> spreadData)
-{
-    if (spreadData == null || spreadData.Count < 2)
-        return 0;
-
-    var mids = spreadData.Select(x => (x.Item1 + x.Item3) / 2.0).ToList();
-    double mean = mids.Average();
-    int zeroCrossings = 0;
-
-    for (int i = 1; i < mids.Count; i++)
-    {
-        double prevOffset = mids[i - 1] - mean;
-        double currOffset = mids[i] - mean;
-
-        if (prevOffset * currOffset < 0) // Sign changed
-            zeroCrossings++;
-    }
-
-    return (UInt32)zeroCrossings;
-}
-
-   /// <summary>
-   /// Computes the half-life of mean reversion.
-   /// Lower half-life means the spread reverts to its mean more quickly.
-   /// </summary>
-   public static double ComputeHalfLife(List<Tuple<double, double, double, double>> spreadData) {
-      var prices = spreadData.Select(x => (x.Item1 + x.Item3) / 2.0).ToList();
-      var lagged = prices.Take(prices.Count - 1).ToArray();
-      var delta = prices.Skip(1).Zip(lagged, (cur, prev) => cur - prev).ToArray();
-
-      if (lagged.Length < 2) return double.NaN;
-
-      var x = Vector<double>.Build.Dense(lagged);
-      var y = Vector<double>.Build.Dense(delta);
-      var beta = x.DotProduct(y) / x.DotProduct(x);
-
-      return -Math.Log(2) / beta;
-   }
-
-   /// <summary>
-   /// Computes the lag-1 autocorrelation of the spread.
-   /// Lower values indicate quicker mean-reversion.
-   /// </summary>
-   public static double ComputeLag1Autocorrelation(List<Tuple<double, double, double, double>> spreadData) {
-      var prices = spreadData.Select(x => (x.Item1 + x.Item3) / 2.0).ToList();
-      var mean = prices.Average();
-      var numerator = 0.0;
-      var denominator = 0.0;
-
-      for (int i = 1; i < prices.Count; i++) {
-         numerator += (prices[i] - mean) * (prices[i - 1] - mean);
-         denominator += Math.Pow(prices[i - 1] - mean, 2);
-      }
-
-      return denominator == 0 ? 0 : numerator / denominator;
-   }
-
-   /// <summary>
-   /// Estimates how quickly extreme Z-score values decay back toward the mean.
-   /// Shorter average decay time indicates stronger mean-reversion.
-   /// </summary>
-   public static double ComputeAverageZScoreDecay(List<Tuple<double, double, double, double>> spreadData, int zScoreWindow = 50) {
-      var prices = spreadData.Select(x => (x.Item1 + x.Item3) / 2.0).ToList();
-      if (prices.Count < zScoreWindow + 2) return double.NaN;
-
-      var decayTimes = new List<int>();
-      for (int i = zScoreWindow; i < prices.Count - 1; i++) {
-         var window = prices.Skip(i - zScoreWindow).Take(zScoreWindow).ToList();
-         double mean = window.Average();
-         double stdDev = Statistics.StandardDeviation(window);
-         if (stdDev == 0) continue;
-
-         double z = (prices[i] - mean) / stdDev;
-         if (Math.Abs(z) > 2) {
-            for (int j = 1; j < prices.Count - i; j++) {
-               double zNext = (prices[i + j] - mean) / stdDev;
-               if (Math.Abs(zNext) < 1) {
-                  decayTimes.Add(j);
-                  break;
-               }
-            }
-         }
-      }
-
-      return decayTimes.Count == 0 ? double.NaN : decayTimes.Average();
-   }
-
-   /// <summary>
-   /// Computes the variance ratio: short-term variance divided by long-term variance.
-   /// Lower ratio suggests stronger mean-reversion.
-   /// </summary>
-   public static double ComputeVarianceRatio(List<Tuple<double, double, double, double>> spreadData, int shortWindow, int longWindow) {
-      var prices = spreadData.Select(x => (x.Item1 + x.Item3) / 2.0).ToList();
-      if (prices.Count < longWindow) return double.NaN;
-
-      double shortVar = Statistics.Variance(prices.Skip(prices.Count - shortWindow).Take(shortWindow));
-      double longVar = Statistics.Variance(prices.Skip(prices.Count - longWindow).Take(longWindow));
-
-      return longVar == 0 ? double.NaN : shortVar / longVar;
-   }
-
-   /// <summary>
-   /// Computes the highest bid and the lowest ask from a spread tick series.
-   /// Useful for evaluating the best market opportunity range.
-   /// </summary>
-   /// <param name="spreadData">List of (bid, bidQty, ask, askQty)</param>
-   /// <returns>Tuple of (MaxBid, MinAsk)</returns>
-   public static (double MaxBid, double MinAsk) EvaluateMaxBidMinAsk(List<Tuple<double, double, double, double>> spreadData)
-{
-    if (spreadData == null || spreadData.Count == 0)
-        return (double.NaN, double.NaN);
-
-    double maxBid = double.NegativeInfinity;
-    double minAsk = double.PositiveInfinity;
-
-    foreach (var (bid, _, ask, _) in spreadData)
-    {
-        if (bid > maxBid) maxBid = bid;
-        if (ask < minAsk) minAsk = ask;
-    }
-
-    return (maxBid, minAsk);
-}
-
-/// <summary>
-/// Computes the distribution of spread mid-prices around the mean,
-/// grouped by caller-defined distance bands (in standard deviations).
-///
-/// This helps answer:
-/// - Is the spread tightly hugging the mean?
-/// - Are there wide excursions or heavy tails?
-///
-/// You define breakpoints like: [0.5, 1.0, 2.0] → it will return:
-/// - "≤ 0.5σ"
-/// - "0.5σ – 1.0σ"
-/// - "1.0σ – 2.0σ"
-/// - "> 2.0σ"
-///
-/// How to Interpret It:
-/// - ≤ 0.5σ        → Tight clustering near the mean (strong mean-reversion)
-/// - 0.5σ – 1.0σ   → Mild oscillation (common trade region)
-/// - 1.0σ – 2.0σ   → Moderate deviation (potential trade signals)
-/// - > 2.0σ        → Rare, extreme moves (tail events, potential entries/exits)
-///
-/// Use this to tune entry thresholds or detect regime shifts.
-/// </summary>
-/// <param name="spreadData">List of (bid, bidQty, ask, askQty)</param>
-/// <param name="regions">List of positive breakpoints in std deviations (e.g., [0.5, 1.0, 2.0])</param>
-/// <returns>Dictionary mapping label to % of mid-prices in that region</returns>
-public static Dictionary<string, double> EvaluateDistributionAroundMean(
-    List<Tuple<double, double, double, double>> spreadData,
-    List<double> regions)
-{
-    var result = new Dictionary<string, double>();
-
-    if (spreadData == null || spreadData.Count < 2 || regions == null || regions.Count == 0)
-        return result;
-
-    var mids = spreadData.Select(x => (x.Item1 + x.Item3) / 2.0).ToList();
-    double mean = mids.Average();
-    double stddev = Math.Sqrt(mids.Select(x => Math.Pow(x - mean, 2)).Average());
-
-    if (stddev == 0)
-    {
-        result["Zero Volatility"] = 100.0;
-        return result;
-    }
-
-    var buckets = new int[regions.Count + 1];
-    foreach (var mid in mids)
-    {
-        double distance = Math.Abs(mid - mean);
-        bool placed = false;
-
-        for (int i = 0; i < regions.Count; i++)
-        {
-            if (distance <= regions[i])
-            {
-                buckets[i]++;
-                placed = true;
-                break;
-            }
-        }
-
-        if (!placed)
-            buckets[regions.Count]++;
-    }
-
-    int total = mids.Count;
-    for (int i = 0; i < regions.Count; i++)
-    {
-        string label = i == 0
-            ? $"≤ {regions[i]:0.##}σ"
-            : $"{regions[i - 1]:0.##}σ – {regions[i]:0.##}σ";
-        result[label] = 100.0 * buckets[i] / total;
-    }
-
-    result[$"> {regions.Last():0.##}σ"] = 100.0 * buckets.Last() / total;
-
-    return result;
-}
-
-/// <summary>
-/// Computes two drift metrics indicating how much the spread's mean changes over time,
-/// based on the standard deviation of rolling mean values.
-///
-/// ✅ Absolute Drift: Standard deviation of the rolling mean
-/// ✅ Relative Drift: Drift normalized by the overall mean (unitless ratio)
-///
-/// This shows whether the center of the spread is stable — critical for mean-reversion.
-///
-/// How to Interpret the Output:
-/// - ✅ Relative Drift < 1% → Very stable mean — ideal
-/// - ⚠️ 1–5% → Mild drift — consider shorter holds or adaptive mean
-/// - ❌ >5% → Drifting mean — fixed mean may no longer work
-/// </summary>
-/// <param name="spreadData">List of (bid, bidQty, ask, askQty)</param>
-/// <param name="window">Window size for rolling mean (e.g., 20 ticks)</param>
-/// <returns>
-/// Tuple of (Absolute Drift, Relative Drift) — where:
-/// - Absolute Drift = StdDev of rolling means
-/// - Relative Drift = Drift / global mean
-/// </returns>
-public static (double AbsoluteDrift, double RelativeDrift) EvaluateMeanDriftScore(
-    List<Tuple<double, double, double, double>> spreadData,
-    int window = 20)
-{
-    if (spreadData == null || spreadData.Count < window)
-        return (double.NaN, double.NaN);
-
-    var mids = spreadData.Select(x => (x.Item1 + x.Item3) / 2.0).ToArray();
-    var rollingMeans = new List<double>();
-
-    for (int i = window; i <= mids.Length; i++)
-    {
-        double rollingMean = mids.Skip(i - window).Take(window).Average();
-        rollingMeans.Add(rollingMean);
-    }
-
-
-    double overallMean = mids.Average(); // use full mean for normalization
-    //double drift = Math.Sqrt(rollingMeans.Select(m => Math.Pow(m - rollingMeans.Average(), 2)).Average());
-    double drift = Math.Sqrt(rollingMeans.Select(m => Math.Pow(m - rollingMeans.Average(), 2)).Average());
-
-
-double relativeDrift = Math.Abs(overallMean) > 1e-8 ? drift / Math.Abs(overallMean) : double.NaN;
-    return (drift, relativeDrift);
-}
-
-/// <summary>
-/// Returns the latest Kalman Filter estimate of the true mean of the spread mid-prices.
-/// This tracks the spread's central value while allowing for gradual drift and suppressing noise.
-///
-/// Internally, this applies a recursive 1D Kalman filter over the input mid-prices.
-/// 
-/// Tuning Parameters:
-/// - Q: Process noise variance (higher = faster adaptation to drift)
-/// - R: Measurement noise variance (higher = more smoothing)
-/// 
-/// Use this instead of SMA if your mean drifts or if price series is noisy.
-///
-/// </summary>
-/// <param name="spreadData">List of (bid, bidQty, ask, askQty)</param>
-/// <param name="Q">Process noise variance (default = 0.0001)</param>
-/// <param name="R">Measurement noise variance (default = 0.01)</param>
-/// <returns>Latest estimated Kalman-filtered mean, or NaN if input is invalid</returns>
-public static double GetLatestKalmanMean(
-    List<Tuple<double, double, double, double>> spreadData,
-    double Q = 0.0001,
-    double R = 0.01)
-{
-    if (spreadData == null || spreadData.Count == 0)
-        return double.NaN;
-
-    var mids = spreadData.Select(x => (x.Item1 + x.Item3) / 2.0).ToList();
-
-    double x = mids[0];  // initial estimate of mean
-    double P = 1.0;       // initial uncertainty
-
-    for (int i = 1; i < mids.Count; i++)
-    {
-        // Predict step
-        double x_prior = x;
-        double P_prior = P + Q;
-
-        // Update step
-        double K = P_prior / (P_prior + R);
-        x = x_prior + K * (mids[i] - x_prior);
-        P = (1 - K) * P_prior;
-    }
-
-    return x; // the latest estimated mean
-}
-
-
-
-
-}
@@ -1,137 +0,0 @@
-namespace bts.SilverPine {
-   using BCG;
-
-   using System.Diagnostics;
-
-   public class SpreadPerformanceStream : ViewStream {
-      ViewStream _spread_price_stream = null;
-      private readonly IBlueTableSchema _schema;
-      private readonly int _window_secs;
-      private readonly double _buy_threshold, _sell_threshold, _flat_threshold;
-      private ViewStream _bbo_stream;
-      public SpreadPerformanceStream(string s1, string s2, double w1, double w2, int window_secs,
-                                       double buy_threshold, double sell_threshold, double flat_threshold)
-      : this(s1, s2, w1, w2, window_secs, buy_threshold, sell_threshold, flat_threshold, ViewStreamDatabase.instance) { }
-
-      public SpreadPerformanceStream(string s1, string s2, double w1, double w2, int window_secs,
-                                      double buy_threshold, double sell_threshold, double flat_threshold,
-                                      ViewStreamDatabase vsd) {
-         _bbo_stream = new BBOStream(vsd);
-         _schema = BlueDataUtility.form_schema("SpreadPerformance", Tuple.Create("id", TypeCode.Int32),                                                
-                                                Tuple.Create("t0", TypeCode.DateTime),
-                                                Tuple.Create("t1", TypeCode.DateTime),
-                                                Tuple.Create("n_ticks", TypeCode.UInt32),
-                                                Tuple.Create("curr_windows_seconds", TypeCode.Double),
-                                                // means / std dev                                          
-                                                Tuple.Create("stddev", TypeCode.Double),
-                                                Tuple.Create("mean", TypeCode.Double),
-                                                Tuple.Create("kalman_filter_mean", TypeCode.Double),
-                                                Tuple.Create("absolute_mean_drift", TypeCode.Double),
-                                                Tuple.Create("relative_mean_drift", TypeCode.Double),
-                                                // statistical measures
-                                                Tuple.Create("hurst_exponent", TypeCode.Double),
-                                                Tuple.Create("half_life", TypeCode.Double),
-                                                Tuple.Create("lag1_autocorrelation", TypeCode.Double),
-                                                Tuple.Create("average_zscore_decay", TypeCode.Double),
-                                                Tuple.Create("variance_ratio", TypeCode.Double),
-                                                Tuple.Create("n_zero_crossings", TypeCode.UInt32),                                                
-                                                // distributions
-                                                Tuple.Create("Z ≤ 0.5σ", TypeCode.Double),
-                                                Tuple.Create("0.5σ – 1.0σ", TypeCode.Double),
-                                                Tuple.Create("1.0σ – 2.0σ", TypeCode.Double),
-                                                Tuple.Create("> 2.0σ", TypeCode.Double),
-                                                ////////////////////////////////////////                                                   
-                                                Tuple.Create("max_bid", TypeCode.Double),
-                                                Tuple.Create("min_ask", TypeCode.Double),
-                                                Tuple.Create("n_buy_signals", TypeCode.UInt32),
-                                                Tuple.Create("cumm_buy_price", TypeCode.Double),
-                                                Tuple.Create("n_sell_signals", TypeCode.UInt32),
-                                                Tuple.Create("cumm_sell_price", TypeCode.Double)
-                                                //Tuple.Create("pnl", TypeCode.Double),
-                                                //Tuple.Create("buy_turns", TypeCode.UInt32),
-                                                //Tuple.Create("sell_turns", TypeCode.UInt32),
-                                                //Tuple.Create("trades", TypeCode.String)
-                                                );
-         _window_secs = window_secs;
-         _buy_threshold = buy_threshold;
-         _sell_threshold = sell_threshold;
-         _flat_threshold = flat_threshold;
-         _spread_price_stream = new SyntheticSpreadStream(s1, s2, w1, w2, _bbo_stream, vsd);
-         _spread_price_stream = new TimeSeriesStream(_spread_price_stream);
-         _spread_price_stream = new ThrottleStream(_spread_price_stream, TimeSpan.FromSeconds(5.0), vsd);
-      }
-
-      protected override void myOnFirstSubscriber() {
-         _spread_price_stream.onChange += _handle_spread_prices;
-      }
-
-      protected override void myOnLastSubscriber() {
-         _spread_price_stream.onChange -= _handle_spread_prices;
-      }
-
-      private void _handle_spread_prices(object dp_obj) {
-         DataPacket dp = dp_obj as DataPacket;
-         List<Tuple<DateTime, double, double, double, double>> spread_ts_prices = new List<Tuple<DateTime, double, double, double, double>>();
-         DateTime min_time = DateTime.MaxValue; DateTime max_time = DateTime.MinValue;
-         var sw = Stopwatch.StartNew();
-         sw.Start();
-         foreach (IBlueRow r in dp.newRows) {
-            DateTime ts = r.get_datetime("timestamp");
-            TimeSpan elapsed_time = DateTime.Now - ts;
-            if (elapsed_time.TotalSeconds >= _window_secs)
-               continue;
-            spread_ts_prices.Add(Tuple.Create(
-                                          r.get_datetime("timestamp"),
-                                          r.get<double>("spread_bid_price"),
-                                          r.get<double>("spread_bid_sz"),
-                                          r.get<double>("spread_ask_price"),
-                                          r.get<double>("spread_ask_sz")));
-            if (ts <= min_time) {
-               min_time = ts;
-            }
-            if (ts >= max_time) {
-               max_time = ts;
-            }
-         }
-         List<Tuple<double, double, double, double>> spread_prices = spread_ts_prices
-                                                                                             .OrderBy(t => t.Item1)
-                                                                                             .Select(t => Tuple.Create(t.Item2, t.Item3, t.Item4, t.Item5))
-                                                                                             .ToList();
-
-         uint n_ticks = (uint)spread_prices.Count;
-         double curr_windows_seconds = (max_time - min_time).TotalSeconds;
-         double hurst_exponent = SpreadPerformanceEvaluator.EvaluateHurstExponent(spread_prices);
-         double half_life = SpreadPerformanceEvaluator.ComputeHalfLife (spread_prices);
-         double lag1_autocorrelation = SpreadPerformanceEvaluator.ComputeLag1Autocorrelation(spread_prices);
-         double average_zscore_decay = SpreadPerformanceEvaluator.ComputeAverageZScoreDecay(spread_prices);
-         double variance_ratio = SpreadPerformanceEvaluator.ComputeVarianceRatio(spread_prices, 60, 360);
-
-         (double mean, double stddev) = SpreadPerformanceEvaluator.EvaluateMeanAndStdDev(spread_prices);
-         double kalman_filter_mean = SpreadPerformanceEvaluator.GetLatestKalmanMean(spread_prices);
-         uint n_zero_crossings = SpreadPerformanceEvaluator.EvaluateZeroCrossingCount(spread_prices);
-         (double max_bid, double min_ask) = SpreadPerformanceEvaluator.EvaluateMaxBidMinAsk(spread_prices);
-         (uint n_buy_signals, double cumm_buy_price, uint n_sell_signals, double cumm_sell_price) = SpreadPerformanceEvaluator.EvaluateExcursionCount(spread_prices, _buy_threshold);
-         //(double pnl, List<SpreadPerformanceEvaluator.TradeAction> trades, int buy_turns, int sell_turns) = SpreadPerformanceEvaluator.EvaluateSimulatedPnL(spread_prices, 30, _buy_threshold, _flat_threshold);
-         //string trade_str = String.Empty;
-         //foreach (var tr in trades) {
-         //   trade_str += tr.ToString() + " , ";
-         //}
-         Dictionary<string, double> dist_map = SpreadPerformanceEvaluator.EvaluateDistributionAroundMean(spread_prices,
-                                                                       new List<double>(new double[] { 0.5, 1.0, 2.0 }));
-         var dist_vals = dist_map.Count < 4 ? new double[] { Double.NaN, Double.NaN, Double.NaN, Double.NaN } : dist_map.Values.ToArray();
-         (double absolute_drift, double relative_drift) = SpreadPerformanceEvaluator.EvaluateMeanDriftScore(spread_prices);
-         IBlueRow nr = new BlueRow(_schema, 0, min_time, max_time, n_ticks, curr_windows_seconds, stddev, mean, kalman_filter_mean, absolute_drift, relative_drift,
-            hurst_exponent, half_life, lag1_autocorrelation, average_zscore_decay, variance_ratio, n_zero_crossings,
-                                    dist_vals[0], dist_vals[1], dist_vals[2], dist_vals[3],
-                                    max_bid, min_ask, n_buy_signals, cumm_buy_price, n_sell_signals, cumm_sell_price);
-         BlueTable t = new BlueTable(_schema);
-         t.addRow(nr);
-         var dp_out = BlueDataUtility.build_dp_inc(t, myLastValue as DataPacket, true);
-         var perf_msecs = sw.ElapsedMilliseconds;
-         sw.Stop();
-         Log.Gen.info($"SpreadPerformanceStream._handle_spread_prices comp_time: {perf_msecs} msecs");
-         myFireEvent(dp_out);
-
-      }
-   }
-}
@@ -1,68 +0,0 @@
-namespace bts.SilverPine{
-   using BCG;
-   public class SpreadStatArbsWeightLinearRegressionStream : ViewStream {
-      private IBlueTableSchema _schema;
-      private ViewStream _leg_price_stream;
-      private readonly double _window_secs;
-      private readonly string _s1, _s2;
-
-      public SpreadStatArbsWeightLinearRegressionStream(string s1, string s2, double window_secs, double sample_period_secs)
-         : this(s1, s2, window_secs, sample_period_secs, ViewStreamDatabase.instance) { }
-
-
-      public SpreadStatArbsWeightLinearRegressionStream(string s1, string s2, double window_secs, double sample_period_secs, ViewStreamDatabase vsd) {
-         _window_secs = window_secs;
-         _s1 = s1;
-         _s2 = s2;
-         _schema = BlueDataUtility.form_schema("LinearRegressionWeights",
-                                                Tuple.Create("id", TypeCode.Int32),
-                                                Tuple.Create("t0", TypeCode.DateTime),
-                                                Tuple.Create("t1", TypeCode.DateTime),
-                                                Tuple.Create("curr_windows_seconds", TypeCode.Double),
-                                                Tuple.Create("n_ticks", TypeCode.UInt32),
-                                                Tuple.Create("s1", TypeCode.String),
-                                                Tuple.Create("w1", TypeCode.Double),
-                                                Tuple.Create("s2", TypeCode.String),
-                                                Tuple.Create("w2", TypeCode.String));
-
-         _leg_price_stream = new TimeSeriesStream(new SyntheticSpreadLegsStream(s1, s2));
-         _leg_price_stream = new ThrottleStream(_leg_price_stream, TimeSpan.FromSeconds(1), vsd);
-      }
-
-      protected override void myOnFirstSubscriber() {
-         _leg_price_stream.onChange += _handle_leg_prices;
-      }
-
-      protected override void myOnLastSubscriber() {
-         _leg_price_stream.onChange -= _handle_leg_prices;
-      }
-
-      private void _handle_leg_prices(object dp_obj){
-         DataPacket dp = dp_obj as DataPacket;
-         List<Tuple<double, double, DateTime>> leg_prices = new List<Tuple<double, double, DateTime>>();
-         foreach (IBlueRow r in dp.newRows){
-            double s1_mid = (r.get<double>("s1_bid_price") + r.get<double>("s1_ask_price")) /2.0;
-            double s2_mid = (r.get<double>("s2_bid_price") + r.get<double>("s2_ask_price")) /2.0;
-            DateTime ts = r.get_datetime("timestamp");
-            leg_prices.Add(new Tuple<double, double, DateTime>(s1_mid, s2_mid, ts));
-         }
-         if (leg_prices.Count <= 2)
-            return;
-         leg_prices = leg_prices.Where(t => (DateTime.Now - t.Item3).TotalSeconds <= _window_secs).ToList();
-         DateTime t0 = leg_prices.Select(t => t.Item3).Min();
-         DateTime t1 = leg_prices.Select(t => t.Item3).Max();
-         double curr_windows_seconds = (t1 - t0).TotalSeconds;
-         int n_ticks = leg_prices.Count;
-         List<double> s1_mids = leg_prices.Select(t => t.Item1).ToList();
-         List<double> s2_mids = leg_prices.Select(t => t.Item2).ToList();
-         (double w1, double w2) = SpreadWeightCalculator.CalculateWeightsLinearRegression(s1_mids, s2_mids);
-         BlueTable state = new BlueTable(_schema);
-         IBlueRow nr = new BlueRow(_schema, 0, t0, t1, curr_windows_seconds, n_ticks,_s1, w1, _s2, w2);
-         state.addRow(nr);
-         var dp_out = BlueDataUtility.build_dp_inc(state, myLastValue as DataPacket, true);
-         myFireEvent(dp_out);
-
-      }
-   }
-}
-
@@ -1,143 +0,0 @@
-namespace bts.SilverPine {
-   using BCG;
-
-   using MathNet.Numerics.LinearAlgebra.Factorization;
-
-   using System;
-   using System.Collections.Generic;
-   using System.Diagnostics;
-   using System.Linq;
-   using System.Text;
-   using System.Threading.Tasks;
-   [DataStreamUserRespresentation("SpreadStatsStream")]
-   // This is the fundamental stream that computes the stats (including buy and sell zones) 
-   // of a spread. the legs and the weights are passed as args
-   public class SpreadStatsStream : ViewStream {
-      private ViewStream _spread_price_stream;
-      private readonly IBlueTableSchema _schema;
-      private readonly TimeSpan _window_sz;
-      private readonly string _spread;
-      public SpreadStatsStream(string s1, string s2, double w1, double w2, int rolling_window_sz_seconds, int window_sampling_sz_secs)
-         : this(s1, s2, w1, w2, rolling_window_sz_seconds, window_sampling_sz_secs, new BBOStream(ViewStreamDatabase.instance), ViewStreamDatabase.instance) { }
-      public SpreadStatsStream(string s1, string s2, double w1, double w2, int rolling_window_sz_seconds, int window_sampling_sz_secs, ViewStream bbo_stream, ViewStreamDatabase vsd) {
-         _window_sz = TimeSpan.FromSeconds(rolling_window_sz_seconds);
-         _spread = $"{w1} X {s1} - {w2} X {s2}";
-         _schema = new BlueTableSchema("SpreaddStats", new ColumnInfo ("id", 0, TypeCode.Int32));
-         _schema = _schema.add_column(TypeCode.String, "spread");
-         _schema = _schema.add_column(TypeCode.String, "window");
-         // basic stats
-         _schema = _schema.add_column(TypeCode.Double, "bid_mean");
-         _schema = _schema.add_column(TypeCode.Double, "ask_mean");
-         _schema = _schema.add_column(TypeCode.Double, "bid_stddev");
-         _schema = _schema.add_column(TypeCode.Double, "ask_stddev");
-         _schema = _schema.add_column(TypeCode.Double, "bid_skewness");
-         _schema = _schema.add_column(TypeCode.Double, "ask_skewness");
-         _schema = _schema.add_column(TypeCode.Double, "bid_kurtosis");
-         _schema = _schema.add_column(TypeCode.Double, "ask_kurtosis");
-         _schema = _schema.add_column(TypeCode.Double, "bid_max");
-         _schema = _schema.add_column(TypeCode.Double, "ask_max");
-         _schema = _schema.add_column(TypeCode.Double, "bid_min");
-         _schema = _schema.add_column(TypeCode.Double, "ask_min");
-         // bid bins
-         _schema = _schema.add_column(TypeCode.Double, "bid Z < -2σ");
-         _schema = _schema.add_column(TypeCode.Double, "bid -2σ ≤ Z < -1σ");
-         _schema = _schema.add_column(TypeCode.Double, "bid -1σ ≤ Z < 0σ");
-         // positive z's
-         _schema = _schema.add_column(TypeCode.Double, "bid 0σ ≤ Z < 1σ");
-         _schema = _schema.add_column(TypeCode.Double, "bid 1σ ≤ Z < 2σ");
-         _schema = _schema.add_column(TypeCode.Double, "bid Z ≥ 2σ");
-         // ask bins
-         _schema = _schema.add_column(TypeCode.Double, "ask Z < -2σ");
-         _schema = _schema.add_column(TypeCode.Double, "ask -2σ ≤ Z < -1σ");
-         _schema = _schema.add_column(TypeCode.Double, "ask -1σ ≤ Z < 0σ");
-         // positive z's
-         _schema = _schema.add_column(TypeCode.Double, "ask 0σ ≤ Z < 1σ");
-         _schema = _schema.add_column(TypeCode.Double, "ask 1σ ≤ Z < 2σ");
-         _schema = _schema.add_column(TypeCode.Double, "ask Z ≥ 2σ");
-         // opps
-         _schema = _schema.add_column(TypeCode.Int32, "n_buy_ticks");
-         _schema = _schema.add_column(TypeCode.Double, "cumm_buy_price");
-         _schema = _schema.add_column(TypeCode.Int32, "n_sell_ticks");
-         _schema = _schema.add_column(TypeCode.Double, "cumm_sell_price");
-         _schema = _schema.add_column(TypeCode.Int32, "n_trade_ticks");
-         _spread_price_stream = new SyntheticSpreadStream(s1, s2, w1, w2, bbo_stream, vsd);
-         _spread_price_stream = new TimeSeriesStream(_spread_price_stream);
-         double window_sampling_sz_smsecs = window_sampling_sz_secs * 1000.0 + new Random(DateTime.Now.Second).Next(-100, 100);
-         TimeSpan sampling_period = TimeSpan.FromMilliseconds(window_sampling_sz_smsecs);
-         _spread_price_stream = new ThrottleStream(_spread_price_stream, sampling_period, vsd);
-      }
-
-      protected override void myOnFirstSubscriber() {
-         _spread_price_stream.onChange += _handle_spread_prices;
-      }
-
-      protected override void myOnLastSubscriber() {
-         _spread_price_stream.onChange -= _handle_spread_prices;
-      }
-
-      private void _handle_spread_prices(object dp_obj) {
-         List<IBlueRow> rows = (dp_obj as DataPacket).newRows.to_list();
-         Func<IBlueRow, bool> row_filter = (r) => {
-            DateTime ts = r.get_datetime("timestamp");
-            DateTime start_time = DateTime.Now.Subtract(_window_sz);
-            return ts >= start_time;
-         };
-         var rolling_win_rows = rows.Where(r => row_filter(r));
-         var spread_prices = rolling_win_rows.Select(r => (
-                                          r.get_datetime("timestamp"),
-                                          r.get<double>("spread_bid_price"),
-                                          r.get<double>("spread_bid_sz"),
-                                          r.get<double>("spread_ask_price"),
-                                          r.get<double>("spread_ask_sz"))).ToList();
-         spread_prices.Sort((a,b) => a.Item1.CompareTo(b.Item1));
-         DateTime t0 = spread_prices.First().Item1, t1 = spread_prices.Last().Item1;
-         var tot_secs = (t1 - t0).TotalSeconds;
-         var stopwatch = Stopwatch.StartNew();
-         Log.Gen.info($"SpreadStatsStream.analyze_spread_quotes executed in {stopwatch.ElapsedMilliseconds} ms");
-         var result = SpreadPerformanceEvaluator.SpreadStatistics.analyze_spread_quotes(spread_prices);
-         (uint n_buy_signals, double cumm_buy_price, uint n_sell_signals, double cumm_sell_price) = 
-              SpreadPerformanceEvaluator.EvaluateExcursionCount(spread_prices.Select(t => Tuple.Create(t.Item2, t.Item3, t.Item4, t.Item5)).ToList(), 2.0);
-         IBlueRow nr = new BlueRow(_schema,
-            0,
-            _spread,
-            $"n_ticks: {spread_prices.Count} t0: {t0.ToLongTimeString()} -> t1: {t1.ToLongTimeString()} ({tot_secs}) secs",
-            result.bid_stats.Mean,
-            result.ask_stats.Mean,
-            result.bid_stats.StandardDeviation,
-            result.ask_stats.StandardDeviation,
-            result.bid_stats.Skewness,
-            result.ask_stats.Skewness,
-            result.bid_stats.Kurtosis,
-            result.ask_stats.Kurtosis,
-            result.bid_stats.Maximum,
-            result.ask_stats.Maximum,
-            result.bid_stats.Minimum,
-            result.ask_stats.Minimum,
-            // bid distribution
-            result.bid_zscore_distribution["Z < -2σ"],
-            result.bid_zscore_distribution["-2σ ≤ Z < -1σ"],
-            result.bid_zscore_distribution["-1σ ≤ Z < 0σ"],
-            result.bid_zscore_distribution["0σ ≤ Z < 1σ"],
-            result.bid_zscore_distribution["1σ ≤ Z < 2σ"],
-            result.bid_zscore_distribution["Z ≥ 2σ"],
-            // ask distributions
-            result.ask_zscore_distribution["Z < -2σ"],
-            result.ask_zscore_distribution["-2σ ≤ Z < -1σ"],
-            result.ask_zscore_distribution["-1σ ≤ Z < 0σ"],
-            result.ask_zscore_distribution["0σ ≤ Z < 1σ"],
-            result.ask_zscore_distribution["1σ ≤ Z < 2σ"],
-            result.ask_zscore_distribution["Z ≥ 2σ"],
-            n_buy_signals,
-            cumm_buy_price,
-            n_sell_signals,
-            cumm_sell_price,
-            n_buy_signals + n_sell_signals
-            );
-         stopwatch.Stop();
-         BlueTable t = new BlueTable(_schema);
-         t.addRow(nr);
-         var dp_out = BlueDataUtility.build_dp(t, myLastValue as DataPacket);
-         myFireEvent(dp_out);
-      }
-   }
-}
@@ -1,49 +0,0 @@
-using System;
-using System.Collections.Generic;
-using MathNet.Numerics.LinearRegression;
-
-namespace bts.SilverPine {
-
-   public static class SpreadWeightCalculator {
-      /// <summary>
-      /// Estimates hedge weights (w1, w2) such that the spread s = w1 * s1 - w2 * s2 
-      /// is potentially mean-reverting. Uses linear regression: s1 = beta * s2 + ε.
-      /// 
-      /// Assumes mid-prices are precomputed and aligned over the same time window.
-      /// </summary>
-      /// <param name="s1MidPrices">List of mid-prices for asset 1 (dependent variable)</param>
-      /// <param name="s2MidPrices">List of mid-prices for asset 2 (independent variable)</param>
-      /// <param name="includeIntercept">
-      /// Whether to include an intercept in the regression model (default: true).
-      /// </param>
-      /// <returns>(w1, w2): Tuple of hedge weights</returns>
-      public static (double w1, double w2) CalculateWeightsLinearRegression(
-          List<double> s1MidPrices,
-          List<double> s2MidPrices,
-          bool includeIntercept = true) {
-         if (s1MidPrices == null || s2MidPrices == null)
-            throw new ArgumentNullException("Price series cannot be null.");
-         if (s1MidPrices.Count != s2MidPrices.Count)
-            throw new ArgumentException("Price series must be of equal length.");
-         if (s1MidPrices.Count < 2)
-            throw new ArgumentException("Need at least two data points for regression.");
-
-         double[] x = s2MidPrices.ToArray(); // Independent variable
-         double[] y = s1MidPrices.ToArray(); // Dependent variable
-
-         double beta;
-
-         if (includeIntercept) {
-            // Returns (intercept, slope)
-            (double intercept, double slope) = SimpleRegression.Fit(x, y);
-            beta = slope;
-         } else {
-            // Returns only slope when forcing regression through origin
-            beta = SimpleRegression.FitThroughOrigin(x, y);
-         }
-
-         // Return fixed w1 = 1.0 and w2 = beta
-         return (1.0, beta);
-      }
-   }
-}
@@ -1,118 +0,0 @@
-// TODO: Remove
-//// namespace bts.SilverPine 
-////    using bts;
-////    using BCG;
-////    using MongoDB.Driver.Linq;
-////    using System.ComponentModel.DataAnnotations;
-////    using ZstdSharp.Unsafe;
-
-////    [DataStreamUserRespresentation("VWAP")]
-////    public class VWAPStream : ViewStream {
-////       // returns the VWAP stream for each contract 
-////       private static ViewStream build_vwap_stream(string symbol, DateTime begin_dt, DateTime end_dt) {
-////          var ohlcv_params = OHLCVStream.get_params(begin_dt, end_dt);
-////          ViewStream ohlc_str = new OHLCVStream(symbol, ohlcv_params);
-////          // stream is grouped per contract and then vwap is calculated per contract
-////          GlobalMapStream rs = new GlobalMapStream(ohlc_str, "ts_event", "contract",
-////          (rows, idx) => {
-////             Log.Gen.info($"VWAPStream.GlobalMapStream delegate n_rows: {rows.Count} idx: {idx}");
-////             IBlueRow ir = rows.ElementAt((int)idx);
-////             var sel_rows = rows.Skip((int)idx).Where(r => r.get_datetime("ts_event") >= end_dt && r.get_datetime("ts_event") <= begin_dt);
-////             var volumeprices = sel_rows.Select(r => Tuple.Create(r.get<int>("volume"), r.get<double>("close"))).ToList();
-////             double vwap = IndicatorCalc.calculate_VWAP(volumeprices);
-////             IBlueTableSchema schema = ir.schema.add_columns(Tuple.Create(typeof(double), "vwap"));
-////             MutableBlueRow nr = new MutableBlueRow(schema, ir);
-////             //vwap = -313.987;
-////             nr.setValue("vwap", vwap);
-////             return nr;
-////          });
-////          return rs;
-////       }
-////       // the args want dt1 > dt2, if its nto we, swap them
-////       static void swap_if_necessary(ref DateTime dt1, ref DateTime dt2) {
-////          if (dt1 < dt2) {
-////             DateTime aux = dt1;
-////             dt1 = dt2;
-////             dt2 = aux;
-////          }
-////       }
-
-////       // this assumes the ohlc_str has diff contracts
-////       public VWAPStream(string symbol_or_contract, DateTime begin_dt, DateTime end_dt) {
-////          swap_if_necessary(ref begin_dt, ref end_dt);
-////          Tuple<ContractUtils.ContractCategory, string> ccat_sym_contr = ContractUtils.get_symbol_category(symbol_or_contract);
-////          string symbol = String.Empty;
-////          symbol = ccat_sym_contr.Item2;
-////          var ohlcv_params = OHLCVStream.get_params(begin_dt, end_dt);
-////          ViewStream vwap_str = build_vwap_stream(symbol, begin_dt, end_dt);
-////          // vwap_str now has vwap for all contracts. if the arg was a symbol, we need to filter it for the active contract,
-////          // whihc we do with FutureActivesContractStream. If the arg was a contract we'll simply filter it
-////          ViewStream rs = vwap_str;
-////          switch (ccat_sym_contr.Item1) {
-////             case ContractUtils.ContractCategory.FutureString:
-////             case ContractUtils.ContractCategory.SpreadString:
-////                rs = new FilterStream(vwap_str, r => r.get<string>("contract").CompareTo(ccat_sym_contr.Item2) == 0);
-////                break;
-////             case ContractUtils.ContractCategory.Symbol:
-////                rs = new FuturesActiveContractStream(vwap_str);
-////                break;
-////          }
-////          rs = new FilterStream(rs, r => r.get_datetime("ts_event").CompareTo(begin_dt) == 0);
-////          rs = new NewProjectStream(rs, "! vwap");
-////          mySetRealSource(rs);
-////       }
-
-////       public VWAPStream(string symbol_or_contract, string begin_dt, string end_dt)
-////       :
-////       this(symbol_or_contract, DateTime.Parse(begin_dt), DateTime.Parse(end_dt)) { }
-////    }
-//// }
-
-//using System;
-
-//using BCG;
-
-//using bts.api;
-
-//namespace bts.SilverPine {
-//   /*
-//      ----------------------------------------------------------------------
-//                                  *Expression*
-//      ----------------------------------------------------------------------
-//          //nl("Find Settlement price of the SP500 future in the first week of 2023.")
-
-//          r=FutureHistoricalBars("ES", "One_Day", "1/1/2024", "2/1/2024")
-//          r=math(r, "vprice = volume * close")
-//          r=aggregate(r, "vprice:Sum, volume:Sum")
-//          r=math(r, "vwap = vprice_sum / volume_sum")
-//          r=newProject(r, "vwap !")
-
-//      ----------------------------------------------------------------------
-//                                    *Schema*
-//      ----------------------------------------------------------------------
-//          id                              System.Int32
-//          vwap                            System.Double
-//   */
-
-//   [DataStreamUserRespresentation("VWAP")]
-//   public class VWAPStream : ViewStream {
-//      public VWAPStream(string symbol_or_contract, DateTime beg_dt, DateTime end_dt)
-//      :
-//      this(symbol_or_contract, beg_dt, end_dt, ViewStreamDatabase.instance) { }
-
-//      public VWAPStream(string symbol_or_contract, DateTime beg_dt, DateTime end_dt, ViewStreamDatabase db) {
-//         //var fc = new ContractUtils.FutureContract(symbol_or_contract);
-//         ViewStream r = new FutureHistoricalBarsStream(symbol_or_contract, beg_dt.ToString(), end_dt.ToString());
-//         // since this is already filtered, we can groupBy iit without a groupBy column
-//         r = new MathStream(r,
-//                 "vprice = volume * close");
-//         r = new NonIncGroupByStream(r,
-//                                      "vprice:Sum, volume:Sum");
-//         r = new MathStream(r,
-//                             "vwap = vprice_sum / volume_sum");
-//         r = new NewProjectStream(r,
-//                                   "vwap !");
-//         mySetRealSource(r);
-//      }
-//   }
-//}
@@ -13,6 +13,7 @@ namespace bts.SilverPine {
   using Spark.Api.MarketData;

   using SecurityKey = System.UInt32;
+   using System.Data.Entity.Infrastructure;

   public class FakeBBOStream : ViewStream, IWorker {
      private string _symbol;
@@ -104,14 +105,14 @@ namespace bts.SilverPine {
   public class BBOStream : ViewStream, IWorker {
      private readonly TimeSpan _poll_period = TimeSpan.FromMilliseconds(500);
      private DateTime _last_poll_time = DateTime.MinValue;
-      private ViewStreamDatabase? _vsd;
-      IReadOnlyDictionary<SecurityKey, BookTuple>? _cached_state = null;
-      MarketDataClient? _md_client;
+      private ViewStreamDatabase _vsd;
+      private int _id = 0;
+      IReadOnlyDictionary<SecurityKey, BookTuple> _cached_state = null;
+      MarketDataClient _md_client;
      internal static IBlueTableSchema _bbo_schema = BlueDataUtility.form_schema("bbo",
                                                Tuple.Create("id", TypeCode.Int32),
                                                Tuple.Create("sec_key", TypeCode.UInt32),
                                                Tuple.Create("sec", TypeCode.String),
-                                                Tuple.Create("exchange", TypeCode.String),
                                                Tuple.Create("bid_price", TypeCode.Double),
                                                Tuple.Create("bid_sz", TypeCode.Single),
                                                Tuple.Create("ask_price", TypeCode.Double),
@@ -123,18 +124,17 @@ namespace bts.SilverPine {
      }

      public BBOStream(ViewStreamDatabase vsd) {
-         bts.Log.Gen.info("BBOStream.cons");
-         _vsd = vsd;
-         _md_client = new MarketDataClient(new MDSource(TimeSpan.FromSeconds(0.5)));
-         //FakeBBOStream rs = new FakeBBOStream("Future", "CME", "ES", 5000, 0.25, vsd);
-         //mySetRealSource(rs);
+         // bts.Log.Gen.info("BBOStream.cons");
+         // _vsd = vsd;
+         // _md_client = new MarketDataClient(new MDSource(TimeSpan.FromSeconds(0.5)));
+         FakeBBOStream rs = new FakeBBOStream("Future", "CME", "ES", 5000, 0.25, vsd);
+         mySetRealSource(rs);
      }

      protected override void myOnFirstSubscriber() {
         _vsd.addWorker(this);
         _md_client.books().Subscribe(_on_books);
-         _vsd?.addWorker(this);
-         _md_client?.books().Subscribe(_on_books);
+
      }

      private void _log_updates(TableUpdate<BookTuple, uint> update) {
@@ -144,24 +144,24 @@ namespace bts.SilverPine {
         }
         foreach (Change<BookTuple> c in update.changes) {
            var bt = c.New;
-            //bts.Log.Gen.info($"BBOStream changes: idx: {i} bid: {bt.book.bids[0].price} / {bt.book.bids[0].size} ask: {bt.book.asks[0].price} / {bt.book.asks[0].size}");
+            bts.Log.Gen.info($"BBOStream changes: idx: {i} bid: {bt.book.bids[0].price} / {bt.book.bids[0].size} ask: {bt.book.asks[0].price} / {bt.book.asks[0].size}");
         }
      }

      private void _on_books(TableUpdate<BookTuple, uint> update) {
         _log_updates(update);
         // just cacching in here, will populate the rows in _poll_scan and fire the datapacket
-         //bts.Log.Gen.info($"BBOStream._on_books state: {update.state.Count()} changes: {update.changes.Count()} added: {update.added.Count()}");
+         bts.Log.Gen.info($"BBOStream._on_books state: {update.state.Count()} changes: {update.changes.Count()} added: {update.added.Count()}");
         _cached_state = update.state;
      }

      protected override void myOnLastSubscriber() {
-         _vsd?.removeWorker(this);
+         _vsd.removeWorker(this);
      }

      private void _scan_bbo() {
         // this shouldn be made incremental using the TableUpdate
-         //bts.Log.Gen.info("BBOStream._scan_bbo");
+         bts.Log.Gen.info("BBOStream._scan_bbo");
         if (_cached_state == null)
            return;
         BlueTable new_state = new BlueTable(_bbo_schema);
@@ -170,23 +170,22 @@ namespace bts.SilverPine {
            var sec = kv.Value.security;
            int id = Convert.ToInt32(skey);
            BookTuple bt = kv.Value;
-            var venue = sec.Exchange;
-            IBlueRow r = new BlueRow(_bbo_schema, id, skey, sec.ToString(), venue.ToString(), bt.book.bids[0].price, bt.book.bids[0].size,
+            IBlueRow r = new BlueRow(_bbo_schema, id, skey, sec.ToString(), bt.book.bids[0].price, bt.book.bids[0].size,
                                     bt.book.asks[0].price, bt.book.asks[0].size);
            new_state.addRow(r);
         }
-         var dp = accumulate(myLastValue as DataPacket, new_state);
+         DataPacket dp = null;
+         if (myLastValue == null) {
+            dp = new DataPacket(new_state, new_state, new BlueTable(_bbo_schema));
+         } else {
+            DataPacket old_dp = myLastValue as DataPacket;
+            IBlueRowCollection added_rows, del_rows;
+            BlueDataUtility.compute_differences_fast(new_state, old_dp.newRows, out del_rows, out added_rows);
+            dp = new DataPacket(new_state, added_rows, del_rows);
+         }
         if (dp.addedRows.Count == 0 && dp.deletedRows.Count == 0)
            return;
         myFireEvent(dp);
-
-         static DataPacket accumulate(DataPacket? old_dp, BlueTable new_state) {
-            if (old_dp is null) {
-               return new(new_state, new_state, new BlueTable(_bbo_schema));
-            }
-            BlueDataUtility.compute_differences_fast(new_state, old_dp.newRows, out var del_rows, out var added_rows);
-            return new(new_state, added_rows, del_rows);
-         }
      }

      public void doWork() {
@@ -196,4 +195,4 @@ namespace bts.SilverPine {
         }
      }
   }
-}
+}
@@ -1,5 +1,4 @@
-namespace bts.BlueOrb {
-   using bts.SilverPine;
+namespace bts.SilverPine {
   using BCG;

   using System.Runtime.CompilerServices;
@@ -46,7 +45,7 @@ namespace bts.BlueOrb {
   }

   [DataStreamUserRespresentation("FuturesHistorical1DayBars")]
-   internal class FuturesHistorical1DayBarsStream : ViewStream {
+   public class FuturesHistorical1DayBarsStream : ViewStream {

      // this gets the raw bars for a symbol (perhaps make it into a raw bars stream?)
      public FuturesHistorical1DayBarsStream(string symbol) :
@@ -104,7 +103,7 @@ namespace bts.BlueOrb {
   }

   [DataStreamUserRespresentation("FuturesHistorical1MinBars")]
-   internal class FuturesHistorical1MinuteBarsStream : ViewStream {
+   public class FuturesHistorical1MinuteBarsStream : ViewStream {
      // same as the above but for one min
      // this gets the raw bars for a symbol
      public FuturesHistorical1MinuteBarsStream(string symbol) :
@@ -188,6 +187,46 @@ namespace bts.BlueOrb {
      }
   }

+   [DataStreamUserRespresentation("FuturePriceDiff")]
+   public class FutureHistoricalBarsPriceDiffStream : ViewStream {
+
+      public FutureHistoricalBarsPriceDiffStream(string future_contract, string dt1, string dt2)
+      :
+      this(new ContractUtils.FutureContract(future_contract), DateTime.Parse(dt1), DateTime.Parse(dt2), "close", ViewStreamDatabase.instance) {
+      }
+
+
+      public FutureHistoricalBarsPriceDiffStream(string future_contract, DateTime dt1, DateTime dt2)
+      :
+      this(new ContractUtils.FutureContract(future_contract), dt1, dt2, "close", ViewStreamDatabase.instance) {
+      }
+
+      public FutureHistoricalBarsPriceDiffStream(string future_contract, DateTime dt1, DateTime dt2, string col)
+      :
+      this(new ContractUtils.FutureContract(future_contract), dt1, dt2, col, ViewStreamDatabase.instance) {
+      }
+
+
+      public FutureHistoricalBarsPriceDiffStream(ContractUtils.FutureContract fc, DateTime dt1, DateTime dt2, string col, ViewStreamDatabase vsd) {
+         BarSize bsize = dt2 - dt1 >= TimeSpan.FromDays(1.0) ? BarSize.ONE_DAY : BarSize.ONE_MIN;
+         ViewStream rs = new FutureHistoricalBarsStream(fc, BarSize.ONE_DAY, new DateTimeRange(dt1, dt2));
+
+         rs = new ColumnDifferenceStream(rs, r => {
+            DateTime r_dt = r.get_datetime("ts_event");
+            var rv = dt1.Equals(r_dt);
+            return rv;
+         },
+         //r => dt2.Equals(r.get_datetime("ts_event")), col, col);
+         r => {
+            DateTime r_dt = r.get_datetime("ts_event");
+            var rv = dt2.Equals(r_dt);
+            return rv;
+         }, col, col);
+         rs = new NewProjectStream(rs, "diff !");
+         mySetRealSource(rs);
+      }
+   }
+
   // takes the contract name and expands it into multuiple columns corresponding to the contracts.
   // the contract columns are a union, of all properties with some of them empty
   public class ContractExpanderStream : ViewStream {
@@ -208,29 +247,3 @@ namespace bts.BlueOrb {
   }

 }
-
-namespace bts.BlueOrb {
-   using BCG;
-
-   using bts.api;
-
-   using Newtonsoft.Json.Serialization;
-
-   [DataStreamUserRespresentation("FuturesContinousContract")]
-   public class FuturesContinousContractStream : ViewStream {
-      // the argument is symbol, like "ES" etc and not the contract itself
-      public FuturesContinousContractStream(string symbol) : this(symbol, ViewStreamDatabase.instance) { }
-      public FuturesContinousContractStream(string symbol, ViewStreamDatabase db) {
-         symbol = symbol.EndsWith("*") ? symbol : symbol + "*";
-         ViewStream r = new FuturesHistorical1DayBarsStream(symbol);
-         r = new ContractExpanderStream(r, "symbol");
-         r = new FilterStream(r, "contract_type='Future'");
-         // finding the max volume across symbols, if we need to do this for open_interest, then join here 
-         // with the open interest stream on symbol/contract
-         r = new NonIncGroupByStream(r, "ts_event", "volume:Max");
-         r = new NewProjectStream(r, "ts_event symbol !");
-         r = new SortStream(r, "-ts_event");
-         mySetRealSource(r);
-      }
-   }
-}
@@ -1,53 +1,46 @@
-//using BCG;
+using BCG;

-//namespace bts.SilverPine {
-//   /*
-//      ----------------------------------------------------------------------
-//                                  *Expression*
-//      ----------------------------------------------------------------------
-//          //r=FutureHistoricalBars("ES*", "02/01/2024 09:30 AM", "02/01/2024 09:30 PM")
-//          r=FutureHistoricalBars("ES*","One_Day", 02/01/2024, 02/01/2025)
-//          r=contractExpander(r,"symbol")
-//          r=filter(r,"contract_type='FutureSpread'")
-//          r=aggregate(r,"symbol","volume:Sum","ts_event")
-//          r=aggregate(r,"volume_sum:Max")
-//          r=newProject(r,"volume_sum_max:volume symbol !")
+namespace bts.SilverPine {
+   /*
+      ----------------------------------------------------------------------
+                                  *Expression*
+      ----------------------------------------------------------------------
+          //r=FutureHistoricalBars("ES*", "02/01/2024 09:30 AM", "02/01/2024 09:30 PM")
+          r=FutureHistoricalBars("ES*","One_Day", 02/01/2024, 02/01/2025)
+          r=contractExpander(r,"symbol")
+          r=filter(r,"contract_type='FutureSpread'")
+          r=aggregate(r,"symbol","volume:Sum","ts_event")
+          r=aggregate(r,"volume_sum:Max")
+          r=newProject(r,"volume_sum_max:volume symbol !")

-//      ----------------------------------------------------------------------
-//                                    *Schema*
-//      ----------------------------------------------------------------------
-//          symbol                          System.String
-//          id                              System.Int32
-//          volume                          System.Double
-//   */
+      ----------------------------------------------------------------------
+                                    *Schema*
+      ----------------------------------------------------------------------
+          symbol                          System.String
+          id                              System.Int32
+          volume                          System.Double
+   */

-//   [DataStreamUserRespresentation("VolumeAgg")]
-//   public class VolumeAggregationStream : ViewStream {
-//      public VolumeAggregationStream(string future_contract, string contract_type, DateTime range_beg, DateTime range_end)
-//      : this(future_contract, contract_type, range_beg, range_end, ViewStreamDatabase.instance) { }
+   [DataStreamUserRespresentation("VolumeAgg")]
+   public class VolumeAggregationStream : ViewStream {
+      public VolumeAggregationStream(string future_contract, string contract_type, DateTime range_beg, DateTime range_end)
+      : this(future_contract, contract_type, range_beg, range_end, ViewStreamDatabase.instance) { }

-//      public VolumeAggregationStream(string future_contract, string contract_type, string range_beg, string range_end)
-//      :
-//      this(future_contract, contract_type, DateTime.Parse(range_beg), DateTime.Parse(range_end), ViewStreamDatabase.instance) { }
+      public VolumeAggregationStream(string future_contract, string contract_type, string range_beg, string range_end)
+      :
+      this(future_contract, contract_type, DateTime.Parse(range_beg), DateTime.Parse(range_end), ViewStreamDatabase.instance) { }

-//      public VolumeAggregationStream(string future_contract, string contract_type, DateTime range_beg, DateTime range_end, ViewStreamDatabase db) {
-//         future_contract = future_contract.EndsWith("*") ? future_contract : future_contract + "*";
-//         ViewStream r;
-//         r = new FutureHistoricalBarsStream(future_contract,
-//                                             range_beg.ToString(), range_end.ToString());
-//         r = new ContractExpanderStream(r, "symbol");
-//         r = new FilterStream(r, $"contract_type='{contract_type}'");
-//         r = new NonIncGroupByStream(r, "symbol", "volume:Sum", "ts_event");
-//         r = new NonIncGroupByStream(r, "volume_sum:Max");
-//         r = new NewProjectStream(r, "volume_sum_max:volume symbol !");
-//         mySetRealSource(r);
-//      }
-//   }
-
-//   [DataStreamUserRespresentation("PeriodicVolumeAgg")]
-//   public class PeriodicVolumeAggregationStream : ViewStream {
-//      public PeriodicVolumeAggregationStream(string future_contract, string contract_type, DateTime range_beg, DateTime range_end,
-//                                             TimeSpan period, ViewStreamDatabase db) {
-//      }
-//   }
-//}
+      public VolumeAggregationStream(string future_contract, string contract_type, DateTime range_beg, DateTime range_end, ViewStreamDatabase db) {
+         future_contract = future_contract.EndsWith("*") ? future_contract : future_contract + "*";
+         ViewStream r;
+         r = new FutureHistoricalBarsStream(future_contract,
+                                             range_beg.ToString(), range_end.ToString());
+         r = new ContractExpanderStream(r, "symbol");
+         r = new FilterStream(r, $"contract_type='{contract_type}'");
+         r = new NonIncGroupByStream(r, "symbol", "volume:Sum", "ts_event");
+         r = new NonIncGroupByStream(r, "volume_sum:Max");
+         r = new NewProjectStream(r, "volume_sum_max:volume symbol !");
+         mySetRealSource(r);
+      }
+   }
+}
@@ -21,7 +21,7 @@ namespace BCG {
      private DateTime _last_poll_time = DateTime.MinValue;
      private ViewStreamDatabase _vsd;
      private int _id = 0;
-      IReadOnlyDictionary<SecurityKey, BookTuple>? _cached_state = null;
+      IReadOnlyDictionary<SecurityKey, BookTuple> _cached_state = null;
      MarketDataClient _md_client;
      readonly Stopwatch _timer = new();
      private readonly IBlueTableSchema _bbo_schema = BlueDataUtility.form_schema("books",
@@ -73,8 +73,14 @@ namespace BCG {

      private Dictionary<SecurityKey, Dictionary<int, int>> _id_map = new Dictionary<SecurityKey, Dictionary<int, int>>();
      private int _get_id(SecurityKey skey, int depth) {
-         var d_map = _id_map.GetOrAdd(skey, _ => new() { {depth, _id++} });
-         if (!d_map.TryGetValue(depth, out int rv)) {
+         Dictionary<int, int> d_map = null;
+         if (!_id_map.TryGetValue(skey, out d_map)) {
+            d_map = new Dictionary<int, int>();
+            d_map.Add(depth, _id++);
+            _id_map.Add(skey, d_map);
+         }
+         int rv = 0;
+         if (!d_map.TryGetValue(depth, out rv)) {
            rv = _id++;
            d_map.Add(depth, rv);
         }
Author	SHA1	Message	Date
Taha Afzal	66ad51280f	Reversehedging fixed	2025-04-08 22:13:33 +00:00
arham	029edd8a86	Format: ran clang format on file	2025-04-07 15:39:57 +00:00
arham	dfdcbab2a0	Add: config changes to make server controller work and set up hk zen server	2025-04-07 15:33:26 +00:00
Taha Afzal	6d7c0702f0	Fixing build:	2025-04-05 23:38:05 +00:00
Taha Afzal	86eaddea93	Fixing a bug, where the spreader was isidentifying the quoting leg	2025-04-05 15:38:30 +00:00