cleanup: Remove unnecessary py_module.cpp files

We used to have separate modules for md and execution. I forgot
to remove these after unifying them to a single module.

.config/dotnet-tools.json

@@ -9,7 +9,7 @@
       ]
     },
     "fable": {
-      "version": "4.0.6",
+      "version": "4.24.0",
       "commands": [
         "fable"
       ]
@@ -27,4 +27,4 @@
       ]
     }
   }
-}
+}

.devcontainer/Dockerfile

@@ -65,7 +65,7 @@ RUN \
    && apt-get install -y cmake
 
 # Dotnet
-RUN apt-get update && apt-get install -y dotnet7
+RUN apt-get update && apt-get install -y dotnet8
 
 # Node
 ARG node_major=18
@@ -90,6 +90,12 @@ RUN apt-get update && apt-get install -y libpcap0.8-dev
 # Databento (deps)
 RUN apt-get update && apt-get -y install libzstd-dev
 
+# Docker in Docker
+RUN apt-get update && apt-get install -y \
+    docker.io \
+    docker-compose \
+    docker-compose-v2
+
 # Set up default executables (update-alternatives)
 RUN \
    update-alternatives --install /usr/bin/clang clang /usr/bin/clang-$clang_major 100; \

.devcontainer/devcontainer.json

@@ -1,15 +1,10 @@
 {
   "dockerFile": "Dockerfile",
-  // Creates local dir to mount to HOME in docker 
-  "initializeCommand": "mkdir -p ${localEnv:HOME}/.bts/devcontainer-mounts/${devcontainerId}",
   "hostRequirements": {
-    "memory": "4gb",
-    "storage": "6gb"
+    "memory": "8gb",
+    "storage": "10gb"
   },
-  "postCreateCommand": "/home/bts/spark/docker/sparkd/dev/devcontainer_post_create.sh",
   "containerUser": "bts",
-  "workspaceMount": "source=${localWorkspaceFolder},target=/home/bts/spark,type=bind",
-  "workspaceFolder": "/home/bts/spark",
   "updateRemoteUserUID": true,
   "shutdownAction": "none",
   // Note: If running multiple containers on the same host, each must use a
@@ -17,34 +12,20 @@
   // example, if 8000-8003 is taken, go with 9000-9003, then 10000-10003, etc.
   "runArgs": [
     "--shm-size",
-    "2gb",
-    "-p",
-    "8000:8000",
-    "-p",
-    "8001:8001",
-    "-p",
-    "8002:8002",
-    "-p",
-    "8003:8003",
-    "-p",
-    "12345:12345"
+    "8gb"
   ],
   "mounts": [
-    "source=${localEnv:HOME}/.bts/devcontainer-mounts/${devcontainerId},target=/home/bts,type=bind"
+    "source=/var/run/docker.sock,target=/var/run/docker.sock,type=bind"
   ],
   "customizations": {
     "vscode": {
-      "settings": {
-        "clangd.arguments": [
-          "--compile-commands-dir=${workspaceFolder}/build/clang/debug"
-        ]
-      },
       "extensions": [
         "eamodio.gitlens",
         "llvm-vs-code-extensions.vscode-clangd",
         "ms-vscode.cmake-tools",
-        "ms-vscode.cpptools"
+        "ms-vscode.cpptools",
+        "xaver.clang-format"
       ]
     }
   }
-}
+}

.env

@@ -0,0 +1,49 @@
+# This file defines the global default configuration for docker deployments.
+# It is tracked in source control. For temporary modifications, please create
+# a separate file called ".env.local" (it's in .gitignore, so won't be tracked).
+# If a variable is defined in both files, the value in .env.local will be used.
+#
+# About the deployment procedure:
+#   We use Docker Compose to build images and launch containers for various services
+#   that make up our software suite:
+#     - sparkd
+#     - TODO: pubsub
+#     - TODO: md-publisher
+#     - TODO: webapp (DSQA on the web)
+#     - TODO: api-server (websocket based)
+#   From inside a devcontainer, you can launch all services in separate containers
+#   on your machine by running:
+#
+#     tools/deploy.sh
+#
+#   This script builds release versions for each service, packaged (generally) in
+#   self-contained directories. Next, it runs a few "docker compose" commands to
+#   build docker images loaded with the self-contained binaries from the previous
+#   step, and start containers with those images to run each service. To view the
+#   commands that deploy.sh runs internally (without actually running them), run:
+#
+#     tools/deploy.sh --dry-run
+#
+#   For additional information/help, run:
+#
+#     tools/deploy.sh --help
+#
+
+### Sparkd ###
+# The internal image for sparkd to use. For more info, view:
+#   src/server/sparkd/Dockerfile
+SPARKD__IMAGE=sparkd
+# The config dir for this service to use
+SPARKD__CONFIG_DIR=src/app_sparkd/configurations/dev
+# Mount a specific volume or dir to "~/sparkdata/data" inside the docker.
+# Defaults to a volume named "sparkd_<config_port>" (e.g., sparkd_8000).
+# SPARKD__DATA_DIR=/home/bts/sparkdata/data
+SPARKD__SOURCE_DIR=install/sparkd-release-gcc
+SPARKD__RUNTIME_CONF_DIR=/home/ubuntu/8000/sparkdata/conf/
+SPARKD__CONFIG_PORT=8000
+SPARKD__QUICKBUS_PORT=8001
+SPARKD__SIM_FIX_PORT=8002
+SPARKD__SHM_SIZE=4gb
+SPARKD__PYTHON_REQUIREMENTS=requirements.txt
+
+# TODO: add config for other services (pubsub, md-publisher, webapp, api-server, etc.)

.gitignore

@@ -59,3 +59,5 @@ trace_data/
 .devcontainer/
 *.Generated.cs
 *.lst
+.env.local
+.secrets

.gitmodules

@@ -1,6 +1,6 @@
 [submodule "externals"]
 	path = externals
-	url = git@bitbucket.org:bluetradesys/externals.git
+	url = https://bitbucket.org/bluetradesys/externals.git
 [submodule "vcpkg"]
 	path = vcpkg
 	url = https://github.com/microsoft/vcpkg.git

.vscode/launch.json

@@ -1,6 +1,14 @@
 {
   "version": "0.2.0",
   "configurations": [
+    {
+      "name": "Python Debugger: Current File with Arguments",
+      "type": "debugpy",
+      "request": "launch",
+      "program": "${file}",
+      "console": "integratedTerminal",
+      "args": "${command:pickArgs}"
+    },
     {
       "name": ".NET Core Attach",
       "type": "coreclr",
@@ -22,7 +30,7 @@
         "--config",
         "${workspaceFolder}/src/DataStreamSuite/DataStreamPublisher/Configurations/${input:dataStreamPublisherConfig}.json"
       ],
-      "cwd": "build/Debug/bin/net7.0",
+      "cwd": "build/Debug/bin/net8.0",
       "stopAtEntry": false,
       "console": "integratedTerminal"
     },
@@ -31,7 +39,39 @@
       "type": "cppdbg",
       "request": "launch",
       "program": "/home/bts/spark/build/sparkd-debug-clang/server/sparkd",
-      "args": ["run"],
+      "args": [
+        "run"
+      ],
+      "stopAtEntry": false,
+      "cwd": "/home/bts/spark/build/sparkd-debug-clang/server",
+      "environment": [],
+      "externalConsole": false,
+      "MIMode": "gdb",
+    },
+    {
+      "name": "spreader tests2",
+      "type": "cppdbg",
+      "request": "launch",
+      "program": "/home/bts/spark/build/sparkd-debug-clang/server/sparkd",
+      "args": [
+        "test",
+        "[spreader2]"
+      ],
+      "stopAtEntry": false,
+      "cwd": "/home/bts/spark/build/sparkd-debug-clang/server",
+      "environment": [],
+      "externalConsole": false,
+      "MIMode": "gdb",
+    },
+    {
+      "name": "spreader tests",
+      "type": "cppdbg",
+      "request": "launch",
+      "program": "/home/bts/spark/build/sparkd-debug-clang/server/sparkd",
+      "args": [
+        "test",
+        "[spreader]"
+      ],
       "stopAtEntry": false,
       "cwd": "/home/bts/spark/build/sparkd-debug-clang/server",
       "environment": [],
@@ -50,7 +90,7 @@
           "--config",
           "${workspaceFolder}/src/DataStreamSuite/DataStreamQueryAnalyzer/Configurations/DataStreamQueryAnalyzer.json"
         ],
-        "cwd": "build/Debug/bin/net7.0-windows",
+        "cwd": "build/Debug/bin/net8.0-windows",
         "env": {
           "PYTHON_EXE": "${workspaceFolder}/.venv/Scripts/python.exe"
         },
@@ -74,7 +114,7 @@
         "--config",
         "${workspaceFolder}/src/DataStreamSuite/PubSubServer/Configurations/${input:pubSubServerConfig}.json"
       ],
-      "cwd": "build/Debug/bin/net7.0",
+      "cwd": "build/Debug/bin/net8.0",
       "stopAtEntry": false,
       "console": "integratedTerminal"
     },
@@ -86,7 +126,19 @@
       "windows": {
         "preLaunchTask": "Build Spark (Debug)",
         "program": "Spark.exe",
-        "cwd": "build/Debug/bin/net7.0-windows",
+        "cwd": "build/Debug/bin/net8.0-windows",
+      },
+      "stopAtEntry": false,
+      "console": "integratedTerminal"
+    },
+    {
+      "name": "Spark (Skip Build)",
+      "type": "coreclr",
+      "request": "launch",
+      "program": "",
+      "windows": {
+        "program": "Spark.exe",
+        "cwd": "build/Debug/bin/net8.0-windows",
       },
       "stopAtEntry": false,
       "console": "integratedTerminal"
@@ -103,27 +155,100 @@
           "--sparkd",
           "${input:sparkConfigServer}"
         ],
-        "cwd": "build/Debug/bin/net7.0-windows",
+        "cwd": "build/Debug/bin/net8.0-windows",
+      },
+      "stopAtEntry": false,
+      "console": "integratedTerminal"
+    },
+    {
+      "name": "Spark Test",
+      "type": "coreclr",
+      "request": "launch",
+      "program": "",
+      "windows": {
+        "preLaunchTask": "Build Spark (Debug)",
+        "program": "Spark.Console.exe",
+        "cwd": "build/Debug/bin/net8.0",
+        "args": [
+          "test",
+          "${input:sparkTest}"
+        ]
+      },
+      "stopAtEntry": false,
+      "console": "integratedTerminal"
+    },
+    {
+      "name": "Spark Test (Skip Build)",
+      "type": "coreclr",
+      "request": "launch",
+      "program": "",
+      "windows": {
+        "program": "Spark.Console.exe",
+        "cwd": "build/Debug/bin/net8.0",
+        "args": [
+          "test",
+          "${input:sparkTest}"
+        ]
       },
       "stopAtEntry": false,
       "console": "integratedTerminal"
     },
     {
       "name": "Sparkd",
-      "type": "coreclr",
+      "type": "cppdbg",
       "request": "launch",
-      "preLaunchTask": "Build Sparkd (Debug | Clang)",
       "program": "",
       "linux": {
-        "program": "sparkd",
+        "program": "${workspaceFolder}/build/sparkd-debug-clang/src/app_sparkd/sparkd",
       },
       "args": [
         "run"
       ],
-      "cwd": "build/sparkd-debug-clang/server",
+      "cwd": "${workspaceFolder}/build/sparkd-debug-clang/src/app_sparkd",
       "stopAtEntry": false,
-      "console": "integratedTerminal"
-    }
+      "externalConsole": false,
+      "MIMode": "gdb",
+      "setupCommands": [
+        {
+          "description": "Enable pretty-printing for gdb",
+          "text": "-enable-pretty-printing",
+          "ignoreFailures": true
+        },
+        {
+          "description": "Set Disassembly Flavor to Intel",
+          "text": "-gdb-set disassembly-flavor intel",
+          "ignoreFailures": true
+        }
+      ]
+    },
+    {
+      "name": "Sparkd (Skip Build)",
+      "type": "cppdbg",
+      "request": "launch",
+      "program": "",
+      "linux": {
+        "program": "${workspaceFolder}/build/sparkd-debug-clang/src/app_sparkd/sparkd",
+      },
+      "args": [
+        "run"
+      ],
+      "cwd": "${workspaceFolder}/build/sparkd-debug-clang/src/app_sparkd",
+      "stopAtEntry": false,
+      "externalConsole": false,
+      "MIMode": "gdb",
+      "setupCommands": [
+        {
+          "description": "Enable pretty-printing for gdb",
+          "text": "-enable-pretty-printing",
+          "ignoreFailures": true
+        },
+        {
+          "description": "Set Disassembly Flavor to Intel",
+          "text": "-gdb-set disassembly-flavor intel",
+          "ignoreFailures": true
+        }
+      ]
+    },
   ],
   "inputs": [
     {
@@ -154,5 +279,15 @@
       "description": "Config Server URL",
       "default": "http://localhost:8000/config",
     },
+    {
+      "id": "sparkTest",
+      "type": "promptString",
+      "description": "Test to run",
+    },
+    {
+      "id": "sparkdTest",
+      "type": "promptString",
+      "description": "Test to run",
+    },
   ]
-}
+}

.vscode/tasks.json

@@ -641,6 +641,21 @@
         "color": "terminal.ansiGreen",
       }
     },
+    {
+      "label": "Run Spark (Skip Build)",
+      "type": "process",
+      "command": "",
+      "windows": {
+        "command": "Spark.exe",
+        "options": {
+          "cwd": "${workspaceFolder}/build/${input:buildConfiguration}/bin/net7.0-windows"
+        },
+      },
+      "icon": {
+        "id": "run",
+        "color": "terminal.ansiGreen",
+      }
+    },
     {
       "label": "Run Spark Config Query",
       "dependsOn": "Build Spark",
@@ -684,6 +699,24 @@
         "color": "terminal.ansiGreen",
       },
     },
+    {
+      "label": "Run Sparkd (Skip Build)",
+      "type": "process",
+      "command": "",
+      "linux": {
+        "command": "sparkd",
+      },
+      "args": [
+        "run"
+      ],
+      "options": {
+        "cwd": "${workspaceFolder}/build/${input:sparkdCmakePreset}/server"
+      },
+      "icon": {
+        "id": "run",
+        "color": "terminal.ansiGreen",
+      },
+    },
     {
       "label": "Run Sparkd Config Query",
       "dependsOn": "Build Sparkd",
@@ -705,6 +738,65 @@
         "color": "terminal.ansiGreen",
       },
     },
+    {
+      "label": "Test Spark (Debug)",
+      "dependsOn": "Build Spark (Debug)",
+      "type": "process",
+      "icon": {
+        "id": "coverage",
+        "color": "terminal.ansiYellow",
+      },
+      "command": "",
+      "windows": {
+        "command": "Spark.Console.exe",
+        "args": [
+          "test",
+          "${input:sparkTest}"
+        ],
+        "options": {
+          "cwd": "${workspaceFolder}/build/Debug/bin/net7.0",
+        }
+      },
+    },
+    {
+      "label": "Test Sparkd",
+      "dependsOn": "Build Sparkd",
+      "type": "process",
+      "command": "",
+      "linux": {
+        "command": "sparkd",
+      },
+      "args": [
+        "test",
+        "${input:sparkdTest}",
+      ],
+      "options": {
+        "cwd": "${workspaceFolder}/build/${input:sparkdCmakePreset}/server"
+      },
+      "icon": {
+        "id": "coverage",
+        "color": "terminal.ansiYellow",
+      },
+    },
+    {
+      "label": "Test Sparkd (Skip Build)",
+      "type": "process",
+      "command": "",
+      "linux": {
+        "command": "sparkd",
+      },
+      "args": [
+        "test",
+        "${input:sparkdTest}",
+      ],
+      "options": {
+        "cwd": "${workspaceFolder}/build/${input:sparkdCmakePreset}/server"
+      },
+      "icon": {
+        "id": "coverage",
+        "color": "terminal.ansiYellow",
+      },
+    },
     {
       "label": "View DataStreamQueryAnalyzer Logs (Last Run)",
       "type": "process",
@@ -823,6 +915,16 @@
       "id": "sparkdConfigQuery",
       "type": "promptString",
       "description": "Config Type(s)"
-    }
+    },
+    {
+      "id": "sparkTest",
+      "type": "promptString",
+      "description": "Test to run",
+    },
+    {
+      "id": "sparkdTest",
+      "type": "promptString",
+      "description": "Test to run",
+    },
   ]
 }

CMakeLists.txt

@@ -1,8 +1,10 @@
 cmake_minimum_required(VERSION 3.12)
 project(spark)
 include(GNUInstallDirs)
+include(${CMAKE_SOURCE_DIR}/cmake/GitVersion.cmake)
 set(CMAKE_CXX_STANDARD 20)
 
+get_version_from_git()
 if(WIN32)
   set(platform "windows")
 else()
@@ -20,9 +22,9 @@ set(
 # can run simultaneously without fighting over the same shm files (e.g., sparkd,
 # api server, etc.).
 set(SHARED_MEMORY_PREFIX "bts" CACHE STRING "Prefix to use when creating shared memory files")
-set(SPARK_BIN "${EXTERNALS_ROOT}/bin")
-set(SPARK_INCLUDE "${EXTERNALS_ROOT}/include")
-set(SPARK_LIB "${EXTERNALS_ROOT}/lib/$<CONFIG>")
+set(EXTERNALS_BIN "${EXTERNALS_ROOT}/bin")
+set(EXTERNALS_INCLUDE "${EXTERNALS_ROOT}/include")
+set(EXTERNALS_LIB "${EXTERNALS_ROOT}/lib/$<CONFIG>")
 set(BUILD_DOCS OFF CACHE BOOL "Build documentation")
 set(BUILD_SDK OFF CACHE BOOL "Build the sdk")
 set(EXPORT_MD_CONSUMER OFF CACHE BOOL "Export (extern) C functions for MD consumer interop")
@@ -34,8 +36,8 @@ else()
   set(BUILD_ADMIN_WEB_APP ON CACHE BOOL "Build the Admin WebApp")
 endif()
 
-include_directories(BEFORE SYSTEM ${SPARK_INCLUDE} ${EXTERNALS_ROOT}/include/zlib)
-link_directories(${SPARK_LIB})
+include_directories(BEFORE SYSTEM ${EXTERNALS_INCLUDE} ${EXTERNALS_ROOT}/include/zlib)
+link_directories(${EXTERNALS_LIB})
 
 add_compile_definitions(SHARED_MEMORY_PREFIX="${SHARED_MEMORY_PREFIX}")
 add_compile_definitions(APP_DATA_DIR="${APP_DATA_DIR}")
@@ -45,12 +47,8 @@ endif()
 add_compile_definitions(EXPORT_MD_CONSUMER="${EXPORT_MD_CONSUMER}")
 if(WIN32)
   # subdirs
-  add_subdirectory(src/lib)
+  add_subdirectory(src/lib_spark)
 else()
-  # make sparkd look for libspark.so in its directory
-  set(CMAKE_BUILD_WITH_INSTALL_RPATH TRUE)
-  set(CMAKE_INSTALL_RPATH ".")
-
   if(BUILD_SPARKD)
     # Note: All macros that begin w/ 'CCAPI' are ccapi's macros (defined and/or
     # read by their code). We override some below.
@@ -88,9 +86,15 @@ else()
     add_compile_definitions(BINANCE_US_PRICE_STATS_PATH="/api/v3/ticker/24hr")
 
     # ** Binance USDS Futures **
-    add_compile_definitions(CCAPI_BINANCE_USDS_FUTURES_URL_REST_BASE="https://dapi.binance.com")
-    add_compile_definitions(BINANCE_USDS_FUTURES_INSTRUMENTS_PATH="/dapi/v1/exchangeInfo")
-    add_compile_definitions(BINANCE_USDS_FUTURES_PRICE_STATS_PATH="/dapi/v1/ticker/24hr")
+    add_compile_definitions(CCAPI_BINANCE_USDS_FUTURES_URL_REST_BASE="https://fapi.binance.com")
+    add_compile_definitions(BINANCE_USDS_FUTURES_INSTRUMENTS_PATH="/fapi/v1/exchangeInfo")
+    add_compile_definitions(BINANCE_USDS_FUTURES_PRICE_STATS_PATH="/fapi/v1/ticker/24hr")
+    add_compile_definitions(BINANCE_USDS_FUTURES_USER_DATA_PATH="/api/v3/userDataStream")
+
+    # ** Binance Coin Futures **
+    add_compile_definitions(CCAPI_BINANCE_COIN_FUTURES_URL_REST_BASE="https://dapi.binance.com")
+    add_compile_definitions(BINANCE_COIN_FUTURES_INSTRUMENTS_PATH="/dapi/v1/exchangeInfo")
+    add_compile_definitions(BINANCE_COIN_FUTURES_PRICE_STATS_PATH="/dapi/v1/ticker/24hr")
 
     # ** Deribit **
     add_compile_definitions(DERIBIT_INSTRUMENT_PATH="/api/v2/public/get_instrument")
@@ -109,11 +113,15 @@ else()
   endif()
 
   # subdirs
+  add_subdirectory(src/lib_spark)
   if (BUILD_SPARKD)
-    add_subdirectory(src/server/sparkd)
+    add_subdirectory(src/lib_python_interop)
+    add_subdirectory(src/lib_onixs_fix)
+    add_subdirectory(src/lib_binance_execution)
+    add_subdirectory(src/lib_okx_execution)
+    add_subdirectory(src/app_sparkd)
     add_subdirectory(src/data_files)
   endif()
-  add_subdirectory(src/lib)
   if (BUILD_ADMIN_WEB_APP)
     add_subdirectory(src/Spark.Admin.WebApp)
   endif()
@@ -123,37 +131,4 @@ else()
   if(BUILD_SDK)
     add_subdirectory(src/sdk)
   endif()
-
-  if(BUILD_SPARKD)
-    add_custom_command(
-      OUTPUT ${CMAKE_BINARY_DIR}/server
-      COMMAND mkdir -p ${CMAKE_BINARY_DIR}/server
-      COMMAND ln ARGS -sf ${CMAKE_BINARY_DIR}/src/server/sparkd/sparkd ${CMAKE_BINARY_DIR}/server
-      COMMAND ln ARGS -sf ${CMAKE_BINARY_DIR}/data_files ${CMAKE_BINARY_DIR}/server
-      COMMAND ln ARGS -sf ${CMAKE_BINARY_DIR}/src/lib/libspark.so ${CMAKE_BINARY_DIR}/server
-      COMMAND ln ARGS -sf ${EXTERNALS_ROOT}/lib/${CMAKE_BUILD_TYPE}/libonixs-fix-engine.so.4 ${CMAKE_BINARY_DIR}/server
-      
-      COMMAND
-        ln ARGS -sf
-        ${EXTERNALS_ROOT}/lib/Release/libOnixS.IceImpactMulticastPriceFeedHandlerCpp.so.3.16.0.0
-        ${CMAKE_BINARY_DIR}/server
-      COMMAND ln ARGS -sf ${EXTERNALS_ROOT}/lib/Release/libACE-5.7.5.so ${CMAKE_BINARY_DIR}/server
-      COMMAND mkdir ARGS -p ${CMAKE_BINARY_DIR}/server/plugins
-      COMMENT "Creating server runtime layout"
-      DEPENDS sparkd data_files_dir)
-
-    add_custom_target(server_layout ALL DEPENDS ${CMAKE_BINARY_DIR}/server)
-
-    install(
-      DIRECTORY "${CMAKE_BINARY_DIR}/server/plugins"
-      DESTINATION "${CMAKE_INSTALL_BINDIR}")
-    install(
-      FILES
-      "${SPARK_LIB}/libACE-5.7.5.so"
-      "${SPARK_LIB}/libOnixS.IceImpactMulticastPriceFeedHandlerCpp.so.3.16.0.0"
-      "${SPARK_LIB}/libonixs-fix-engine.so.4.10.1"
-      "${SPARK_LIB}/libonixs-fix-engine.so.4"
-      "${SPARK_LIB}/libonixs-fix-engine.so"
-      DESTINATION "${CMAKE_INSTALL_BINDIR}")
-  endif()
 endif()

CMakePresets.json

@@ -26,13 +26,13 @@
         "CMAKE_CXX_FLAGS_DEBUG": "$env{SPARK_CMAKE_CXX_FLAGS} /MTd",
         "CMAKE_CXX_FLAGS_RELEASE": "$env{SPARK_CMAKE_CXX_FLAGS} /MT",
         "CMAKE_CXX_STANDARD": "20",
+        "CMAKE_INSTALL_PREFIX": "${sourceDir}/install/${presetName}",
         "CMAKE_SHARED_LINKER_FLAGS_DEBUG": "/DEBUG",
         "CMAKE_SHARED_LINKER_FLAGS_RELEASE": "/OPT:REF /OPT:ICF /LTCG",
         "CMAKE_TOOLCHAIN_FILE": "${sourceDir}/vcpkg/scripts/buildsystems/vcpkg.cmake",
         "EXTERNALS_ROOT": "${sourceDir}/externals/windows",
-        "CONFIG_DEFAULTS_PATH": "config/profiles/default/*.conf",
-        "DOTNET_BINARY_DIR": "${sourceDir}/build/$<CONFIG>/bin/net7.0",
-        "DOTNET_WINDOWS_BINARY_DIR": "${sourceDir}/build/$<CONFIG>/bin/net7.0-windows",
+        "CONFIG_DEFAULTS_PATH": "spark/config/profiles/default/*.conf",
+        "DOTNET_BINARY_DIR": "${sourceDir}/build/$<CONFIG>/bin/net8.0;${sourceDir}/build/$<CONFIG>/bin/net8.0-windows",
         "EXPORT_MD_CONSUMER": {
           "type": "BOOL",
           "value": "ON"
@@ -57,12 +57,18 @@
         "APP_DATA_DIR_OVERRIDE": "${dollar}{SPARKD_DATA_DIR}",
         "CMAKE_BUILD_WITH_INSTALL_RPATH": {
           "type": "BOOL",
-          "value": "ON"
+          "value": "OFF"
         },
         "CMAKE_CXX_STANDARD": "20",
         "CMAKE_INSTALL_RPATH": ".",
         "CMAKE_INSTALL_PREFIX": "${sourceDir}/install/${presetName}",
+        "CMAKE_PREFIX_PATH": "${sourceDir}/cmake/;${sourceDir}/.venv/lib/python3.10/site-packages/pybind11/share/cmake/pybind11",
         "CMAKE_EXE_LINKER_FLAGS": "-Wno-undef",
+        "CMAKE_TOOLCHAIN_FILE": "${sourceDir}/vcpkg/scripts/buildsystems/vcpkg.cmake",
+        "DATABENTO_USE_EXTERNAL_JSON": {
+          "type": "BOOL",
+          "value": "ON"
+        },
         "EXTERNALS_ROOT": "${sourceDir}/externals/linux",
         "BUILD_SDK": {
           "type": "BOOL",
@@ -87,7 +93,7 @@
           "value": "ON"
         },
         "CCAPI_LOG_LEVEL": "Info",
-        "CONFIG_DEFAULTS_PATH": "config/profiles/md_dev/*.conf;config/profiles/default/*.conf"
+        "CONFIG_DEFAULTS_PATH": "spark/config/profiles/md_dev/*.conf;spark/config/profiles/default/*.conf"
       }
     },
 
@@ -98,7 +104,7 @@
       "cacheVariables": {
         "CMAKE_BUILD_TYPE": "Release",
         "CCAPI_LOG_LEVEL": "Warn",
-        "CONFIG_DEFAULTS_PATH": "config/profiles/md_prod/*.conf;config/profiles/default/*.conf"
+        "CONFIG_DEFAULTS_PATH": "spark/config/profiles/md_prod/*.conf;spark/config/profiles/default/*.conf"
       }
     },
 
@@ -180,8 +186,12 @@
         "CMAKE_CXX_FLAGS_DEBUG": "-g -fPIC -Wall -pedantic -Wextra -Werror -Wno-missing-braces -Wno-ignored-qualifiers --std=c++20 -fdiagnostics-color -DDEBUG",
         "CMAKE_CXX_FLAGS_RELEASE": "-fPIC -Wall -pedantic -Wextra -Werror -Wno-missing-braces -Wno-ignored-qualifiers --std=c++20 -fdiagnostics-color -fno-omit-frame-pointer -DNDEBUG -O3",
         "CMAKE_INSTALL_PREFIX": "${sourceDir}/install/${presetName}",
-        "CONFIG_DEFAULTS_PATH": "config/profiles/default/*.conf",
-        "DOTNET_BINARY_DIR": "${sourceDir}/build/$<CONFIG>/bin/net7.0",
+        "CMAKE_EXPORT_COMPILE_COMMANDS": {
+          "type": "BOOL",
+          "value": "ON"
+        },
+        "CONFIG_DEFAULTS_PATH": "spark/config/profiles/default/*.conf",
+        "DOTNET_BINARY_DIR": "${sourceDir}/build/$<CONFIG>/bin/net8.0",
         "EXPORT_MD_CONSUMER": {
           "type": "BOOL",
           "value": "ON"
@@ -221,7 +231,7 @@
       "cacheVariables": {
         "APP_DATA_DIR": "${dollar}{APPDATA}/bts/DataStream",
         "APP_DATA_DIR_OVERRIDE": "${dollar}{DATA_STREAM_DATA_DIR}",
-        "CONFIG_DEFAULTS_PATH": "config/profiles/default/*.conf;config/profiles/data_stream/*.conf",
+        "CONFIG_DEFAULTS_PATH": "spark/config/profiles/default/*.conf;spark/config/profiles/data_stream/*.conf",
         "SHARED_MEMORY_PREFIX": "data_stream"
       }
     },
@@ -256,8 +266,7 @@
         "CMAKE_CXX_COMPILER": "clang++",
         "CMAKE_CXX_FLAGS_DEBUG": "-g -fPIC -Wall -pedantic -Wextra -Werror -Wno-missing-braces -Wno-ignored-qualifiers --std=c++20 -fdiagnostics-color -DDEBUG",
         "CMAKE_CXX_FLAGS_RELEASE": "-fPIC -Wall -pedantic -Wextra -Werror -Wno-missing-braces -Wno-ignored-qualifiers --std=c++20 -fdiagnostics-color -fno-omit-frame-pointer -DNDEBUG -O3",
-        "CMAKE_INSTALL_PREFIX": "${sourceDir}/install/${presetName}",
-        "CONFIG_DEFAULTS_PATH": "config/profiles/default/*.conf;config/profiles/data_stream/*.conf",
+        "CONFIG_DEFAULTS_PATH": "spark/config/profiles/default/*.conf;spark/config/profiles/data_stream/*.conf",
         "DOTNET_BINARY_DIR": "${sourceDir}/build/$<CONFIG>/bin/net7.0",
         "EXPORT_MD_CONSUMER": {
           "type": "BOOL",
@@ -302,16 +311,7 @@
         "lhs": "${hostSystemName}",
         "rhs": "Linux"
       },
-      "hidden": true,
-      "targets": [
-        "data_files_dir",
-        "admin_web_app_bundle",
-        "install",
-        "libspark",
-        "rapidjson_project",
-        "server_layout",
-        "sparkd"
-      ]
+      "hidden": true
     },
 
     {

cmake/ExportToParent.cmake

@@ -0,0 +1,7 @@
+include_guard(GLOBAL)
+
+macro(export_to_parent)
+  foreach(var IN ITEMS ${ARGN})
+    set(${var} ${${var}} PARENT_SCOPE)
+  endforeach()
+endmacro()

cmake/GitVersion.cmake

@@ -0,0 +1,48 @@
+include_guard(GLOBAL)
+
+function(get_version_from_git)
+  find_package(Git QUIET)
+  if(NOT Git_FOUND)
+    message(WARNING "Git not found")
+    return()
+  endif()
+
+  execute_process(
+    COMMAND ${GIT_EXECUTABLE} describe --tags --always
+    WORKING_DIRECTORY ${CMAKE_SOURCE_DIR}
+    OUTPUT_VARIABLE GIT_TAG
+    OUTPUT_STRIP_TRAILING_WHITESPACE
+    RESULT_VARIABLE GIT_RESULT
+  )
+
+  if(NOT GIT_RESULT EQUAL 0)
+    message(WARNING "Failed to get git tag")
+    return()
+  endif()
+
+  execute_process(
+    COMMAND ${GIT_EXECUTABLE} rev-parse --short=7 HEAD
+    WORKING_DIRECTORY ${CMAKE_SOURCE_DIR}
+    OUTPUT_VARIABLE GIT_COMMIT_SHORT_HASH
+    OUTPUT_STRIP_TRAILING_WHITESPACE
+  )
+  set(PROJECT_VERSION_BUILD_METADATA ${GIT_COMMIT_SHORT_HASH})
+  set(PROJECT_VERSION_BUILD_METADATA ${GIT_COMMIT_SHORT_HASH} PARENT_SCOPE)
+
+  string(REGEX REPLACE "^v" "" CLEAN_TAG "${GIT_TAG}")
+  if(CLEAN_TAG MATCHES "^([0-9]+)\\.([0-9]+)\\.([0-9]+)(-.*)?$")
+    set(PROJECT_VERSION_MAJOR ${CMAKE_MATCH_1})
+    set(PROJECT_VERSION_MAJOR ${CMAKE_MATCH_1} PARENT_SCOPE)
+    set(PROJECT_VERSION_MINOR ${CMAKE_MATCH_2})
+    set(PROJECT_VERSION_MINOR ${CMAKE_MATCH_2} PARENT_SCOPE)
+    set(PROJECT_VERSION_PATCH ${CMAKE_MATCH_3})
+    set(PROJECT_VERSION_PATCH ${CMAKE_MATCH_3} PARENT_SCOPE)
+
+    set(PROJECT_VERSION "${CMAKE_MATCH_1}.${CMAKE_MATCH_2}.${CMAKE_MATCH_3}") 
+    set(PROJECT_VERSION "${CMAKE_MATCH_1}.${CMAKE_MATCH_2}.${CMAKE_MATCH_3}" PARENT_SCOPE)
+    set(PROJECT_VERSION_FULL "${CMAKE_MATCH_1}.${CMAKE_MATCH_2}.${CMAKE_MATCH_3}+${GIT_COMMIT_SHORT_HASH}")
+    set(PROJECT_VERSION_FULL "${CMAKE_MATCH_1}.${CMAKE_MATCH_2}.${CMAKE_MATCH_3}+${GIT_COMMIT_SHORT_HASH}" PARENT_SCOPE)
+  else()
+    message(WARNING "Tag '${CLEAN_TAG}' does not match semver format")
+  endif()
+endfunction()

cmake/InitLibrary.cmake

@@ -0,0 +1,142 @@
+include_guard(DIRECTORY)
+include(${CMAKE_SOURCE_DIR}/cmake/ExportToParent.cmake)
+include(${CMAKE_SOURCE_DIR}/cmake/InstallBasic.cmake)
+include(${CMAKE_SOURCE_DIR}/cmake/ProtobufGenerateCpp.cmake)
+include(${CMAKE_SOURCE_DIR}/cmake/SetFirstDefined.cmake)
+include(${CMAKE_SOURCE_DIR}/cmake/safe_dep_glob.cmake)
+
+function(init_library)
+  set(_options SHARED STATIC)
+  set(_one_value_args
+    NAME
+    SOURCE_DIR
+    INCLUDE_DIR
+  )
+  set(_multi_value_args)
+  cmake_parse_arguments(_ARGS "${_options}" "${_one_value_args}" "${_multi_value_args}" ${ARGN})
+  if (_ARGS_SHARED AND _ARGS_STATIC)
+    message(FATAL_ERROR "Invalid args passed to init_library: cannot specify STATIC and SHARED")
+  endif()
+  set_first_defined(_name _ARGS_NAME PROJECT_NAME)
+  set_first_defined(_source_dir _ARGS_SOURCE_DIR _name)
+  set_first_defined(_include_dir _ARGS_INCLUDE_DIR _source_dir)
+
+  # Cpp sources
+  file(
+    GLOB_RECURSE ${_name}_SOURCES
+    RELATIVE ${CMAKE_CURRENT_SOURCE_DIR}
+    ${_source_dir}/*.cpp
+  )
+  # Note: cpp files inside 'tests' dir(s) are considered tests.
+  # Remove tests from main sources, and add them to test sources
+  foreach(item ${${_name}_SOURCES})
+    if(${item} MATCHES "tests/.*\.cpp$")
+      list(APPEND ${_name}_TEST_SOURCES ${item})
+    endif()
+  endforeach()
+  foreach(item ${${_name}_TEST_SOURCES})
+    list(REMOVE_ITEM ${_name}_SOURCES ${item})
+  endforeach()
+
+  # Proto sources 
+  file(
+    GLOB_RECURSE ${_name}_PROTO_SOURCES
+    RELATIVE ${CMAKE_CURRENT_SOURCE_DIR}
+    ${_source_dir}/protobufs/*.proto
+  )
+  update_deps_file("${${_name}_PROTO_SOURCES};")
+  protobuf_generate_cpp(
+    CXX_SOURCES ${_name}_CXX_PROTO_SOURCES
+    CXX_HEADERS ${_name}_CXX_PROTO_HEADERS
+    CXX_OUT_DIR ${CMAKE_CURRENT_BINARY_DIR}
+    PROTO_INCLUDE_DIRS
+      ${EXTERNALS_INCLUDE}
+      ${CMAKE_CURRENT_SOURCE_DIR}
+    PROTO_SOURCES
+      ${${_name}_PROTO_SOURCES}
+  )
+
+  export_to_parent(
+    ${_name}_SOURCES
+    ${_name}_TEST_SOURCES
+    ${_name}_PROTO_SOURCES
+    ${_name}_CXX_PROTO_SOURCES
+    ${_name}_CXX_PROTO_HEADERS
+  )
+
+  # Add main lib
+  if (_ARGS_STATIC)
+    add_library(${_name} STATIC ${${_name}_SOURCES} ${${_name}_CXX_PROTO_SOURCES})
+  elseif(_ARGS_SHARED)
+    add_library(${_name} SHARED ${${_name}_SOURCES} ${${_name}_CXX_PROTO_SOURCES})
+  else()
+    add_library(${_name} ${${_name}_SOURCES} ${${_name}_CXX_PROTO_SOURCES})
+  endif()
+  target_include_directories(
+    ${_name}
+    SYSTEM
+    BEFORE
+    PUBLIC
+      $<BUILD_INTERFACE:${EXTERNALS_ROOT}/include>
+      $<BUILD_INTERFACE:${EXTERNALS_ROOT}/include/zlib>
+  )
+  target_include_directories(
+    ${_name}
+    PUBLIC
+      $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}>
+      $<INSTALL_INTERFACE:${CMAKE_INSTALL_INCLUDEDIR}>
+  )
+  # Include protos (if any)
+  if (${_name}_PROTO_SOURCES)
+    target_include_directories(
+      ${_name}
+      PUBLIC
+        $<BUILD_INTERFACE:${CMAKE_CURRENT_BINARY_DIR}>
+    )
+  endif()
+  target_link_directories(
+    ${_name}
+    PUBLIC
+      $<BUILD_INTERFACE:${CMAKE_CURRENT_BINARY_DIR}>
+      $<BUILD_INTERFACE:${EXTERNALS_ROOT}/lib/$<CONFIG>>
+      $<INSTALL_INTERFACE:${CMAKE_INSTALL_LIBDIR}>
+  )
+  # Install main lib
+  foreach(_hdr ${${_name}_CXX_PROTO_HEADERS})
+    file(RELATIVE_PATH _relative_path ${CMAKE_CURRENT_BINARY_DIR} ${_hdr})
+    install(FILES ${_hdr} DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}/${_relative_path})
+  endforeach()
+  install_basic(
+    NAME ${_name}
+    INCLUDE_DIR ${_include_dir}
+  )
+
+  # Add tests executable (if there are any)
+  if (${_name}_TEST_SOURCES AND NOT WIN32)
+    add_executable(${_name}-tests ${${_name}_TEST_SOURCES})
+
+    get_target_property(_lib_type ${_name} TYPE)
+    if (_lib_type STREQUAL "SHARED_LIBRARY")
+      # Set RPATH so the tests exe knows where to find the shared lib.
+      set_target_properties(
+        ${_name}-tests
+        PROPERTIES
+          BUILD_RPATH "$ORIGIN"
+          INSTALL_RPATH "$ORIGIN/../${CMAKE_INSTALL_LIBDIR}"
+      )
+    endif()
+
+    # Link main lib
+    target_link_directories(
+      ${_name}-tests
+      PRIVATE
+        $<BUILD_INTERFACE:${CMAKE_CURRENT_BINARY_DIR}>
+        $<INSTALL_INTERFACE:${CMAKE_INSTALL_LIBDIR}>
+    )
+    target_link_libraries(${_name}-tests PRIVATE ${_name})
+
+    # Catch2
+    find_package(Catch2 CONFIG REQUIRED COMPONENTS Catch2WithMain)
+    target_link_libraries(${_name}-tests PRIVATE Catch2::Catch2WithMain)
+  endif()
+endfunction()

cmake/InitPythonModule.cmake

@@ -0,0 +1,71 @@
+include_guard(DIRECTORY)
+include(GNUInstallDirs)
+include(${CMAKE_SOURCE_DIR}/cmake/ExportToParent.cmake)
+include(${CMAKE_SOURCE_DIR}/cmake/SetFirstDefined.cmake)
+
+function(init_python_module)
+  set(_options)
+  set(_one_value_args
+    NAME
+  )
+  set(_multi_value_args
+    SOURCES
+  )
+  cmake_parse_arguments(_ARGS "${_options}" "${_one_value_args}" "${_multi_value_args}" ${ARGN})
+  set_first_defined(_name _ARGS_NAME PROJECT_NAME)
+  set_first_defined(_sources _ARGS_SOURCES ${_name}_SOURCES)
+
+  set(module_proj ${_name}_module)
+  set(${module_proj}_SOURCES ${_sources})
+  pybind11_add_module(${module_proj} SHARED ${_sources})
+  target_link_libraries(
+    ${module_proj}
+    PUBLIC
+      pybind11::embed
+      spark
+      ${_name}
+  )
+  target_include_directories(
+    ${module_proj}
+    PRIVATE
+      $<BUILD_INTERFACE:${pybind11_INCLUDE_DIRS}>
+  )
+  target_include_directories(
+    ${module_proj}
+    PUBLIC
+      ${Python3_INCLUDE_DIRS}
+      $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}>
+      $<INSTALL_INTERFACE:${CMAKE_INSTALL_INCLUDEDIR}>
+  )
+  add_custom_command(
+    OUTPUT ${CMAKE_CURRENT_BINARY_DIR}/stubs/${_name}.pyi
+    DEPENDS ${_name} ${module_proj}
+    COMMAND PYTHONPATH=${CMAKE_CURRENT_BINARY_DIR} pybind11-stubgen --numpy-array-use-type-var --ignore-invalid-expressions '.*' ${_name}_module
+    COMMENT "Generating py stubs"
+  )
+  add_custom_target(
+    "${_name}_py_stubs"
+    ALL
+    DEPENDS
+      ${CMAKE_CURRENT_BINARY_DIR}/stubs/${_name}.pyi
+  )
+  install(
+    TARGETS ${module_proj}
+    EXPORT ${module_proj}Targets
+    RUNTIME
+      DESTINATION ${CMAKE_INSTALL_BINDIR}
+    LIBRARY
+      DESTINATION ${CMAKE_INSTALL_LIBDIR}
+    ARCHIVE
+      DESTINATION ${CMAKE_INSTALL_LIBDIR}
+  )
+  foreach(dir ${pybind11_INCLUDE_DIRS})
+    install(DIRECTORY ${dir} DESTINATION ${CMAKE_INSTALL_INCLUDEDIR})
+  endforeach()
+  install(
+    EXPORT ${module_proj}Targets
+    FILE ${module_proj}Targets.cmake
+    NAMESPACE ${_name}::
+    DESTINATION ${CMAKE_INSTALL_LIBDIR}/cmake/${_name}
+  )
+endfunction()

cmake/InitPythonPlugin.cmake

@@ -0,0 +1,65 @@
+include_guard(DIRECTORY)
+include(GNUInstallDirs)
+
+function(init_python_plugin)
+  set(_options)
+  set(_one_value_args NAME PLUGIN_NAME)
+  set(_multi_value_args SOURCES)
+  cmake_parse_arguments(_ARGS "${_options}" "${_one_value_args}" "${_multi_value_args}" ${ARGN})
+  set_first_defined(_name _ARGS_NAME PROJECT_NAME)
+  set(_plugin_name ${_ARGS_PLUGIN_NAME})
+  if (NOT DEFINED _ARGS_PLUGIN_NAME)
+    set(_plugin_name ${_name}_plugins)
+  endif()
+  set(${_plugin_name}_SOURCES ${_ARGS_SOURCES})
+  if (NOT DEFINED _ARGS_SOURCES)
+    file(
+      GLOB_RECURSE ${_plugin_name}_SOURCES
+      RELATIVE ${CMAKE_CURRENT_SOURCE_DIR}
+      ${_plugin_name}/*.py)
+  endif()
+  set(src_outs)
+  foreach(_src ${${_plugin_name}_SOURCES})
+    set(_src_in ${CMAKE_CURRENT_SOURCE_DIR}/${_src})
+    set(_src_out ${CMAKE_CURRENT_BINARY_DIR}/${_src})
+    add_custom_command(
+      OUTPUT ${_src_out}
+      COMMAND ${CMAKE_COMMAND} -E copy ${_src_in} ${_src_out}
+      DEPENDS ${_src_in}
+      COMMENT "Copying ${_plugin_name} plugin ${_src} to ${_src_out}"
+    )
+    list(APPEND src_outs ${_src_out})
+  endforeach()
+  
+  add_custom_target(copy_${_plugin_name} ALL DEPENDS ${src_outs})
+  
+  add_library(${_plugin_name} INTERFACE)
+  add_library(${_name}::${_plugin_name} ALIAS ${_plugin_name})
+  
+  set_target_properties(
+    ${_plugin_name}
+    PROPERTIES
+      INTERFACE_PYTHON_SCRIPTS_DIR
+        $<BUILD_INTERFACE:${CMAKE_CURRENT_BINARY_DIR}/${_plugin_name}>$<INSTALL_INTERFACE:${CMAKE_INSTALL_BINDIR}/${_plugin_name}>
+  )
+  
+  set_target_properties(
+    ${_plugin_name}
+    PROPERTIES
+    INTERFACE_PYTHON_SCRIPTS $<BUILD_INTERFACE:${CMAKE_CURRENT_BINARY_DIR}/${${_plugin_name}_SOURCES}>$<INSTALL_INTERFACE:${CMAKE_INSTALL_BINDIR}/${${_plugin_name}_SOURCES}>
+  )
+  
+  install(
+    TARGETS ${_plugin_name}
+    EXPORT ${_plugin_name}Targets
+    DESTINATION ${CMAKE_INSTALL_BINDIR}
+  )
+  
+  install(
+    EXPORT ${_plugin_name}Targets
+    FILE ${_plugin_name}Targets.cmake
+    NAMESPACE ${_name}::
+    DESTINATION ${CMAKE_INSTALL_LIBDIR}/cmake/${_name}
+  )
+  install(DIRECTORY ${_plugin_name} DESTINATION ${CMAKE_INSTALL_BINDIR})
+endfunction()

cmake/InstallBasic.cmake

@@ -0,0 +1,39 @@
+include_guard(DIRECTORY)
+include(GNUInstallDirs)
+include(${CMAKE_SOURCE_DIR}/cmake/PackBasic.cmake)
+include(${CMAKE_SOURCE_DIR}/cmake/SetFirstDefined.cmake)
+
+function(install_basic)
+  set(_options)
+  set(_one_value_args NAME INCLUDE_DIR)
+  set(_multi_value_args)
+  cmake_parse_arguments(_ARGS "${_options}" "${_one_value_args}" "${_multi_value_args}" ${ARGN})
+  set_first_defined(_name _ARGS_NAME PROJECT_NAME)
+  set_first_defined(_include_dir _ARGS_INCLUDE_DIR _name)
+
+  file(
+    GLOB_RECURSE ${_name}_HEADERS
+    RELATIVE ${CMAKE_CURRENT_SOURCE_DIR}
+    ${_include_dir}/*.h)
+
+  target_sources(${_name}
+    PUBLIC
+      FILE_SET HEADERS FILES ${${_name}_HEADERS})
+
+  install(
+    TARGETS ${_name}
+    EXPORT ${_name}Targets
+    RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR}
+    LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR}
+    ARCHIVE DESTINATION ${CMAKE_INSTALL_LIBDIR}
+    FILE_SET HEADERS DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}
+  )
+
+  install(
+    EXPORT ${_name}Targets
+    FILE ${_name}Targets.cmake
+    NAMESPACE ${_name}::
+    DESTINATION ${CMAKE_INSTALL_LIBDIR}/cmake/${_name})
+
+  pack_basic()
+endfunction()

cmake/PackBasic.cmake

@@ -0,0 +1,44 @@
+include_guard(DIRECTORY)
+include(CMakePackageConfigHelpers)
+include(${CMAKE_SOURCE_DIR}/cmake/SetFirstDefined.cmake)
+
+function(pack_basic)
+  set(_options)
+  set(_one_value_args NAME)
+  set(_multi_value_args)
+  cmake_parse_arguments(_ARGS "${_options}" "${_one_value_args}" "${_multi_value_args}" ${ARGN})
+  set_first_defined(_name _ARGS_NAME PROJECT_NAME)
+
+  find_package(Git QUIET)
+
+  if(GIT_FOUND)
+    execute_process(
+      COMMAND ${GIT_EXECUTABLE} describe --tags --always --dirty
+      WORKING_DIRECTORY ${CMAKE_SOURCE_DIR}
+      OUTPUT_VARIABLE PROJECT_VERSION
+      OUTPUT_STRIP_TRAILING_WHITESPACE
+    )
+  endif()
+
+  if(NOT GIT_FOUND OR PROJECT_VERSION STREQUAL "")
+    set(PROJECT_VERSION "0.0.0")
+  endif()
+
+  write_basic_package_version_file(
+    "${CMAKE_CURRENT_BINARY_DIR}/${_name}ConfigVersion.cmake"
+    VERSION ${PROJECT_VERSION}
+    COMPATIBILITY AnyNewerVersion
+  )
+
+  configure_package_config_file(
+    "${CMAKE_CURRENT_SOURCE_DIR}/Config.cmake.in"
+    "${CMAKE_CURRENT_BINARY_DIR}/${_name}Config.cmake"
+    INSTALL_DESTINATION ${CMAKE_INSTALL_LIBDIR}/cmake/${_name}
+  )
+
+  install(
+    FILES
+      "${CMAKE_CURRENT_BINARY_DIR}/${_name}Config.cmake"
+      "${CMAKE_CURRENT_BINARY_DIR}/${_name}ConfigVersion.cmake"
+    DESTINATION ${CMAKE_INSTALL_LIBDIR}/cmake/${_name})
+endfunction()

cmake/Protobuf.cmake

@@ -1,12 +1,23 @@
-set(Protobuf_PROTOC_EXECUTABLE "${SPARK_BIN}/protoc")
-set(PROTOBUF_INCLUDE_DIRS "${SPARK_INCLUDE}")
+set(Protobuf_PROTOC_EXECUTABLE "${EXTERNALS_BIN}/protoc")
+set(PROTOBUF_INCLUDE_DIRS "${EXTERNALS_INCLUDE}")
 set(PROTOBUF_STATIC_LIB "libprotobuf")
+set(_protobuf_src_dir ${CMAKE_SOURCE_DIR}/src/lib_spark)
+set(_protobuf_bin_dir ${CMAKE_BINARY_DIR}/src/lib_spark)
 
-function(Protobuf_Generate_Cpp SRCS HDRS)
-  set(${SRCS})
-  set(${HDRS})
+function(Protobuf_Generate_Cpp)
+  set(_options)
+  set(_one_value_args CXX_SOURCES CXX_HEADERS CXX_OUT_DIR)
+  set(_multi_value_args PROTO_INCLUDE_DIRS PROTO_SOURCES)
+  cmake_parse_arguments(_ARGS "${_options}" "${_one_value_args}" "${_multi_value_args}" ${ARGN})
+  set(${_ARGS_CXX_SOURCES})
+  set(${_ARGS_CXX_HEADERS})
 
-  foreach(FIL ${ARGN})
+  set(_proto_include_cli_args)
+  foreach(_dir ${_ARGS_PROTO_INCLUDE_DIRS})
+    list(APPEND _proto_include_cli_args -I ${_dir})
+  endforeach()
+
+  foreach(FIL ${_ARGS_PROTO_SOURCES})
     get_filename_component(ABS_FIL ${FIL} ABSOLUTE)
     get_filename_component(FIL_WE ${FIL} NAME_WE)
     get_filename_component(FIL_DIR ${FIL} DIRECTORY)
@@ -14,48 +25,56 @@ function(Protobuf_Generate_Cpp SRCS HDRS)
       set(FIL_WE "${FIL_DIR}/${FIL_WE}")
     endif()
 
-    list(APPEND ${SRCS} "${CMAKE_CURRENT_BINARY_DIR}/${FIL_WE}.pb.cc")
-    list(APPEND ${HDRS} "${CMAKE_CURRENT_BINARY_DIR}/${FIL_WE}.pb.h")
+    list(APPEND ${_ARGS_CXX_SOURCES} "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.pb.cc")
+    list(APPEND ${_ARGS_CXX_HEADERS} "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.pb.h")
 
     add_custom_command(
-      OUTPUT "${CMAKE_CURRENT_BINARY_DIR}/${FIL_WE}.pb.cc"
-             "${CMAKE_CURRENT_BINARY_DIR}/${FIL_WE}.pb.h"
+      OUTPUT "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.pb.cc"
+             "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.pb.h"
       COMMAND
-        ${Protobuf_PROTOC_EXECUTABLE} ARGS --cpp_out=${CMAKE_BINARY_DIR}/src -I
-        ${PROTOBUF_INCLUDE_DIRS} -I ${CMAKE_SOURCE_DIR}/src ${ABS_FIL}
+        ${Protobuf_PROTOC_EXECUTABLE} ARGS --cpp_out=${_ARGS_CXX_OUT_DIR} ${_proto_include_cli_args} ${ABS_FIL}
       DEPENDS ${ABS_FIL}
       COMMENT "Running C++ protocol buffer compiler on ${FIL}"
       VERBATIM)
   endforeach()
 
-  set_source_files_properties(${${SRCS}} ${${HDRS}} PROPERTIES GENERATED TRUE)
+  set_source_files_properties(${${_ARGS_CXX_SOURCES}} ${${_ARGS_CXX_HEADERS}} PROPERTIES GENERATED TRUE)
 
   # remove some warnings from protobuf generated files
   if(WIN32)
-    set_source_files_properties(${${SRCS}} PROPERTIES COMPILE_FLAGS
+    set_source_files_properties(${${_ARGS_CXX_SOURCES}} PROPERTIES COMPILE_FLAGS
                                                       "/wd4309 /wd4838")
   else()
     set_source_files_properties(
-      ${${SRCS}}
+      ${${_ARGS_CXX_SOURCES}}
       PROPERTIES
         COMPILE_FLAGS
         "-Wno-unused-parameter -Wno-effc++ -Wno-pedantic -Wno-sign-compare -Wno-uninitialized"
     )
   endif()
 
-  set(${SRCS}
-      ${${SRCS}}
+  set(${_ARGS_CXX_SOURCES}
+      ${${_ARGS_CXX_SOURCES}}
       PARENT_SCOPE)
-  set(${HDRS}
-      ${${HDRS}}
+  set(${_ARGS_CXX_HEADERS}
+      ${${_ARGS_CXX_HEADERS}}
       PARENT_SCOPE)
 endfunction()
 
-function(Service_Compiler_Generate_Cpp SRCS HDRS)
-  set(${SRCS})
-  set(${HDRS})
+function(Service_Compiler_Generate_Cpp)
+  set(_options)
+  set(_one_value_args CXX_SOURCES CXX_HEADERS CXX_OUT_DIR)
+  set(_multi_value_args PROTO_INCLUDE_DIRS PROTO_SOURCES)
+  cmake_parse_arguments(_ARGS "${_options}" "${_one_value_args}" "${_multi_value_args}" ${ARGN})
+  set(${_ARGS_CXX_SOURCES})
+  set(${_ARGS_CXX_HEADERS})
 
-  foreach(FIL ${ARGN})
+  set(_proto_include_cli_args)
+  foreach(_dir ${_ARGS_PROTO_INCLUDE_DIRS})
+    list(APPEND _proto_include_cli_args -I ${_dir})
+  endforeach()
+
+  foreach(FIL ${_ARGS_PROTO_SOURCES})
     get_filename_component(ABS_FIL ${FIL} ABSOLUTE)
     get_filename_component(FIL_WE ${FIL} NAME_WE)
     get_filename_component(FIL_DIR ${FIL} DIRECTORY)
@@ -63,58 +82,58 @@ function(Service_Compiler_Generate_Cpp SRCS HDRS)
       set(FIL_WE "${FIL_DIR}/${FIL_WE}")
     endif()
 
-    list(APPEND ${SRCS} "${CMAKE_CURRENT_BINARY_DIR}/${FIL_WE}.pb.cc")
-    list(APPEND ${HDRS} "${CMAKE_CURRENT_BINARY_DIR}/${FIL_WE}.pb.h")
+    list(APPEND ${_ARGS_CXX_SOURCES} "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.pb.cc")
+    list(APPEND ${_ARGS_CXX_HEADERS} "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.pb.h")
 
-    list(APPEND ${SRCS} "${CMAKE_CURRENT_BINARY_DIR}/${FIL_WE}.client.pb.cc")
-    list(APPEND ${HDRS} "${CMAKE_CURRENT_BINARY_DIR}/${FIL_WE}.client.pb.h")
+    list(APPEND ${_ARGS_CXX_SOURCES} "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.client.pb.cc")
+    list(APPEND ${_ARGS_CXX_HEADERS} "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.client.pb.h")
 
-    list(APPEND ${SRCS} "${CMAKE_CURRENT_BINARY_DIR}/${FIL_WE}.server.pb.cc")
-    list(APPEND ${HDRS} "${CMAKE_CURRENT_BINARY_DIR}/${FIL_WE}.server.pb.h")
+    list(APPEND ${_ARGS_CXX_SOURCES} "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.server.pb.cc")
+    list(APPEND ${_ARGS_CXX_HEADERS} "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.server.pb.h")
 
     set(TEMPLATES
-        ${CMAKE_CURRENT_SOURCE_DIR}/api/service_compiler/templates/cpp/client.pb.cc
-        ${CMAKE_CURRENT_SOURCE_DIR}/api/service_compiler/templates/cpp/client.pb.h
-        ${CMAKE_CURRENT_SOURCE_DIR}/api/service_compiler/templates/cpp/server.pb.cc
-        ${CMAKE_CURRENT_SOURCE_DIR}/api/service_compiler/templates/cpp/server.pb.h
+        ${CMAKE_SOURCE_DIR}/src/lib_spark/spark/api/service_compiler/templates/cpp/client.pb.cc
+        ${CMAKE_SOURCE_DIR}/src/lib_spark/spark/api/service_compiler/templates/cpp/client.pb.h
+        ${CMAKE_SOURCE_DIR}/src/lib_spark/spark/api/service_compiler/templates/cpp/server.pb.cc
+        ${CMAKE_SOURCE_DIR}/src/lib_spark/spark/api/service_compiler/templates/cpp/server.pb.h
     )
 
+    set(_plugin_py ${CMAKE_SOURCE_DIR}/src/lib_spark/spark/api/service_compiler/plugin.py)
     if(WIN32)
+      set(_plugin_cmd ${CMAKE_SOURCE_DIR}/src/lib_spark/spark/api/service_compiler/plugin.cmd)
       add_custom_command(
-        OUTPUT "${CMAKE_CURRENT_BINARY_DIR}/${FIL_WE}.pb.cc"
-               "${CMAKE_CURRENT_BINARY_DIR}/${FIL_WE}.pb.h"
-               "${CMAKE_CURRENT_BINARY_DIR}/${FIL_WE}.client.pb.cc"
-               "${CMAKE_CURRENT_BINARY_DIR}/${FIL_WE}.client.pb.h"
-               "${CMAKE_CURRENT_BINARY_DIR}/${FIL_WE}.server.pb.cc"
-               "${CMAKE_CURRENT_BINARY_DIR}/${FIL_WE}.server.pb.h"
+        OUTPUT "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.pb.cc"
+               "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.pb.h"
+               "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.client.pb.cc"
+               "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.client.pb.h"
+               "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.server.pb.cc"
+               "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.server.pb.h"
         COMMAND SET PYTHONPATH=${SPARK_PYTHON}
         COMMAND ${CMAKE_SOURCE_DIR}/.venv/Scripts/activate.bat
         COMMAND
-          ${Protobuf_PROTOC_EXECUTABLE} ARGS --cpp_out=${CMAKE_BINARY_DIR}/src
-          --plugin=protoc-gen-service=${CMAKE_SOURCE_DIR}/src/lib/api/service_compiler/plugin.cmd
-          --service_out=lang=cpp:${CMAKE_BINARY_DIR}/src -I
-          ${PROTOBUF_INCLUDE_DIRS} -I ${CMAKE_SOURCE_DIR}/src ${ABS_FIL}
-        DEPENDS ${ABS_FIL} ${TEMPLATES}
-                ${CMAKE_CURRENT_SOURCE_DIR}/api/service_compiler/plugin.py
+          ${Protobuf_PROTOC_EXECUTABLE} ARGS --cpp_out=${_ARGS_CXX_OUT_DIR}
+          --plugin=protoc-gen-service=${_plugin_cmd}
+          --service_out=lang=cpp:${_ARGS_CXX_OUT_DIR} -I
+          ${PROTOBUF_INCLUDE_DIRS} -I ${_protobuf_src_dir} ${ABS_FIL}
+        DEPENDS ${ABS_FIL} ${TEMPLATES} ${_plugin_cmd} ${_plugin_py}
         COMMENT "Running C++ service compiler on ${FIL}"
         VERBATIM)
     else()
       add_custom_command(
-        OUTPUT "${CMAKE_CURRENT_BINARY_DIR}/${FIL_WE}.pb.cc"
-               "${CMAKE_CURRENT_BINARY_DIR}/${FIL_WE}.pb.h"
-               "${CMAKE_CURRENT_BINARY_DIR}/${FIL_WE}.client.pb.cc"
-               "${CMAKE_CURRENT_BINARY_DIR}/${FIL_WE}.client.pb.h"
-               "${CMAKE_CURRENT_BINARY_DIR}/${FIL_WE}.server.pb.cc"
-               "${CMAKE_CURRENT_BINARY_DIR}/${FIL_WE}.server.pb.h"
+        OUTPUT "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.pb.cc"
+               "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.pb.h"
+               "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.client.pb.cc"
+               "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.client.pb.h"
+               "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.server.pb.cc"
+               "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.server.pb.h"
         COMMAND ${CMAKE_SOURCE_DIR}/tools/venv_exec.sh
         ARGS
           ${Protobuf_PROTOC_EXECUTABLE}
-          --cpp_out=${CMAKE_BINARY_DIR}/src
-          --plugin=protoc-gen-service=${CMAKE_SOURCE_DIR}/src/lib/api/service_compiler/plugin.py
-          --service_out=lang=cpp,make_file_name=${ABS_FIL}:${CMAKE_BINARY_DIR}/src
-          -I ${PROTOBUF_INCLUDE_DIRS} -I ${CMAKE_SOURCE_DIR}/src ${ABS_FIL}
-        DEPENDS ${ABS_FIL} ${TEMPLATES}
-                ${CMAKE_CURRENT_SOURCE_DIR}/api/service_compiler/plugin.py
+          --cpp_out=${_ARGS_CXX_OUT_DIR}
+          --plugin=protoc-gen-service=${_plugin_py}
+          --service_out=lang=cpp,make_file_name=${ABS_FIL}:${_ARGS_CXX_OUT_DIR}
+          ${_proto_include_cli_args} ${ABS_FIL}
+        DEPENDS ${ABS_FIL} ${TEMPLATES} ${_plugin_py}
         COMMENT "Running C++ service compiler on ${FIL}"
         VERBATIM)
     endif()
@@ -123,22 +142,22 @@ function(Service_Compiler_Generate_Cpp SRCS HDRS)
 
   # remove some warnings from protobuf generated files
   if(WIN32)
-    set_source_files_properties(${${SRCS}} PROPERTIES COMPILE_FLAGS
+    set_source_files_properties(${${_ARGS_CXX_SOURCES}} PROPERTIES COMPILE_FLAGS
                                                       "/wd4309 /wd4838")
   else()
     set_source_files_properties(
-      ${${SRCS}}
+      ${${_ARGS_CXX_SOURCES}}
       PROPERTIES
         COMPILE_FLAGS
         "-Wno-unused-parameter -Wno-effc++ -Wno-pedantic -Wno-sign-compare -Wno-uninitialized"
     )
   endif()
 
-  set(${SRCS}
-      ${${SRCS}}
+  set(${_ARGS_CXX_SOURCES}
+      ${${_ARGS_CXX_SOURCES}}
       PARENT_SCOPE)
-  set(${HDRS}
-      ${${HDRS}}
+  set(${_ARGS_CXX_HEADERS}
+      ${${_ARGS_CXX_HEADERS}}
       PARENT_SCOPE)
 
 endfunction()

cmake/ProtobufGenerateCpp.cmake

@@ -0,0 +1,83 @@
+include_guard(GLOBAL)
+
+function(protobuf_generate_cpp)
+  set(_options)
+  set(_one_value_args
+    CXX_SOURCES
+    CXX_HEADERS
+    CXX_OUT_DIR
+    PROTOC
+  )
+  set(_multi_value_args
+    PROTO_INCLUDE_DIRS
+    PROTO_SOURCES
+  )
+  cmake_parse_arguments(_ARGS "${_options}" "${_one_value_args}" "${_multi_value_args}" ${ARGN})
+
+  if (NOT DEFINED _ARGS_PROTOC)
+    if (WIN32)
+      set(_ARGS_PROTOC "${EXTERNALS_BIN}/protoc.exe")
+    else()
+      set(_ARGS_PROTOC "${EXTERNALS_BIN}/protoc")
+    endif()
+  endif()
+  set(${_ARGS_CXX_SOURCES})
+  set(${_ARGS_CXX_HEADERS})
+
+  set(_proto_include_cli_args)
+  foreach(_dir ${_ARGS_PROTO_INCLUDE_DIRS})
+    list(APPEND _proto_include_cli_args -I ${_dir})
+  endforeach()
+
+  foreach(_fil ${_ARGS_PROTO_SOURCES})
+    get_filename_component(_abs_fil ${_fil} ABSOLUTE)
+    get_filename_component(_fil_we ${_fil} NAME_WE)
+    get_filename_component(_fil_dir ${_fil} DIRECTORY)
+    if(_fil_dir)
+      set(_fil_we "${_fil_dir}/${_fil_we}")
+    endif()
+    list(APPEND ${_ARGS_CXX_SOURCES} "${_ARGS_CXX_OUT_DIR}/${_fil_we}.pb.cc")
+    list(APPEND ${_ARGS_CXX_HEADERS} "${_ARGS_CXX_OUT_DIR}/${_fil_we}.pb.h")
+    message(STATUS "add_custom_command for ${_fil} -> ${_ARGS_CXX_OUT_DIR}/${_fil_we}.pb.h")
+    # TODO: Pass proto sources by relative path (so this fn can be called from
+    # arbitrary subdirs).
+    add_custom_command(
+      OUTPUT
+        "${_ARGS_CXX_OUT_DIR}/${_fil_we}.pb.cc"
+        "${_ARGS_CXX_OUT_DIR}/${_fil_we}.pb.h"
+      COMMAND
+        ${_ARGS_PROTOC}
+      ARGS
+        --cpp_out=${_ARGS_CXX_OUT_DIR} ${_proto_include_cli_args} ${_abs_fil}
+      DEPENDS ${_abs_fil}
+      COMMENT "Running C++ protocol buffer compiler on ${_fil}"
+      VERBATIM
+    )
+  endforeach()
+
+  set_source_files_properties(
+    ${${_ARGS_CXX_SOURCES}}
+    ${${_ARGS_CXX_HEADERS}}
+    PROPERTIES
+      GENERATED TRUE
+  )
+
+  # remove some warnings from protobuf generated files
+  if(WIN32)
+    set_source_files_properties(
+      ${${_ARGS_CXX_SOURCES}}
+      PROPERTIES
+        COMPILE_FLAGS "/wd4309 /wd4838"
+    )
+  else()
+    set_source_files_properties(
+      ${${_ARGS_CXX_SOURCES}}
+      PROPERTIES
+        COMPILE_FLAGS "-Wno-unused-parameter -Wno-effc++ -Wno-pedantic -Wno-sign-compare -Wno-uninitialized"
+    )
+  endif()
+
+  set(${_ARGS_CXX_SOURCES} ${${_ARGS_CXX_SOURCES}} PARENT_SCOPE)
+  set(${_ARGS_CXX_HEADERS} ${${_ARGS_CXX_HEADERS}} PARENT_SCOPE)
+endfunction()
+

cmake/ProtobufGenerateCppConfig.cmake

@@ -0,0 +1,61 @@
+include_guard(GLOBAL)
+
+function(protobuf_generate_cpp_config)
+  set(_options)
+  set(_one_value_args
+    PROTO_SOURCES_HEADER
+    PROTO_DEFAULTS_HEADER
+  )
+  set(_multi_value_args
+    PROTO_SOURCES
+    PROTO_DEFAULTS
+  )
+  cmake_parse_arguments(_ARGS "${_options}" "${_one_value_args}" "${_multi_value_args}" ${ARGN})
+
+  # Configure the 'config_proto_files' header. It provides an array of strings populated with the
+  # files in PROTO_SOURCES. At runtime, we use this array to generate a special 'config' proto
+  # message with fields for each message defined in said files. A global instance of this 'config'
+  # proto is then constructed, and populated with [compile-time] defaults, and [runtime] overrides
+  # (i.e., in spark.conf). Calls to `bts::config::get<T>()` retrieve the value from the field
+  # associated with type `T` in this global instance.
+  message(STATUS "PROTO_SOURCES: ${_ARGS_PROTO_SOURCES}")
+  message(STATUS "PROTO_SOURCES_HEADER: ${_ARGS_PROTO_SOURCES_HEADER}")
+  message(STATUS "PROTO_DEFAULTS_HEADER: ${_ARGS_PROTO_DEFAULTS_HEADER}")
+  if (_ARGS_PROTO_SOURCES_HEADER)
+    set(PROTO_SOURCES)
+    set(PROTO_SOURCES_CXX_ARRAY_ITEMS "")
+    foreach(_proto_path ${_ARGS_PROTO_SOURCES})
+      list(APPEND PROTO_SOURCES ${_proto_path})
+      string(APPEND PROTO_SOURCES_CXX_ARRAY_ITEMS "\"${_proto_path}\",\n")
+    endforeach()
+    list(LENGTH PROTO_SOURCES PROTO_SOURCES_LENGTH)
+    message(STATUS "CONFIGURING PROTO_SOURCES_HEADER): ${_ARGS_PROTO_SOURCES_HEADER}")
+    configure_file(${CMAKE_SOURCE_DIR}/cmake/config_proto_files.h.in ${_ARGS_PROTO_SOURCES_HEADER})
+  endif()
+
+  # Generate static defaults. Ideally, this would be a single string, but we use an array because
+  # the compiler complains if the string is longer than 65k.
+  set(_cfg_defaults_str "inline const char * config_static_defaults[] = { ")
+
+  foreach(_cfg_file_path ${_ARGS_PROTO_DEFAULTS})
+    file(READ ${_cfg_file_path} _cfg_file_content)
+    string(REPLACE "\"" "\\\"" _cfg_file_content_escaped ${_cfg_file_content})
+    string(REPLACE "\n" ";" _cfg_lines ${_cfg_file_content_escaped})
+
+    foreach(_line ${_cfg_lines})
+      string(APPEND _cfg_defaults_str "\n  \"${_line}\",")
+    endforeach(_line)
+  endforeach(_cfg_file_path)
+
+  string(APPEND _cfg_defaults_str "};")
+  set(_old_cfg_defaults_str "")
+
+  if(EXISTS ${_ARGS_PROTO_DEFAULTS_HEADER})
+    file(READ ${_ARGS_PROTO_DEFAULTS_HEADER} _old_cfg_defaults_str)
+  endif()
+
+  if(NOT _cfg_defaults_str STREQUAL _old_cfg_defaults_str)
+    file(WRITE "${_ARGS_PROTO_DEFAULTS_HEADER}" "${_cfg_defaults_str}")
+  endif()
+endfunction()
+

cmake/ProtobufGenerateFs.cmake

@@ -0,0 +1,49 @@
+include_guard(DIRECTORY)
+
+function(protobuf_generate_fs)
+  set(_options)
+  set(_one_value_args TARGET_NAME OUT_DIR PROTOC)
+  set(_multi_value_args PROTO_INCLUDE_DIRS PROTO_SOURCES)
+  cmake_parse_arguments(_ARGS "${_options}" "${_one_value_args}" "${_multi_value_args}" ${ARGN})
+
+  if (NOT DEFINED _ARGS_TARGET_NAME)
+    set(_ARGS_TARGET_NAME "${PROJECT_NAME}_generate_fs_protos")
+  endif()
+  if (NOT DEFINED _ARGS_PROTOC)
+    set(_ARGS_PROTOC "${EXTERNALS_BIN}/protoc")
+  endif()
+
+  set(_proto_include_cli_args)
+  foreach(_dir ${_ARGS_PROTO_INCLUDE_DIRS})
+    list(APPEND _proto_include_cli_args -I ${_dir})
+  endforeach()
+
+  set(_outputs)
+  foreach(_proto_source ${_ARGS_PROTO_SOURCES})
+    # get_filename_component(_abs_proto_source ${_proto_source} ABSOLUTE)
+    get_filename_component(_proto_source_we ${_proto_source} NAME_WE)
+    # get_filename_component(_proto_source_dir ${_proto_source} DIRECTORY)
+    list(APPEND _outputs ${_ARGS_OUT_DIR}/${_proto_source_we}.fable.fs)
+  endforeach()
+
+  set(_venv_exec ${CMAKE_SOURCE_DIR}/tools/venv_exec.sh)
+  set(_plugin_py ${CMAKE_SOURCE_DIR}/src/lib_spark/spark/api/service_compiler/plugin.py)
+  set(_template_fs ${CMAKE_SOURCE_DIR}/src/lib_spark/spark/api/service_compiler/templates/fs/fable.fs)
+  add_custom_command(
+    OUTPUT ${_outputs}
+    WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}
+    COMMAND ${_venv_exec}
+      ${_ARGS_PROTOC}
+      --plugin=protoc-gen-service=${_plugin_py}
+      --service_out=lang=fs,strip_out_path=1:${_ARGS_OUT_DIR}
+      ${_proto_include_cli_args}
+      ${_ARGS_PROTO_SOURCES}
+    DEPENDS
+      ${_venv_exec}
+      ${_plugin_py}
+      ${_template_fs}
+      ${CMAKE_CURRENT_SOURCE_DIR}/package-lock.json
+      ${_ARGS_PROTO_SOURCES}
+  )
+  add_custom_target(${_ARGS_TARGET_NAME} ALL DEPENDS "${_outputs}")
+endfunction()

cmake/ProtobufGenerateJs.cmake

@@ -0,0 +1,33 @@
+include_guard(DIRECTORY)
+
+function(protobuf_generate_js)
+  set(_options)
+  set(_one_value_args TARGET_NAME OUTPUT)
+  set(_multi_value_args PROTO_INCLUDE_DIRS PROTO_SOURCES)
+  cmake_parse_arguments(_ARGS "${_options}" "${_one_value_args}" "${_multi_value_args}" ${ARGN})
+
+  if (NOT DEFINED _ARGS_TARGET_NAME)
+    set(_ARGS_TARGET_NAME "${PROJECT_NAME}_generate_js_protos")
+  endif()
+
+  set(_proto_include_cli_args)
+  foreach(_dir ${_ARGS_PROTO_INCLUDE_DIRS})
+    list(APPEND _proto_include_cli_args -p ${_dir})
+  endforeach()
+
+  add_custom_command(
+    OUTPUT ${_ARGS_OUTPUT}
+    WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}
+    COMMAND npx
+      --no-install pbjs
+      -t json-module
+      -w commonjs
+      -o ${_ARGS_OUTPUT}
+      ${_proto_include_cli_args}
+      ${_ARGS_PROTO_SOURCES}
+    DEPENDS
+      ${CMAKE_CURRENT_SOURCE_DIR}/package-lock.json
+      ${_ARGS_PROTO_SOURCES}
+  )
+  add_custom_target(${_ARGS_TARGET_NAME} ALL DEPENDS ${_ARGS_OUTPUT})
+endfunction()

cmake/SetFirstDefined.cmake

@@ -0,0 +1,10 @@
+include_guard(GLOBAL)
+
+macro(set_first_defined RESULT)
+  foreach(_arg ${ARGN})
+    if (DEFINED ${_arg})
+      set(${RESULT} ${${_arg}})
+      break()
+    endif()
+  endforeach()
+endmacro()

cmake/SymLink.cmake

@@ -0,0 +1,49 @@
+include_guard(DIRECTORY)
+include(${CMAKE_SOURCE_DIR}/cmake/TernaryOperator.cmake)
+
+function(sym_link_dir)
+  set(_options)
+  set(_one_value_args TARGET_NAME SOURCE DEST COMMENT)
+  set(_multi_value_args)
+  cmake_parse_arguments(_ARGS "${_options}" "${_one_value_args}" "${_multi_value_args}" ${ARGN})
+  add_custom_command(
+    OUTPUT ${_ARGS_DEST}
+    COMMAND ln ARGS -sf ${_ARGS_SOURCE} ${_ARGS_DEST}
+  )
+  set_ternary(_comment _ARGS_COMMENT "${_ARGS_COMMENT}" "linking dir ${_ARGS_SOURCE} -> ${_ARGS_DEST}")
+  add_custom_target(
+    ${_ARGS_TARGET_NAME} ALL
+    DEPENDS ${_ARGS_DEST}
+    COMMENT "${_comment}"
+  )
+endfunction()
+
+function(sym_link_file)
+  set(_options)
+  set(_one_value_args SOURCE DEST INSTALL_DEST)
+  set(_multi_value_args)
+  cmake_parse_arguments(_ARGS "${_options}" "${_one_value_args}" "${_multi_value_args}" ${ARGN})
+  file(
+    CREATE_LINK
+      ${_ARGS_SOURCE}
+      ${_ARGS_DEST}
+    SYMBOLIC
+  )
+  if (DEFINED _ARGS_INSTALL_DEST)
+    install(
+      FILES
+      ${_ARGS_SOURCE}
+      DESTINATION ${_ARGS_INSTALL_DEST}
+    )
+  endif()
+endfunction()
+
+function(sym_link_local_lib LIB_NAME)
+  set(_lib_file_name "${CMAKE_SHARED_LIBRARY_PREFIX}${LIB_NAME}.so")
+  file(
+    CREATE_LINK
+      "${CMAKE_BINARY_DIR}/src/lib_${LIB_NAME}/${_lib_file_name}"
+      "${CMAKE_CURRENT_BINARY_DIR}/${_lib_file_name}"
+    SYMBOLIC
+  )
+endfunction()

cmake/SymLinkPythonModule.cmake

@@ -0,0 +1,39 @@
+include_guard(DIRECTORY)
+include(${CMAKE_SOURCE_DIR}/cmake/SymLink.cmake)
+
+function(sym_link_python_module)
+  set(_options)
+  set(_one_value_args LIB_NAME MODULE_NAME PLUGIN_NAME)
+  set(_multi_value_args)
+  cmake_parse_arguments(_ARGS "${_options}" "${_one_value_args}"
+                        "${_multi_value_args}" ${ARGN})
+  if(NOT DEFINED _ARGS_LIB_NAME)
+    message(
+      FATAL_ERROR "LIB_NAME must be specified for sym_link_python_module()")
+  endif()
+  set(_lib_name ${_ARGS_LIB_NAME})
+  set(_module_name ${_ARGS_MODULE_NAME})
+  if(NOT DEFINED _ARGS_MODULE_NAME)
+    set(_module_name ${_lib_name}_module)
+  endif()
+  set(_plugin_name ${_ARGS_PLUGIN_NAME})
+  set(_deps ${_lib_name} ${_module_name})
+  if(DEFINED _ARGS_PLUGIN_NAME)
+    list(APPEND _deps ${_plugin_name} copy_${_plugin_name})
+  endif()
+  add_dependencies(${PROJECT_NAME} ${_deps})
+  sym_link_local_lib(${_lib_name})
+  set(_module_suffix cpython-310-x86_64-linux-gnu.so)
+  sym_link_file(
+    SOURCE
+    ${CMAKE_BINARY_DIR}/src/lib_${_lib_name}/${_module_name}.${_module_suffix}
+    DEST
+    ${CMAKE_CURRENT_BINARY_DIR}/${_module_name}.${_module_suffix}
+    INSTALL_DEST
+    ${CMAKE_INSTALL_BINDIR}/)
+  if(DEFINED _ARGS_PLUGIN_NAME)
+    get_target_property(plugins ${_plugin_name} INTERFACE_PYTHON_SCRIPTS_DIR)
+    sym_link_dir(TARGET_NAME ${PROJECT_NAME}_link_${_plugin_name} SOURCE
+                 ${plugins} DEST ${CMAKE_CURRENT_BINARY_DIR}/${_plugin_name})
+  endif()
+endfunction()

cmake/TernaryOperator.cmake

@@ -0,0 +1,9 @@
+include_guard(GLOBAL)
+
+macro(set_ternary RESULT CONDITION TRUE_VALUE FALSE_VALUE)
+  if(${CONDITION})
+    set(${RESULT} ${TRUE_VALUE})
+  else()
+    set(${RESULT} ${FALSE_VALUE})
+  endif()
+endmacro()

cmake/config_proto_files.h.in

@@ -0,0 +1,11 @@
+#pragma once
+
+#include <array>
+
+namespace bts::@PROJECT_NAME@::protobufs {
+
+inline std::array<const char *, @PROTO_SOURCES_LENGTH@> config_proto_files = {
+@PROTO_SOURCES_CXX_ARRAY_ITEMS@
+};
+
+} // namespace bts::@PROJECT_NAME@::protobufs

cmake/version.h.in

@@ -0,0 +1,11 @@
+#ifndef BTS_SPARK_VERSION_H
+#define BTS_SPARK_VERSION_H
+
+#define BTS_SPARK_VERSION_MAJOR @PROJECT_VERSION_MAJOR@
+#define BTS_SPARK_VERSION_MINOR @PROJECT_VERSION_MINOR@
+#define BTS_SPARK_VERSION_PATCH @PROJECT_VERSION_PATCH@
+#define BTS_SPARK_VERSION_BUILD_METADATA @PROJECT_VERSION_BUILD_METADATA@
+#define BTS_SPARK_VERSION "@PROJECT_VERSION@"
+#define BTS_SPARK_VERSION_FULL "@PROJECT_VERSION_FULL@"
+
+#endif

cmake/vs_filters.cmake

@@ -1,24 +0,0 @@
-cmake_minimum_required(VERSION 3.5)
-
-# Explicitly set policy to convert relative paths to absolute (although this is a default policy,
-# setting it explicitly prevents a compile-warning)
-cmake_policy(SET CMP0076 NEW)
-
-# add protos to the library to facilitate source_group
-target_sources(libspark PUBLIC ${Config_Protos} ${NonConfig_Protos} ${Service_Protos})
-
-function(assign_source_group)
-  foreach(_source IN ITEMS ${ARGN})
-    if(IS_ABSOLUTE "${_source}")
-      file(RELATIVE_PATH _source_rel "${CMAKE_CURRENT_SOURCE_DIR}" "${_source}")
-    else()
-      set(_source_rel "${_source}")
-    endif()
-    get_filename_component(_source_path "${_source_rel}" PATH)
-    string(REPLACE "/" "\\" _source_path_msvc "${_source_path}")
-    source_group("${_source_path_msvc}" FILES "${_source}")
-    set(SOURCE_FILES "${_source}")
-  endforeach()
-endfunction(assign_source_group)
-
-assign_source_group(${SparkLib_Sources} ${Config_Protos} ${NonConfig_Protos} ${Service_Protos})

compose.yaml

@@ -0,0 +1,106 @@
+services:
+  sparkd:
+    volumes:
+      - ${SPARKD__DATA_DIR:-sparkd_data}:/home/bts/sparkdata/data
+    build:
+      dockerfile: src/app_sparkd/Dockerfile
+      args:
+        - IMAGE=${SPARKD__IMAGE}
+        - SOURCE_DIR=${SPARKD__SOURCE_DIR}
+        - CONFIG_DIR=${SPARKD__CONFIG_DIR}
+        - PYTHON_REQUIREMENTS=${SPARKD__PYTHON_REQUIREMENTS}
+    shm_size: "${SPARKD__SHM_SIZE:-512mb}"
+    command: >
+      sh -c "
+        echo bts | sudo -S chown -R bts:bts /home/bts/sparkdata/data
+        if [ -e "/home/bts/sparkdata/data/logging.sparkd/log" ]; then
+          mv "/home/bts/sparkdata/data/logging.sparkd/log" "/home/bts/sparkdata/data/logging.sparkd/log_$(date '+%Y%m%d-%H%M')"
+        fi
+        if [ -e "/home/bts/sparkdata/conf/.env.local" ]; then
+          echo 'Sourcing /home/bts/sparkdata/conf/.env.local'
+          set -a
+          . /home/bts/sparkdata/conf/.env.local
+          set +a
+        fi
+        ./sparkd run"
+    env_file:
+      - .env
+      - ${SPARKD__CONFIG_DIR}/.env
+    ports:
+      - "${SPARKD__CONFIG_PORT}:${SPARKD__CONFIG_PORT}"
+      - "${SPARKD__QUICKBUS_PORT}:${SPARKD__QUICKBUS_PORT}"
+      - "${SPARKD__SIM_FIX_PORT}:${SPARKD__SIM_FIX_PORT}"
+    deploy:
+      resources:
+        limits:
+          memory: "${SPARKD__MEMORY:-4gb}"
+
+  sparkd-symbols:
+    volumes:
+      - ${SPARKD__DATA_DIR:-sparkd_data}:/home/bts/sparkdata/data
+      - ${SPARKD__RUNTIME_CONF_DIR:-sparkd_conf}:/home/bts/sparkdata/conf
+    build:
+      dockerfile: src/app_sparkd/Dockerfile
+      args:
+        - IMAGE=${SPARKD__IMAGE}
+        - SOURCE_DIR=${SPARKD__SOURCE_DIR}
+        - CONFIG_DIR=${SPARKD__CONFIG_DIR}
+        - PYTHON_REQUIREMENTS=${SPARKD__PYTHON_REQUIREMENTS}
+    shm_size: "${SPARKD__SHM_SIZE:-512mb}"
+    command: >
+      sh -c "
+        echo bts | sudo -S chown -R bts:bts /home/bts/sparkdata
+        if [ -e "/home/bts/sparkdata/data/logging.sparkd/log" ]; then
+          mv "/home/bts/sparkdata/data/logging.sparkd/log" "/home/bts/sparkdata/data/logging.sparkd/log_$(date '+%Y%m%d-%H%M')"
+        fi
+        ls -alh /home/bts/sparkdata/conf/
+        cat /home/bts/sparkdata/conf/.env.local
+        if [ -e "/home/bts/sparkdata/conf/.env.local" ]; then
+          echo 'Sourcing /home/bts/sparkdata/conf/.env.local'
+          set -a
+          . /home/bts/sparkdata/conf/.env.local
+          set +a
+        fi
+        ./sparkd run"
+    env_file:
+      - .env
+      - ${SPARKD__CONFIG_DIR}/.env
+    ports:
+      - "${SPARKD__CONFIG_PORT}:${SPARKD__CONFIG_PORT}"
+      - "${SPARKD__QUICKBUS_PORT}:${SPARKD__QUICKBUS_PORT}"
+      - "${SPARKD__SIM_FIX_PORT}:${SPARKD__SIM_FIX_PORT}"
+    deploy:
+      restart_policy:
+        condition: any
+        delay: 3s
+      resources:
+        limits:
+          memory: "${SPARKD__MEMORY:-4gb}"
+
+  sparkd-tail:
+    image: bluetradingsystems/linux_dotnet7
+    volumes:
+      - type: volume
+        read_only: true
+        source: sparkd_data
+        target: /home/bts/sparkdata/data/logging.sparkd
+        volume:
+          subpath: logging.sparkd
+    command: ["tail", "-f", "/home/bts/sparkdata/data/logging.sparkd/log"]
+
+  sparkd-ping-test:
+    depends_on:
+      - sparkd
+    profiles:
+      - ping-test
+    build:
+      context: .
+      dockerfile_inline: |
+        FROM bluetradingsystems/linux_dotnet7
+        USER "root"
+        RUN apt-get update && apt-get -y install iputils-ping
+    command: [ "ping", "sparkd" ]
+
+volumes:
+  sparkd_data:
+    name: "sparkd_${SPARKD__CONFIG_PORT:-8000}"

doc/api/content/client/getting-started/services.md

@@ -124,8 +124,8 @@ service OrderService {
 ```
 ### C++ Example
 ```cpp
-#include "lib/api/public/ServiceContext.h"
-#include "lib/api/services/OrderService.client.pb.h"
+#include "spark/api/public/ServiceContext.h"
+#include "spark/api/services/OrderService.client.pb.h"
 using namespace bts::api;
 ServiceContext ctx;
 auto orders = ctx.locate<orderman::OrderService>();

doc/api/content/client/services/marketdata.md

@@ -29,9 +29,9 @@ namespace sample {
 ```
 
 ```cpp
-#include "lib/api/public/ServiceContext.h"
-#include "spark/lib/domain_model/Security.h"
-#include "lib/api/services/MarketDataService.client.pb.h"
+#include "spark/api/public/ServiceContext.h"
+#include "spark/domain_model/Security.h"
+#include "spark/api/services/MarketDataService.client.pb.h"
 #include <thread>
 
 using namespace bts::api;

docker/images/bluetradingsystems/linux_dotnet7/Dockerfile

@@ -0,0 +1,38 @@
+FROM ubuntu:22.04
+
+ARG DEBIAN_FRONTEND=noninteractive
+
+RUN apt-get update && apt-get install -y pkg-config
+RUN apt-get update && apt-get install -y locales
+RUN apt-get update && apt-get install -y ca-certificates
+RUN apt-get update && apt-get install -y libssh2-1-dev
+RUN apt-get update && apt-get install -y software-properties-common
+RUN apt-get update && apt-get install -y sudo
+
+# GCC
+ARG gcc_major=13
+RUN add-apt-repository ppa:ubuntu-toolchain-r/test \
+  && apt-get update \
+  && apt-get install -y g++-$gcc_major
+
+RUN apt-get update \
+  && apt-get install -y dotnet7
+
+RUN locale-gen en_US.UTF-8
+
+ARG docker_user="bts"
+ARG docker_uid="1000"
+ARG docker_pass="bts"
+
+RUN \
+  groupadd -g "$docker_uid" "$docker_user" \
+  && useradd \
+  -m \
+  -u "$docker_uid" \
+  -g "$docker_uid" \
+  -G sudo \
+  --password "$(openssl passwd -1 "$docker_pass")" \
+  "$docker_user"
+
+USER bts
+WORKDIR /home/bts

requirements.txt

@@ -1,22 +1,42 @@
 astroid==2.7.3
 autopep8==1.5.7
+blinker==1.9.0
+boto3==1.37.9
+botocore==1.37.9
 certifi==2023.5.7
 charset-normalizer==3.1.0
+click==8.1.8
 colorama==0.4.4
+Flask==3.1.0
 idna==3.4
 isort==5.9.3
+itsdangerous==2.2.0
+Jinja2==3.1.6
+jmespath==1.0.1
 lazy-object-proxy==1.6.0
+MarkupSafe==3.0.2
 mccabe==0.6.1
+numpy==2.2.6
 platformdirs==2.3.0
 protobuf==4.21.12
 pybars3==0.9.7
+pybind11==2.13.6
+pybind11-stubgen==2.5.3
+pybit==5.11.0
 pycodestyle==2.7.0
 pycparser==2.20
+pycryptodome==3.23.0
 pylint==2.10.2
 PyMeta3==0.5.1
+python-dateutil==2.9.0.post0
+python-dotenv==1.0.1
 requests==2.31.0
+s3transfer==0.11.4
+six==1.17.0
 termcolor==1.1.0
 toml==0.10.2
 urllib3==2.0.3
+websocket-client==1.8.0
+Werkzeug==3.1.3
 wrapt==1.12.1
 yapf==0.31.0

src/DataStreamSuite/ArithmeticExpressions/ArithmeticExpressions.csproj

@@ -1,7 +1,7 @@
 <Project Sdk="Microsoft.NET.Sdk">
 
   <PropertyGroup>
-    <TargetFramework>net7.0</TargetFramework>
+    <TargetFramework>net8.0</TargetFramework>
     <ImplicitUsings>enable</ImplicitUsings>
     <Nullable>enable</Nullable>
   </PropertyGroup>

src/DataStreamSuite/BaseUtilities/BaseUtilities.csproj

@@ -1,7 +1,7 @@
 <Project Sdk="Microsoft.NET.Sdk">
 
   <PropertyGroup>
-    <TargetFramework>net7.0</TargetFramework>
+    <TargetFramework>net8.0</TargetFramework>
     <ImplicitUsings>enable</ImplicitUsings>
     <Nullable>enable</Nullable>
   </PropertyGroup>

src/DataStreamSuite/BlueRegistry/BlueRegistry.csproj

@@ -1,7 +1,7 @@
 <Project Sdk="Microsoft.NET.Sdk">
 
   <PropertyGroup>
-    <TargetFramework>net7.0</TargetFramework>
+    <TargetFramework>net8.0</TargetFramework>
     <ImplicitUsings>enable</ImplicitUsings>
     <Nullable>enable</Nullable>
   </PropertyGroup>

src/DataStreamSuite/BooleanExpressions/BooleanExpressions.csproj

@@ -1,14 +1,14 @@
 <Project Sdk="Microsoft.NET.Sdk">
 
   <PropertyGroup>
-    <TargetFramework>net7.0</TargetFramework>
+    <TargetFramework>net8.0</TargetFramework>
     <ImplicitUsings>enable</ImplicitUsings>
     <Nullable>enable</Nullable>
   </PropertyGroup>
 
   <ItemGroup>
     <PackageReference Include="Google.Protobuf" Version="3.24.3" />
-    <PackageReference Include="YaccLexTools" Version="1.2.0" />
+    <PackageReference Include="YaccLexTools" Version="1.2.3" />
   </ItemGroup>
 
   <ItemGroup>

src/DataStreamSuite/BooleanExpressions/GplexBuffers.cs

@@ -24,8 +24,6 @@ namespace QUT.GplexBuffers
         public BufferException(string message) : base(message) { }
         public BufferException(string message, Exception innerException)
             : base(message, innerException) { }
-        protected BufferException(SerializationInfo info, StreamingContext context)
-            : base(info, context) { }
     }
 
     internal abstract class ScanBuff
@@ -60,12 +58,10 @@ namespace QUT.GplexBuffers
             return new BuildBuffer(source);
         }
 
-#if (!BYTEMODE)
         public static ScanBuff GetBuffer(Stream source, int fallbackCodePage)
         {
             return new BuildBuffer(source, fallbackCodePage);
         }
-#endif // !BYTEMODE
 #endif // !NOFILES
     }
 
@@ -353,17 +349,15 @@ namespace QUT.GplexBuffers
         {
             FileStream fStrm = (stream as FileStream);
             if (fStrm != null) FileName = fStrm.Name;
-            NextBlk = BlockReaderFactory.Raw(stream);
+            NextBlk = BlockReaderFactory.Get(stream);
         }
 
-#if (!BYTEMODE)
         public BuildBuffer(Stream stream, int fallbackCodePage)
         {
             FileStream fStrm = (stream as FileStream);
             if (fStrm != null) FileName = fStrm.Name;
             NextBlk = BlockReaderFactory.Get(stream, fallbackCodePage);
         }
-#endif
 
         /// <summary>
         /// Marks a conservative lower bound for the buffer,
@@ -434,43 +428,19 @@ namespace QUT.GplexBuffers
     //
     internal static class BlockReaderFactory
     {
-        public static BlockReader Raw(Stream stream)
-        {
-            return delegate(char[] block, int index, int number)
-            {
-                byte[] b = new byte[number];
-                int count = stream.Read(b, 0, number);
-                int i = 0;
-                int j = index;
-                for (; i < count; i++, j++)
-                    block[j] = (char)b[i];
-                return count;
-            };
-        }
 
-#if (!BYTEMODE)
-        public static BlockReader Get(Stream stream, int fallbackCodePage)
+        public static BlockReader Get(Stream stream, int fallbackCodePage = -2)
         {
             Encoding encoding;
             int preamble = Preamble(stream);
 
             if (preamble != 0)  // There is a valid BOM here!
                 encoding = Encoding.GetEncoding(preamble);
-            else if (fallbackCodePage == -1) // Fallback is "raw" bytes
-                return Raw(stream);
-            else if (fallbackCodePage != -2) // Anything but "guess"
+            else if (fallbackCodePage >= 0) // Anything but "guess"
                 encoding = Encoding.GetEncoding(fallbackCodePage);
             else // This is the "guess" option
-            {
-                int guess = new Guesser(stream).GuessCodePage();
-                stream.Seek(0, SeekOrigin.Begin);
-                if (guess == -1) // ==> this is a 7-bit file
-                    encoding = Encoding.ASCII;
-                else if (guess == 65001)
-                    encoding = Encoding.UTF8;
-                else             // ==> use the machine default
-                    encoding = Encoding.Default;
-            }
+                encoding = GuessEncoding(stream);
+
             StreamReader reader = new StreamReader(stream, encoding);
             return reader.Read;
         }
@@ -488,6 +458,14 @@ namespace QUT.GplexBuffers
             int b2 = stream.ReadByte();
             if (b0 == 0xef && b1 == 0xbb && b2 == 0xbf)
                 return 65001; // UTF8
+
+            int b3 = stream.ReadByte();
+            // Check for UTF-32 BOM (Byte Order Mark)
+            if (b0 == 0x00 && b1 == 0x00 && b2 == 0xFE && b3 == 0xFF)
+                return 12001; // UTF-32BE
+            if (b0 == 0xFF && b1 == 0xFE && b2 == 0x00 && b3 == 0x00)
+                return 12000; // UTF-32LE
+
             //
             // There is no unicode preamble, so we
             // return denoter for the machine default.
@@ -495,7 +473,150 @@ namespace QUT.GplexBuffers
             stream.Seek(0, SeekOrigin.Begin);
             return 0;
         }
-#endif // !BYTEMODE
+
+        private static Encoding GuessEncoding(Stream stream)
+        {
+            stream.Seek(0, SeekOrigin.Begin);
+            if (IsUtf16(stream, out Encoding encoding))
+            {
+                stream.Seek(0, SeekOrigin.Begin);
+                return encoding;
+            }
+
+            stream.Seek(0, SeekOrigin.Begin);
+            if (IsUtf8(stream))
+            {
+                stream.Seek(0, SeekOrigin.Begin);
+                return Encoding.UTF8;
+            }
+
+            stream.Seek(0, SeekOrigin.Begin);
+            if (IsAscii(stream))
+            {
+                stream.Seek(0, SeekOrigin.Begin);
+                return Encoding.ASCII;
+            }
+
+#if !NET462_OR_GREATER
+            stream.Seek(0, SeekOrigin.Begin);
+            if (IsLatin1(stream))
+            {
+                stream.Seek(0, SeekOrigin.Begin);
+                return Encoding.Latin1;
+            }
+#endif
+            stream.Seek(0, SeekOrigin.Begin);
+            return Encoding.Default;
+        }
+
+        private static bool IsUtf8(Stream stream)
+        {
+            int b;
+            while ((b = stream.ReadByte()) != -1)
+            {
+                if ((b & 0x80) == 0) // 0xxxxxxx
+                {
+                    continue;
+                }
+                else if ((b & 0xE0) == 0xC0) // 110xxxxx 10xxxxxx
+                {
+                    if (!IsValidUtf8ContinuationByte(stream)) return false;
+                }
+                else if ((b & 0xF0) == 0xE0) // 1110xxxx 10xxxxxx 10xxxxxx
+                {
+                    if (!IsValidUtf8ContinuationByte(stream) || !IsValidUtf8ContinuationByte(stream)) return false;
+                }
+                else if ((b & 0xF8) == 0xF0) // 11110xxx 10xxxxxx 10xxxxxx 10xxxxxx
+                {
+                    if (!IsValidUtf8ContinuationByte(stream) || !IsValidUtf8ContinuationByte(stream) || !IsValidUtf8ContinuationByte(stream)) return false;
+                }
+                else
+                {
+                    return false;
+                }
+            }
+            return true;
+        }
+
+        private static bool IsValidUtf8ContinuationByte(Stream stream)
+        {
+            int b = stream.ReadByte();
+            return b != -1 && (b & 0xC0) == 0x80; // 10xxxxxx
+        }
+
+        private static bool IsUtf16(Stream stream, out Encoding encoding)
+        {
+            encoding = Encoding.Default;
+
+            if (stream.Length >= 2)
+            {
+                // Read the first two bytes
+                int b0 = stream.ReadByte();
+                int b1 = stream.ReadByte();
+
+                // Check for UTF-16 BOM (Byte Order Mark)
+                if (b0 == 0xFF && b1 == 0xFE)
+                {
+                    encoding = Encoding.Unicode;
+                    return true; // UTF-16LE
+                }
+                if (b0 == 0xFE && b1 == 0xFF)
+                {
+                    encoding = Encoding.BigEndianUnicode;
+                    return true; // UTF-16BE
+                }
+
+                // Reset the stream position
+                stream.Seek(0, SeekOrigin.Begin);
+
+                // Check for common UTF-16 patterns
+                byte[] buffer = new byte[2];
+                while (stream.Read(buffer, 0, 2) == 2)
+                {
+                    if (buffer[0] == 0x00 && buffer[1] != 0x00)
+                    {
+                        encoding = Encoding.BigEndianUnicode;
+                        return true; // Likely UTF-16BE
+                    }
+                    if (buffer[0] != 0x00 && buffer[1] == 0x00)
+                    {
+                        encoding = Encoding.Unicode;
+                        return true; // Likely UTF-16LE
+                    }
+                }
+            }
+
+            return false;
+        }
+
+        private static bool IsAscii(Stream stream)
+        {
+            int b;
+            while ((b = stream.ReadByte()) != -1)
+            {
+                // Latin1 includes byte values from 0x00 to 0xFF
+                if (b > 0x7F)
+                {
+                    return false;
+                }
+            }
+            return true;
+        }
+
+        private static bool IsLatin1(Stream stream)
+        {
+            int b;
+            while ((b = stream.ReadByte()) != -1)
+            {
+                // Latin1 includes byte values from 0x00 to 0xFF
+                if (b > 0xFF)
+                {
+                    return false;
+                }
+            }
+            return true;
+        }
+
     }
 #endif // !NOFILES
     #endregion Buffer classes
@@ -540,180 +661,6 @@ namespace QUT.GplexBuffers
             return 0;
         }
     }
-#region guesser
-#if (!BYTEMODE)
-    // ==============================================================
-    // ============          Encoding Guesser           =============
-    // ==============================================================
-
-    /// <summary>
-    /// This class provides a simple finite state automaton that
-    /// scans the file looking for (1) valid UTF-8 byte patterns,
-    /// (2) bytes >= 0x80 which are not part of a UTF-8 sequence.
-    /// The method then guesses whether it is UTF-8 or maybe some 
-    /// local machine default encoding.  This works well for the
-    /// various Latin encodings.
-    /// </summary>
-    internal class Guesser
-    {
-        ScanBuff buffer;
-
-        public int GuessCodePage() { return Scan(); }
-
-        const int maxAccept = 10;
-        const int initial = 0;
-        const int eofNum = 0;
-        const int goStart = -1;
-        const int INITIAL = 0;
-        const int EndToken = 0;
-
-        #region user code
-        /* 
-         *  Reads the bytes of a file to determine if it is 
-         *  UTF-8 or a single-byte code page file.
-         */
-        public long utfX;
-        public long uppr;
-        #endregion user code
-
-        int state;
-        int currentStart = startState[0];
-        int code;
-
-        #region ScannerTables
-        static int[] startState = new int[] { 11, 0 };
-
-        #region CharacterMap
-        static sbyte[] map = new sbyte[256] {
-/*     '\0' */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 
-/*   '\x10' */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 
-/*   '\x20' */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 
-/*      '0' */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 
-/*      '@' */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 
-/*      'P' */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 
-/*      '`' */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 
-/*      'p' */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 
-/*   '\x80' */ 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 
-/*   '\x90' */ 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 
-/*   '\xA0' */ 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 
-/*   '\xB0' */ 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 
-/*   '\xC0' */ 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 
-/*   '\xD0' */ 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 
-/*   '\xE0' */ 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 
-/*   '\xF0' */ 4, 4, 4, 4, 4, 4, 4, 4, 5, 5, 5, 5, 5, 5, 5, 5 };
-        #endregion
-
-        static sbyte[][] nextState = new sbyte[][] {
-            new sbyte[] {0, 0, 0, 0, 0, 0},
-            new sbyte[] {-1, -1, 10, -1, -1, -1},
-            new sbyte[] {-1, -1, -1, -1, -1, -1},
-            new sbyte[] {-1, -1, 8, -1, -1, -1},
-            new sbyte[] {-1, -1, 5, -1, -1, -1},
-            new sbyte[] {-1, -1, 6, -1, -1, -1},
-            new sbyte[] {-1, -1, 7, -1, -1, -1},
-            null,
-            new sbyte[] {-1, -1, 9, -1, -1, -1},
-            null,
-            null,
-            new sbyte[] {-1, 1, 2, 3, 4, 2}
-        };
-
-
-        [SuppressMessage("Microsoft.Performance", "CA1810:InitializeReferenceTypeStaticFieldsInline")]
-        // Reason for suppression: cannot have self-reference in array initializer.
-        static Guesser()
-        {
-            nextState[7] = nextState[2];
-            nextState[9] = nextState[2];
-            nextState[10] = nextState[2];
-        }
-
-        int NextState()
-        {
-            if (code == ScanBuff.EndOfFile)
-                return eofNum;
-            else
-                return nextState[state][map[code]];
-        }
-        #endregion
-
-        public Guesser(System.IO.Stream file) { SetSource(file); }
-
-        public void SetSource(System.IO.Stream source)
-        {
-            this.buffer = new BuildBuffer(source);
-            code = buffer.Read();
-        }
-
-        int Scan()
-        {
-            for (; ; )
-            {
-                int next;
-                state = currentStart;
-                while ((next = NextState()) == goStart)
-                    code = buffer.Read();
-
-                state = next;
-                code = buffer.Read();
-
-                while ((next = NextState()) > eofNum)
-                {
-                    state = next;
-                    code = buffer.Read();
-                }
-                if (state <= maxAccept)
-                {
-                    #region ActionSwitch
-#pragma warning disable 162
-                    switch (state)
-                    {
-                        case eofNum:
-                            switch (currentStart)
-                            {
-                                case 11:
-                                    if (utfX == 0 && uppr == 0) return -1; /* raw ascii */
-                                    else if (uppr * 10 > utfX) return 0;   /* default code page */
-                                    else return 65001;                     /* UTF-8 encoding */
-                                    break;
-                            }
-                            return EndToken;
-                        case 1: // Recognized '{Upper128}',	Shortest string "\xC0"
-                        case 2: // Recognized '{Upper128}',	Shortest string "\x80"
-                        case 3: // Recognized '{Upper128}',	Shortest string "\xE0"
-                        case 4: // Recognized '{Upper128}',	Shortest string "\xF0"
-                            uppr++;
-                            break;
-                        case 5: // Recognized '{Utf8pfx4}{Utf8cont}',	Shortest string "\xF0\x80"
-                            uppr += 2;
-                            break;
-                        case 6: // Recognized '{Utf8pfx4}{Utf8cont}{2}',	Shortest string "\xF0\x80\x80"
-                            uppr += 3;
-                            break;
-                        case 7: // Recognized '{Utf8pfx4}{Utf8cont}{3}',	Shortest string "\xF0\x80\x80\x80"
-                            utfX += 3;
-                            break;
-                        case 8: // Recognized '{Utf8pfx3}{Utf8cont}',	Shortest string "\xE0\x80"
-                            uppr += 2;
-                            break;
-                        case 9: // Recognized '{Utf8pfx3}{Utf8cont}{2}',	Shortest string "\xE0\x80\x80"
-                            utfX += 2;
-                            break;
-                        case 10: // Recognized '{Utf8pfx2}{Utf8cont}',	Shortest string "\xC0\x80"
-                            utfX++;
-                            break;
-                        default:
-                            break;
-                    }
-#pragma warning restore 162
-                    #endregion
-                }
-            }
-        }
-    } // end class Guesser
-    
-#endif // !BYTEMODE
-#endregion
 #endif // !NOFILES
 
 // End of code copied from embedded resource

src/DataStreamSuite/Bts.ColumnExpressions/Bts.ColumnExpressions.csproj

@@ -1,7 +1,7 @@
 <Project Sdk="Microsoft.NET.Sdk">
 
   <PropertyGroup>
-    <TargetFramework>net7.0</TargetFramework>
+    <TargetFramework>net8.0</TargetFramework>
     <ImplicitUsings>enable</ImplicitUsings>
     <Nullable>enable</Nullable>
   </PropertyGroup>

src/DataStreamSuite/Bts.ColumnSortExpressions/Bts.ColumnSortExpressions.csproj

@@ -1,7 +1,7 @@
 <Project Sdk="Microsoft.NET.Sdk">
 
   <PropertyGroup>
-    <TargetFramework>net7.0</TargetFramework>
+    <TargetFramework>net8.0</TargetFramework>
     <ImplicitUsings>enable</ImplicitUsings>
     <Nullable>enable</Nullable>
   </PropertyGroup>

src/DataStreamSuite/ColumnMapExpressions/ColumnMapExpressions.csproj

@@ -1,7 +1,7 @@
 <Project Sdk="Microsoft.NET.Sdk">
 
   <PropertyGroup>
-    <TargetFramework>net7.0</TargetFramework>
+    <TargetFramework>net8.0</TargetFramework>
     <ImplicitUsings>enable</ImplicitUsings>
     <Nullable>enable</Nullable>
   </PropertyGroup>

src/DataStreamSuite/DataStream/BlueOrb/CountStream.cs

@@ -0,0 +1,49 @@
+using System;
+
+namespace BCG {
+   [DataStreamUserRespresentation("count")]
+   public class CountStream : ViewStream {
+      private DataPacket myLastFiredDataPacket;
+      private IBlueTableSchema mySchema = new BlueTableSchema("count",
+         new ColumnInfo("id", 0, typeof(int)),
+         new ColumnInfo("count", 1, typeof(int)));
+      private int myId = 0;
+      private ViewStream myVS;
+
+      public CountStream(ViewStream vs) {
+         myVS = vs;
+      }
+
+      public CountStream(string tableName)
+         : this(new TableStream(tableName)) { }
+
+      protected override void myOnFirstSubscriber() {
+         myVS.onChange += new ChangeDelegate(myHandleChanges);
+      }
+      protected override void myOnLastSubscriber() {
+         myVS.onChange -= new ChangeDelegate(myHandleChanges);
+      }
+
+
+      private void myHandleChanges(object newValue) {
+         myFireEvent(
+            myLastFiredDataPacket =
+               myLastFiredDataPacket == null ?
+                  new DataPacket(myGetRows(newValue, ++myId), myGetRows(newValue, myId),
+                  new BlueTable(mySchema))
+                  :
+                  new DataPacket(myGetRows(newValue, ++myId), myGetRows(newValue, myId),
+                  myLastFiredDataPacket.newRows)
+            );
+      }
+
+      private IBlueRowCollection myGetRows(object dp, int id) {
+         BlueTable retVal = new BlueTable(mySchema);
+         retVal.addRow
+            (new BlueRow(mySchema, id, (dp as DataPacket).newRows.Count));
+         return retVal;
+      }
+
+
+   }
+}

src/DataStreamSuite/DataStream/BlueOrb/DateTimeExpanderStream.cs

@@ -0,0 +1,38 @@
+namespace bts.BlueOrb {
+   using bts;
+
+   using BCG;
+
+   using System;
+   [DataStreamUserRespresentation("DateTimeExpanderStream")]
+   public class DateTimeExpanstionStream : ViewStream {
+      public DateTimeExpanstionStream(ViewStream parent_stream, string inp_col) {
+         Func<IBlueTableSchema, IBlueTableSchema> get_schema = (s) => {
+            IBlueTableSchema rv = s.add_column(System.TypeCode.DateTime, $"{inp_col}_date");
+            rv = rv.add_column(System.TypeCode.String, $"{inp_col}_day_of_week");
+            rv = rv.add_column(System.TypeCode.Int32, $"{inp_col}_day_of_month");
+            rv = rv.add_column(System.TypeCode.Int32, $"{inp_col}_month");
+            rv = rv.add_column(System.TypeCode.Int32, $"{inp_col}_hour");
+            rv = rv.add_column(System.TypeCode.Int32, $"{inp_col}_minute");
+            rv = rv.add_column(System.TypeCode.Int32, $"{inp_col}_seconds_of_day");
+            return rv;
+         };
+         ViewStream rs = new MapStream(parent_stream,
+         s => get_schema(s),
+         r => {
+            DateTime dt = r.get_datetime(inp_col);
+            IBlueTableSchema s = get_schema(r.schema);
+            MutableBlueRow rv = new MutableBlueRow(s, r);
+            rv.setValue($"{inp_col}_date", dt.Date);
+            rv.setValue($"{inp_col}_day_of_week", dt.DayOfWeek.ToString());
+            rv.setValue($"{inp_col}_day_of_month", dt.Day);
+            rv.setValue($"{inp_col}_month", dt.Date.Month);
+            rv.setValue($"{inp_col}_hour", dt.Hour);
+            rv.setValue($"{inp_col}_minute", dt.Minute);
+            rv.setValue($"{inp_col}_seconds_of_day", (dt - dt.Date).TotalSeconds);
+            return rv;
+         });
+         mySetRealSource(rs);
+      }
+   }
+}

src/DataStreamSuite/DataStream/BlueOrb/FilterStream.cs

@@ -0,0 +1,158 @@
+namespace bts.BlueOrb {
+   using BCG;
+   using System;
+   using System.Text.RegularExpressions;
+
+
+   public class RegExpFilter : IStreamFilter {
+      private Regex[] myRegExps;
+      private string[] myCols;
+
+      public RegExpFilter(string col, string regExp)
+         : this(new string[] { col }, new string[] { regExp }) { }
+
+      public RegExpFilter(string[] cols, string[] regExps) {
+         myCols = cols;
+         myRegExps = new Regex[regExps.Length];
+         for (int i = 0; i < regExps.Length; i++)
+            myRegExps[i] = new Regex(regExps[i]);
+      }
+
+      #region IStreamFilter Members
+      public bool filter(IBlueRow dr) {
+         bool rv = true;
+         for (int i = 0; i < myCols.Length; i++) {
+            rv &= myRegExps[i].IsMatch(dr[myCols[i]] is DBNull ? "" : dr[myCols[i]].ToString());
+            if (!rv)
+               return rv;
+         }
+         return rv;
+      }
+      #endregion
+   }
+
+   //[DataStreamUserReprsentation("FilterStream")]
+   public class FilterStream : ViewStream {
+      private BlueTable myLastFiredRows;
+      private IBlueRowCollection myLastReceivedRows;
+      private IStreamFilter myFilter;
+      private ViewStream myParentStream;
+      private string myFilterString;
+      private bool myUseNewFilter = false;// Configuration.get<DataStreamConfig>().UseNewFilter;
+
+      #region constructors
+      public delegate bool OnFilter(IBlueRow r);
+      private class OnFilterWrapper : IStreamFilter {
+         public OnFilterWrapper(OnFilter f) { this.f = f; }
+
+         public bool filter(IBlueRow dr) { return f(dr); }
+
+         private OnFilter f;
+      }
+      public FilterStream(ViewStream vs, OnFilter f)
+         : this(vs, new OnFilterWrapper(f)) { }
+
+
+      public FilterStream(ViewStream vs, IStreamFilter filter) {
+         {
+            myParentStream = vs;
+            myFilter = filter;
+            myFilterString = filter.ToString();
+            if (filter is IDynamicStreamFilter)
+               (filter as IDynamicStreamFilter).onFilterChange += new FilterChangeDelegate(myHandleFilterChange);
+         }
+      }
+
+      public FilterStream(ViewStream vs, string filterExpression)
+         : this(vs, new BooleanExpressionFilter(filterExpression)) {
+         myFilterString = filterExpression;
+      }
+
+      public FilterStream(ViewStream vs, string col, string regExp)
+         : this(vs, new RegExpFilter(col, regExp)) {
+         myFilterString = col + " , " + regExp;
+      }
+
+      public FilterStream(ViewStream vs)
+         : this(vs, new BinaryFilter(true)) {
+         myFilterString = true.ToString();
+      }
+      #endregion
+
+      #region overridden methods
+      protected override void myOnFirstSubscriber() { myParentStream.onChange += new ChangeDelegate(myHandleCallbacks); }
+
+      protected override void myOnLastSubscriber() {
+         myLastFiredRows = null;
+         myParentStream.onChange -= new ChangeDelegate(myHandleCallbacks);
+      }
+
+      protected override void myFireEvent(object o) {
+         DataPacket dp = o as DataPacket;
+         if (myLastValue == null || dp.addedRows.Count + dp.deletedRows.Count > 0)
+            base.myFireEvent(o);
+      }
+      #endregion
+
+      private void myHandleCallbacks(object newValue) {
+         DataPacket retVal;
+         DataPacket arrivedPacket = newValue as DataPacket;
+         myLastReceivedRows = arrivedPacket.newRows;
+         if (myLastFiredRows != null) {
+            retVal = new DataPacket(myLastFiredRows, new BlueTable(myLastFiredRows.schema),
+               new BlueTable(myLastFiredRows.schema));
+            foreach (IBlueRow r in arrivedPacket.deletedRows)
+               if (myLastFiredRows.containsID((int)r["id"])) {
+                  myLastFiredRows.deleteRow(r);
+                  (retVal.deletedRows as BlueTable).addRow(r);
+               }
+            foreach (IBlueRow r in arrivedPacket.addedRows)
+               if (myFilter.filter(r)) {
+                  myLastFiredRows.addRow(r);
+                  (retVal.addedRows as BlueTable).addRow(r);
+               }
+         } else {
+            myLastFiredRows = new BlueTable(arrivedPacket.schema);
+            retVal = new DataPacket(myLastFiredRows, myLastFiredRows, new BlueTable(myLastFiredRows.schema));
+            foreach (IBlueRow r in arrivedPacket.newRows)
+               if (myFilter.filter(r))
+                  myLastFiredRows.addRow(r);
+         }
+         myFireEvent(retVal);
+      }
+
+      public IStreamFilter filter {
+         set {
+            {
+               myFilter = value;
+               if (numberOfSubscribers > 0) {
+                  if (myLastReceivedRows != null) {
+                     BlueTable nRows = new BlueTable(myLastReceivedRows.schema);
+                     foreach (IBlueRow r in myLastReceivedRows)
+                        if (myFilter.filter(r))
+                           nRows.addRow(r);
+                     IBlueRowCollection dRows = new BlueTable(nRows.schema),
+                        aRows = new BlueTable(nRows.schema);
+                     BlueDataUtility.computeDifferences(nRows, myLastFiredRows, out dRows, out aRows);
+                     myLastFiredRows = nRows;
+                     myFireEvent(new DataPacket(nRows, aRows, dRows));
+                  }
+               }
+            }
+         }
+         get {            
+               return myFilter;
+         }
+      }
+
+      //public string filterString { set { filter = new BooleanExpressionFilter(value); } }
+
+      public override string ToString() {
+         return GetType().ToString() + " " + myFilterString;
+      }
+
+      private void myHandleFilterChange() {
+         filter = myFilter;
+      }
+   }
+}

src/DataStreamSuite/DataStream/BlueOrb/GroupByStream.cs

@@ -0,0 +1,121 @@
+using System;
+using System.Collections.Generic;
+using System.Diagnostics;
+using System.Linq;
+using System.Text.Json;
+
+using bts;
+//using bts.SilverPine;
+using BCG;
+
+namespace bts.BlueOrb {
+   [DataStreamUserRespresentation("GroupByStream")]
+   public class GroupByStream : ViewStream {
+
+      public interface INonIncGroupByEvaluator {
+         // grouped_rows
+         IBlueRowCollection evaluate(IDictionary<int, List<IBlueRow>> grouped_rows);
+         string[] get_group_by_cols();
+      }
+
+      private class DelNonIncGroupByEvaluator : INonIncGroupByEvaluator {
+         Func<string[]> _groupby_cols_func;
+         Func<List<IBlueRow>, List<IBlueRow>> _evaluate_func;
+         public DelNonIncGroupByEvaluator(Func<string[]> groupby_cols_func,
+                                          Func<List<IBlueRow>, List<IBlueRow>> evaluate_func) {
+            _groupby_cols_func = groupby_cols_func;
+            _evaluate_func = evaluate_func;
+         }
+         public IBlueRowCollection evaluate(IDictionary<int, List<IBlueRow>> grouped_rows) {
+            BlueTable rv = null;
+            foreach (var kv in grouped_rows) {
+               List<IBlueRow> g_rows = kv.Value;
+               List<IBlueRow> out_rows = _evaluate_func.Invoke(g_rows);
+               if (out_rows != null && out_rows.Count > 0) {
+                  rv = rv ?? new BlueTable(out_rows.ElementAt(0).schema);
+               }
+               rv.addRows(out_rows);
+            }
+            return rv;
+         }
+         public string[] get_group_by_cols() {
+            return _groupby_cols_func();
+         }
+
+      }
+      private ViewStream _main_stream;
+      private INonIncGroupByEvaluator _groupby_func;
+      private BlueRowHasher _row_hasher;
+      private string[] _groupby_cols;
+      private Stopwatch _timer = new();
+      private int _n_total_fires = 0, _n_expensive_fires = 0;
+
+      public GroupByStream(ViewStream main_stream, INonIncGroupByEvaluator groupby_func) {
+         _main_stream = main_stream;
+         _groupby_func = groupby_func;
+         _groupby_cols = _groupby_func.get_group_by_cols();
+         _row_hasher = new BlueRowHasher(_groupby_cols == null ? 0 : _groupby_cols.Length);
+      }
+
+      public GroupByStream(ViewStream main_stream, Func<string[]> groupby_cols_func,
+                                 Func<List<IBlueRow>, List<IBlueRow>> evaluate_func)
+                                 : this(main_stream, new DelNonIncGroupByEvaluator(groupby_cols_func, evaluate_func)) { }
+
+      public GroupByStream(ViewStream parent_str, string groupby_col, string standard_group_by_text)
+      : this(parent_str, groupby_col, standard_group_by_text, String.Empty) { }
+
+      // this is the main constructor that does everything
+      public GroupByStream(ViewStream parent_str, string groupby_col, string standard_group_by_ops, string optional_sort_col) {
+         MultiRowOpHelper std_group_by = MultiRowOpHelper.parse_ops(standard_group_by_ops);
+         Func<string[]> groupby_cols_func = () => {
+            return String.IsNullOrEmpty(groupby_col) ? null : new string[] { groupby_col };
+         };
+         Func<List<IBlueRow>, List<IBlueRow>> evaluate_func = (group_rows) => {
+            if (!String.IsNullOrEmpty(optional_sort_col))
+               group_rows.Sort((r1, r2) => BlueDataUtility.compare_rows(r2, r1, optional_sort_col));
+            // this si where the group by aggregation happens
+            IBlueRow rv = MultiRowOpHelper.group_by_compute(group_rows, std_group_by);
+            return new IBlueRow[] { rv }.ToList();
+         };
+         ViewStream rs = new GroupByStream(parent_str, groupby_cols_func, evaluate_func);
+         mySetRealSource(rs);
+      }
+      public GroupByStream(ViewStream parent_str, string standard_group_by_text) :
+      this(parent_str, String.Empty, standard_group_by_text) {
+      }
+
+
+      protected override void myOnFirstSubscriber() { _main_stream.onChange += _main_stream_onchange; }
+
+      protected override void myOnLastSubscriber() { _main_stream.onChange -= _main_stream_onchange; }
+
+      private void _main_stream_onchange(object nv) {
+         DataPacket dp = nv as DataPacket;
+         ///grouping together
+         _timer.Restart();
+         IDictionary<int, List<IBlueRow>> grouped_rows = (_groupby_cols == null || _groupby_cols.Length == 0) ?
+         new Dictionary<int, List<IBlueRow>>() { { 0, dp.newRows.to_list() } } :
+         BlueDataUtility.group_together(dp.newRows, _groupby_cols, ref _row_hasher);
+         _timer.Stop();
+         var groupElapsedMs = _timer.ElapsedMilliseconds;
+         if (groupElapsedMs >= 50) {
+            Log.DataFlow.info($"Time to group {dp.newRows.Count} rows into {grouped_rows.Count} groups: {_timer.ElapsedMilliseconds}ms");
+         }
+         // Log.log(bts.Log.Level.Info, bts.log.Subsystem.gen, String.Format("Time to group {0} rows into {1} groups = {2}", dp.newRows.Count, grouped_rows.Count, _timer.lastIntervalTime));
+         /// reducing
+         _timer.Restart();
+         IBlueRowCollection total_state = _groupby_func.evaluate(grouped_rows);
+         _timer.Stop();
+         var applyElapsedMs = _timer.ElapsedMilliseconds;
+         _n_total_fires++;
+         if (applyElapsedMs >= 50) {
+            _n_expensive_fires++;
+            Log.DataFlow.info(
+               $"Time to evaluate {grouped_rows.Count} groups {_timer.ElapsedMilliseconds}ms, ratio {_n_expensive_fires}/{_n_total_fires}");
+         }
+         // TODO,
+         if (total_state != null)
+            myFireEvent(new DataPacket(total_state, total_state, myLastValue == null ? new BlueTable(total_state.schema) : (myLastValue as DataPacket).newRows));
+      }
+   }
+}

src/DataStreamSuite/DataStream/BlueOrb/Join/INonIncJoinInlineMapper.cs

@@ -0,0 +1,17 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+using BCG;
+
+namespace bts.BlueOrb {   
+   public interface INonIncJoinInlineMapper
+   {      
+      IBlueTableSchema get_output_schema(IBlueTableSchema schema);
+
+      IBlueRowCollection map_rows(IBlueRowCollection rows);
+      
+      event MapperChangeDelegate onMapperChange;
+   }
+}

src/DataStreamSuite/DataStream/BlueOrb/Join/NonIncJoinBookkeeper.cs

@@ -0,0 +1,152 @@
+using bts;
+using bts.collections;
+
+using mbts;
+
+using System;
+using System.Collections.Generic;
+using System.Diagnostics;
+using System.Linq;
+using System.Security.Cryptography;
+using BCG;
+
+namespace bts.BlueOrb {   
+   internal class NonIncJoinBookkeeper : IDisposable {
+      public Action on_new_data;
+      private readonly NonIncJoinSpec _join_spec;
+      private readonly int _join_position;
+      private readonly int _start_index_in_output;
+      private readonly BlueRowHasher _row_hasher;
+      private readonly bool _is_left_most;
+      private readonly bool _is_tag_along;
+      private int _n_total_fires = 0, _n_expensive_fires = 0;
+
+      private IBlueTableSchema _schema = null;
+      private IDictionary<int, IBlueRow> _row_map = new Dictionary<int, IBlueRow>();
+      private List<Pair<int, IBlueRow>> _row_list = new List<Pair<int, IBlueRow>>();
+      private IBlueRow _solitary_row = null;
+
+      private int[] _join_col_indices = null,
+                     _data_col_indices = null;
+
+      public NonIncJoinBookkeeper(NonIncJoinSpec the_join_spec, int join_position, int start_index_in_output, BlueRowHasher row_hasher) {
+         _join_spec = the_join_spec;
+         _join_position = join_position;
+         _start_index_in_output = start_index_in_output;
+         _row_hasher = row_hasher;
+         _is_left_most = _join_position == 0;
+         _is_tag_along = _join_spec.join_col_names.Length == 0;
+      }
+
+      private bool _filled_missing_data = false;
+      private void _fill_missing_data(DataPacket dp) {
+         if (_filled_missing_data)
+            return;
+         Func<string[], bool> null_or_empty = (str_arr) => {
+            if (str_arr == null || str_arr.Length == 0)
+               return true;
+            foreach (string s in str_arr) {
+               if (!String.IsNullOrEmpty(s)) {
+                  return false;
+               }
+            }
+            return true;
+         };
+         Func<string[], string[]> remove_bad_cols = (col_names) => {
+            HashSet<string> schema_col_map = new HashSet<string>(dp.schema.columnNames);
+            List<string> cols_list = new List<string>(col_names);
+            cols_list.RemoveWhere(c => !schema_col_map.Contains(c));
+            return cols_list.ToArray();
+         };
+         if (null_or_empty(_join_spec.data_col_names)) {
+            _join_spec.data_col_names = dp.schema.columnNames;
+         }
+         _join_spec.data_col_names = remove_bad_cols(_join_spec.data_col_names);
+         if (null_or_empty(_join_spec.data_col_names)) {
+            _join_spec.data_col_names = dp.schema.columnNames;
+         }
+         if (null_or_empty(_join_spec.projected_col_names)) {
+            _join_spec.projected_col_names = _join_spec.data_col_names;
+         }
+      }
+
+      public void init() {
+         _join_spec.stream.onChange += _on_stream_cb;
+      }
+
+      public void _on_stream_cb(object dp_obj) {
+         //Log.Gen.info($"NonIncJoinBookkeeper._on_stream_cb dp_obj: {dp_obj}");
+         var timer = Stopwatch.StartNew();
+         DataPacket dp = (DataPacket)dp_obj;
+         _fill_missing_data(dp);
+         if (_schema == null) {
+            _schema = dp.schema;
+            _join_col_indices = BlueDataUtility.get_column_indices(_schema, _join_spec.join_col_names);
+            _data_col_indices = BlueDataUtility.get_column_indices(_schema, _join_spec.data_col_names);
+         }
+
+         if (_is_left_most) {
+            _row_list.Clear();
+            foreach (IBlueRow r in dp.newRows)
+               _row_list.Add(new Pair<int, IBlueRow>(_row_hasher.get_hash(r, _join_col_indices), r));
+         } else {
+            if (_is_tag_along)
+               _solitary_row = BlueDataUtility.extractRow(dp.newRows, 0);
+            else {
+               _row_map.Clear();
+               foreach (IBlueRow r in dp.newRows)
+                  _row_map[_row_hasher.get_hash(r, _join_col_indices)] = r;
+            }
+         }
+         timer.Stop();
+         _n_total_fires++;
+         if (timer.ElapsedMilliseconds >= 50) {
+            _n_expensive_fires++;
+            Log.Gen.info($"NonIncJoinBookkeepingData._on_stream_cb {dp.newRows.Count} rows {_join_col_indices.Length} "
+               + $"key cols {_join_position} join pos time {timer.Elapsed} ratio {_n_expensive_fires}/{_n_total_fires}");
+         }
+         if (on_new_data != null)
+            on_new_data();
+      }
+
+      public bool is_ready { get { return _schema != null; } }
+
+      public void constribute_cols_to_schema(List<ColumnInfo> cis) {
+         for (int i = 0; i < _join_spec.data_col_names.Length; i++)
+            cis.Add(new ColumnInfo(_join_spec.projected_col_names[i], cis.Count, _schema[_join_spec.data_col_names[i]].colType));
+      }
+
+      public List<Pair<int, IBlueRow>> get_list() { return _row_list; }
+
+      public void copy_master_row_to_output_row(int row_index, object[] output_row) {
+         IBlueRow this_row = _row_list[row_index].second;
+         BlueDataUtility.copy_to_row(this_row, _data_col_indices, output_row, _start_index_in_output);
+      }
+
+      public void copy_matching_row_to_output_row(int row_hash, object[] output_row) {
+         if (_is_tag_along) {
+            if (_solitary_row != null)
+               BlueDataUtility.copy_to_row(_solitary_row, _data_col_indices, output_row, _start_index_in_output);
+            else
+               copy_dbnull_to_output_row(output_row);
+         } else {
+            IBlueRow this_row = null;
+            if (_row_map.TryGetValue(row_hash, out this_row))
+               BlueDataUtility.copy_to_row(this_row, _data_col_indices, output_row, _start_index_in_output);
+            else
+               copy_dbnull_to_output_row(output_row);
+         }
+      }
+
+      private void copy_dbnull_to_output_row(object[] output_row) {
+         for (int i = 0; i < _data_col_indices.Length; i++)
+            output_row[i + _start_index_in_output] = DBNull.Value;
+      }
+
+      #region IDisposable
+
+      public void Dispose() { _join_spec.stream.onChange -= _on_stream_cb; }
+
+      #endregion
+   }
+}

src/DataStreamSuite/DataStream/BlueOrb/Join/NonIncJoinSpec.cs

@@ -0,0 +1,67 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+
+using BCG;
+
+using mbts;
+
+namespace bts.BlueOrb {   
+   // this a collection of spec, theres no joining logic in here
+   public class NonIncJoinSpec {
+      public ViewStream stream;
+      // join col names are the names on which the streams are joined (for equality),
+      // data col names are the names in the original stream
+      // projected col names are their new names, ie if its x:l_x, then x is in the 
+      // data_col_names and l_x in projected cols
+      public string[] join_col_names, data_col_names, projected_col_names;
+
+      public NonIncJoinSpec() { }
+
+      public NonIncJoinSpec(ViewStream the_stream, string[] the_join_col_names, string[] the_data_col_names)
+         : this(the_stream, the_join_col_names, the_data_col_names, the_data_col_names) { }
+
+      public NonIncJoinSpec(ViewStream the_stream, string[] the_join_col_names, string[] the_data_col_names, string[] the_projected_col_names) {
+         stream = the_stream;
+         join_col_names = the_join_col_names;
+         data_col_names = the_data_col_names;
+         projected_col_names = the_projected_col_names;
+      }
+
+      public static NonIncJoinSpec[] from_triplet(params Triplet<ViewStream, string, string>[] join_specs) {
+         List<NonIncJoinSpec> rv = new List<NonIncJoinSpec>();
+         foreach (Triplet<ViewStream, string, string> r in join_specs) {
+            string[] keys = String.IsNullOrEmpty(r.second) ? new string[] { } : r.second.Split(' ');
+            string[] visible_cols = r.third.Split(' ');
+            List<string> data_col_names = new List<string>(), mapped_col_names = new List<string>();
+            foreach (string s in visible_cols) {
+               string[] toks = s.Split(':');
+               if (toks.Length == 1) {
+                  data_col_names.Add(toks[0]);
+                  mapped_col_names.Add(toks[0]);
+               } else if (toks.Length == 2) {
+                  data_col_names.Add(toks[0]);
+                  mapped_col_names.Add(toks[1]);
+               }
+            }
+            rv.Add(new NonIncJoinSpec(r.first, keys, data_col_names.ToArray(), mapped_col_names.ToArray()));
+         }
+         return rv.ToArray();
+      }
+
+      public override string ToString() {
+         Func<string[], string> arr_txt = (a) => {
+            string rv = "";
+            foreach (string s in a) {
+               rv += "{s} ";
+            }
+            rv = rv.Trim();
+            return rv;
+         };
+         string rv = $"stream: {stream} join_col_names: {arr_txt(join_col_names)} data_col_names: {arr_txt(data_col_names)} projected_col_names: {arr_txt(projected_col_names)}";
+         return rv;
+      }
+   }
+}

src/DataStreamSuite/DataStream/BlueOrb/Join/NonIncJoinStream.cs

@@ -0,0 +1,165 @@
+using bts;
+
+using mbts;
+
+using System;
+using System.Collections.Generic;
+using System.Diagnostics;
+using BCG;
+
+namespace bts.BlueOrb {   
+   [DataStreamUserRespresentation("JoinStream")]
+   public class NonIncJoinStream : ViewStream, BCG.Utilities.General.IWorker {
+      private readonly NonIncJoinSpec[] _join_specs;
+      private readonly INonIncJoinInlineMapper _output_mapper;
+      private readonly BlueRowHasher _row_hasher;
+      private readonly TimeSpan _throttle_period;
+      private readonly ViewStreamDatabase _vsd;
+
+      private List<NonIncJoinBookkeeper> _book_keeping_data = new List<NonIncJoinBookkeeper>();
+      private IBlueTableSchema _output_schema = null;
+      private DateTime _last_fire_time = DateTime.MinValue;
+      private bool _is_dirty = false;
+
+      public NonIncJoinStream(ViewStream left, string left_join_cols, string left_proj,
+                              ViewStream right, string right_join_cols, string right_proj)
+         : this(new Triplet<ViewStream, string, string>[]
+               {
+                  new Triplet<ViewStream, string, string> (left, left_join_cols, left_proj),
+                  new Triplet<ViewStream, string, string> (right, right_join_cols, right_proj)
+               }) { }
+
+      public NonIncJoinStream(params Triplet<ViewStream, string, string>[] join_specs)
+         : this(NonIncJoinSpec.from_triplet(join_specs), null, 0) { }
+
+      public NonIncJoinStream(ViewStreamDatabase vsd, INonIncJoinInlineMapper output_mapper, int throttles_ms,
+                              params Triplet<ViewStream, string, string>[] join_specs)
+         : this(vsd, NonIncJoinSpec.from_triplet(join_specs), output_mapper, throttles_ms) { 
+      }
+
+      public NonIncJoinStream(NonIncJoinSpec[] join_specs, INonIncJoinInlineMapper output_mapper, int throttles_ms = 0)
+         : this(ViewStreamDatabase.instance, join_specs, output_mapper, throttles_ms) { }
+
+      public NonIncJoinStream(ViewStreamDatabase vsd, NonIncJoinSpec[] join_specs, INonIncJoinInlineMapper output_mapper, int throttles_ms = 0) {
+         _join_specs = join_specs;
+         _output_mapper = output_mapper;
+         _row_hasher = new BlueRowHasher(_join_specs[0].join_col_names.Length);
+         _throttle_period = throttles_ms == 0 ? TimeSpan.Zero : new TimeSpan(0, 0, 0, 0, throttles_ms);
+         _vsd = vsd;
+      }
+
+      protected override void myOnFirstSubscriber() {
+         int data_col_no = 1;
+         for (int jp = 0; jp < _join_specs.Length; jp++) {
+            NonIncJoinBookkeeper s = new NonIncJoinBookkeeper(_join_specs[jp], jp, data_col_no, _row_hasher);
+            //Log.Gen.info($"NonIncJoinStream.myOnFirstSubscriber jp: {jp} spec: {_join_specs[jp]}");
+            data_col_no += _join_specs[jp].data_col_names.Length;
+            s.on_new_data += _stream_callback;
+            _book_keeping_data.Add(s);
+         }
+         if (_output_mapper != null)
+            _output_mapper.onMapperChange += _on_mapper_change;
+         _vsd.addWorker(this);
+         foreach (var bkd in _book_keeping_data)
+            bkd.init();
+      }
+
+      protected override void myOnLastSubscriber() {
+         foreach (NonIncJoinBookkeeper d in _book_keeping_data)
+            d.Dispose();
+         if (_output_mapper != null)
+            _output_mapper.onMapperChange -= _on_mapper_change;
+         _vsd.removeWorker(this);
+      }
+
+      private void _stream_callback() {
+         //Log.Gen.info("NonIncJoinStream._stream_callback received callback");
+         _is_dirty = true;
+         doWork();
+      }
+
+      private void _on_mapper_change() {
+         _is_dirty = true;
+         doWork();
+      }
+
+      private bool _is_ready() {
+         bool rv = true;
+         foreach (NonIncJoinBookkeeper d in _book_keeping_data)
+            rv &= d.is_ready;
+         return rv;
+      }
+
+      private void _construct_schema() {
+         if (_output_schema == null) {
+            List<ColumnInfo> cis = new List<ColumnInfo>(new ColumnInfo[] { new ColumnInfo("id", 0, typeof(int)) });
+            foreach (NonIncJoinBookkeeper d in _book_keeping_data)
+               d.constribute_cols_to_schema(cis);
+            _output_schema = new BlueTableSchema("NonIncJoinStream", cis.ToArray());
+         }
+      }
+
+      private int _n_total_fires = 0, _n_expensive_fires = 0;
+      private void _form_and_fire() {
+         //Log.Gen.info($"NonIncJoinStream._form_and_fire _is_ready(): {_is_ready()}");
+         if (_is_ready()) {
+            _construct_schema();
+            var timer = Stopwatch.StartNew();
+            IList<Pair<int, IBlueRow>> left_row_list = _book_keeping_data[0].get_list();
+            List<object[]> output_rows = new List<object[]>();
+
+            for (int i = 0; i < left_row_list.Count; i++) {
+               object[] or = new object[_output_schema.columnInfo.Length];
+               or[0] = i;
+               output_rows.Add(or);
+            }
+
+            for (int ji = 0; ji < _book_keeping_data.Count; ji++) {
+               NonIncJoinBookkeeper bk_data = _book_keeping_data[ji];
+               for (int i = 0; i < left_row_list.Count; i++) {
+                  if (ji == 0)
+                     bk_data.copy_master_row_to_output_row(i, output_rows[i]);
+                  else
+                     bk_data.copy_matching_row_to_output_row(left_row_list[i].first, output_rows[i]);
+               }
+            }
+            IBlueRowCollection dp_output_rows = _apply_map(new SequentialIDBlueTable(_output_schema, output_rows));
+            timer.Stop();
+            _n_total_fires++;
+            //if (timer.ElapsedMilliseconds >= 250) {
+            _n_expensive_fires++;
+            //string log_msg = $"Join and map time for {_book_keeping_data.Count} "
+            //   + $"tables {timer.Elapsed.TotalMilliseconds} msecs, ratio {_n_expensive_fires}/{_n_total_fires}";
+            //Log.Gen.info(log_msg);
+            //}
+            var dp_out = BlueDataUtility.build_dp_inc(dp_output_rows, myLastValue as DataPacket, true);
+            myFireEvent(dp_out);
+            //myFireEvent(new DataPacket(dp_output_rows, dp_output_rows, myLastValue == null ? new BlueTable(_output_schema) : (myLastValue as DataPacket).newRows));
+         }
+      }
+
+      private IBlueRowCollection _apply_map(IBlueRowCollection unmapped_rows) {
+         if (_output_mapper != null) {
+            IBlueRowCollection output_rows;
+            var timer = Stopwatch.StartNew();
+            output_rows = _output_mapper.map_rows(unmapped_rows);
+            timer.Stop();
+            if (timer.ElapsedMilliseconds >= 50) {
+               Log.Gen.info($"Time to map {unmapped_rows.Count} rows into {output_rows.Count} rows {timer.Elapsed}");
+            }
+            return output_rows;
+         } else {
+            return unmapped_rows;
+         }
+      }
+
+      public void doWork() {
+         //Log.Gen.info($"NonIncJoinStream.doWork _last_fire_time: {_last_fire_time} _is_dirty: {_is_dirty}");
+         if (DateTime.Now - _last_fire_time >= _throttle_period && _is_dirty) {
+            _is_dirty = false;
+            _last_fire_time = DateTime.Now;
+            _form_and_fire();
+         }
+      }
+   }
+}

src/DataStreamSuite/DataStream/BlueOrb/JsonSaverStream.cs

@@ -0,0 +1,28 @@
+//namespace bts.Blueorb {
+//   using BCG;
+
+//   public class JsonSaverStream : ViewStream {
+//      private ViewStream _stream;
+//      private string _file_name;
+//      public JsonSaverStream(ViewStream vs, string file_name) {
+//         _stream = vs;
+//         _file_name = file_name;
+//      }
+
+//      protected override void myOnFirstSubscriber() {
+//         _stream.onChange += _on_data;
+//      }
+
+//      private void _on_data(object dp_obj) {
+//         if (dp_obj == null) {
+//            return;
+//         }
+//         DataPacket dp = dp_obj as DataPacket;
+//         dp.newRows.write_to_json_file(dp.newRows, _file_name);
+//      }
+
+//      protected override void myOnLastSubscriber() {
+//         _stream.onChange -= _on_data;
+//      }
+//   }
+//}

src/DataStreamSuite/DataStream/BlueOrb/LiteralStream.cs

@@ -0,0 +1,91 @@
+using System;
+using System.Collections;
+using System.Collections.Generic;
+
+using Amazon.Runtime.Internal.Util;
+
+using BCG;
+using BCG.Utilities;
+
+using bts;
+
+namespace bts.BlueOrb{
+   public class LiteralStream : ViewStream {
+
+      public static List<Tuple<ColumnInfo, object>> parse_columns(string key_value_pairs) {
+         List<Tuple<ColumnInfo, object>> rv = new List<Tuple<ColumnInfo, object>>();
+         string[] pairs = key_value_pairs.Split(' ');
+         foreach (string p in pairs) {
+            string[] kvp = p.Split(':');
+            string col_name = kvp[0];
+            string val = kvp[1];
+            var tup = BlueDataUtility.guess_type(val);
+            var ci = new ColumnInfo(col_name, rv.Count, tup.Item1);
+            rv.Add(Tuple.Create(ci, tup.Item2));
+         }
+         return rv;
+      }
+
+      //literal("table1", "x:123, y:1.001, z:abc;x:123, y:1.001, z:abc")
+      // col_name: col_value ... ; col_name: col_value
+      public LiteralStream(string table_name, string rows) {
+         Func<string, int, IBlueRow> get_row = (json_row, id) => {
+            Dictionary<string, object> map = new Dictionary<string, object>();
+            string[] toks = json_row.Split(',', StringSplitOptions.TrimEntries);
+            //Logger.assert(col_vals.Length == 2);
+            foreach (string t in toks) {
+               var col_vals = t.Split(':', StringSplitOptions.TrimEntries);
+               string col = col_vals[0];
+               string val = col_vals[1];
+               map.Add(col, val);
+            }
+            map["id"] = id;
+            return new BlueRow(map);
+         };
+         string[] row_arr = rows.Split(';', StringSplitOptions.TrimEntries);
+         BlueTable table = null;
+         foreach (string json_string in row_arr) {
+            IBlueRow r = //BlueDataUtility.convert_json_to_row(json_string, table == null ? 0 : table.Count);
+            get_row(json_string, table == null ? 0 : table.Count);
+            table = table ?? new BlueTable(r.schema);
+            table.addRow(r);
+         }
+         ViewStream str = new BlueTableStream(table);
+         str = new SchemaGuesserStream(str);
+         mySetRealSource(str);
+      }
+
+      public LiteralStream(string tableName, string schema, string value) {
+         string[] columns = schema.Split(' ');
+
+
+         ColumnInfo[] ci = new ColumnInfo[columns.Length + 1];
+         ci[0] = new ColumnInfo("id", 0, typeof(int));
+         int n = 1;
+         foreach (string c in columns) {
+            string[] columnAndType = c.Split('/');
+            if (columnAndType.Length == 1)
+               ci[n] = new ColumnInfo(c, n, typeof(double));
+            else
+               ci[n] = new ColumnInfo(columnAndType[0], n, Type.GetType("System." + columnAndType[1], true));
+            n++;
+         }
+
+
+         BlueTable t = new BlueTable(new BlueTableSchema(tableName, ci));
+         string[] rows = value.Split(';');
+         n = 0;
+         foreach (string r in rows)
+            try {
+               if (r == string.Empty)
+                  continue;
+               t.addRow(
+                  BlueDataUtility.createRowFromCSVFileRow(r, t.schema, ++n));
+            } catch (Exception e) {
+               throw (new Exception("Error occured at line: " + n.ToString() + " " + e.Message));
+            }
+
+         mySetRealSource(new BlueRowCollectionStaticStream(t));
+      }
+   }
+}

src/DataStreamSuite/DataStream/BlueOrb/MillisecondStream.cs

@@ -0,0 +1,89 @@
+using System;
+
+using BCG.Utilities.General;
+using BCG;
+using bts;
+
+namespace bts.BlueOrb {
+   [DataStreamUserRespresentation("MillisecondStream")]
+   public class MillisecondStream : ViewStream, IWorker {
+      private DateTime _last_fire_time = DateTime.MinValue;
+      private readonly TimeSpan _period;
+      private readonly int _size;
+      private readonly ViewStreamDatabase _vsd;
+      private IBlueTableSchema _schema;
+      private bts.CircularBufferTable _table;
+
+      public MillisecondStream(int interval_msecs, int size)
+      : this(ViewStreamDatabase.instance, interval_msecs, size) {
+      }
+
+      public MillisecondStream(ViewStreamDatabase vsd, int interval_msecs, int size) {
+         _vsd = vsd;
+         _period = TimeSpan.FromMilliseconds(interval_msecs);
+         _size = size;
+         _schema = BlueDataUtility.form_schema("Milliseconds",
+               Tuple.Create("id", TypeCode.Int32),
+               Tuple.Create("time", TypeCode.DateTime),
+               Tuple.Create("display_time", TypeCode.String));
+         _table = new CircularBufferTable(_schema, _size);
+      }
+
+      protected override void myOnFirstSubscriber() {
+         _vsd.addWorker(this);
+      }
+
+      protected override void myOnLastSubscriber() {
+         _vsd.removeWorker(this);
+      }
+
+      private int _id = 0;
+      public void doWork() {
+         if (DateTime.Now - _last_fire_time >= _period) {
+            _last_fire_time = DateTime.Now;
+            IBlueRow nr = new BlueRow(_schema, _id++, _last_fire_time,
+            _last_fire_time.ToString("yyyy-MM-dd HH:mm:ss.fff"));
+            _table.add_row(nr);
+            var dp = BlueDataUtility.build_dp_inc(_table, myLastValue as DataPacket);
+            myFireEvent(dp);
+         }
+      }
+   }
+   public class MSecondStream : TimeUnitStream {
+      public MSecondStream(int intervalInms, int size)
+         : base(DateTime.Now, new TimeSpan(0, 0, 0, 0, intervalInms), size, ViewStreamDatabase.instance) { }
+
+      protected override DateTime myRoundTime(DateTime dt) { return dt; }
+   }
+
+   public class DisplayableMillisecondStream : ViewStream, IStreamMapper {
+      private IBlueTableSchema mySchema = new BlueTableSchema("ms",
+                                                               new ColumnInfo("id", 0, typeof(int)),
+                                                               new ColumnInfo("time", 1, typeof(string)));
+      public DisplayableMillisecondStream(int intervalInMs, int sz) {
+         mySetRealSource(new MapStream(new MSecondStream(intervalInMs, sz), this));
+      }
+      #region IStreamMapper Members
+
+      public IBlueTableSchema getOutputSchema(IBlueTableSchema inputSchema) {
+         return mySchema;
+      }
+
+      #endregion
+
+      #region IDynamicStreamMapper Members
+
+      public IBlueRow map(IBlueRow row) {
+         DateTime time = row.get_datetime("time");
+         var time_txt = time.ToString("yyyy-MM-dd HH:mm:ss.fff");
+         return new BlueRow(mySchema, row["id"], time_txt);
+      }
+
+      #endregion
+
+      private string myToString(DateTime dt) {
+         string sep = ":";
+         return dt.Hour + sep + dt.Minute + sep + dt.Second + sep + dt.Millisecond;
+      }
+   }
+}

src/DataStreamSuite/DataStream/BlueOrb/MultiRowOpHelper.cs

@@ -0,0 +1,127 @@
+namespace bts.BlueOrb {
+   using BCG;
+
+   using System;
+   using System.Collections.Generic;
+   using System.Linq;
+   //using MathNet.Numerics.Statistics;
+
+
+   public enum MultiRowOp {
+      Mean,
+      Median,
+      Mode,
+      Range,
+      StdDev,
+      Max,
+      Min,
+      First,
+      Last,
+      Sum,
+      EdgeDiff,
+      EdgeDiffPercentage,
+      //IsEdgeDiff,
+   }
+   public class MultiRowOpHelper {
+      public readonly List<Tuple<string, MultiRowOp>> operations;
+
+      public static MultiRowOpHelper parse_ops(string text) {
+         // text is inp_col: op, inp_col: op, inp_col:op
+         List<Tuple<string, MultiRowOp>> rv = new List<Tuple<string, MultiRowOp>>();
+         string[] calcs = text.Split(',');
+         foreach (string c in calcs) {
+            string[] toks = c.Split(':');
+            string inp_col = toks[0].Trim();
+            string op_txt = toks[1].Trim();
+            MultiRowOp op = Enum.Parse<MultiRowOp>(op_txt, true);
+            rv.Add(Tuple.Create(inp_col, op));
+         }
+         return new MultiRowOpHelper(rv);
+      }
+
+      public MultiRowOpHelper(List<Tuple<string, MultiRowOp>> operations_) {
+         operations = operations_;
+      }
+
+      // aggregates multiple rows into one - ie does group_by compute
+      public static IBlueRow group_by_compute(List<IBlueRow> rows, MultiRowOpHelper parameters) {
+         if (rows.Count() == 0) {
+            return null;
+         }
+
+         IBlueRow ir = rows.ElementAt(0);
+         IBlueTableSchema in_schema = ir.schema;
+         IBlueTableSchema out_schema = in_schema;
+         IDictionary<string, Tuple<string, MultiRowOp>> out_col_map =
+            new Dictionary<string, Tuple<string, MultiRowOp>>();
+         foreach (var t in parameters.operations) {
+            string out_col_name = $"{t.Item1}_{t.Item2}".ToLower();
+            out_schema = out_schema.add_column(TypeCode.Double, out_col_name);
+            out_col_map[out_col_name] = t;
+         }
+         MutableBlueRow or = new MutableBlueRow(out_schema, ir);
+         foreach (var kv in out_col_map) {
+            string out_col_name = kv.Key;
+            string in_col_name = kv.Value.Item1;
+            MultiRowOp op = kv.Value.Item2;
+            IEnumerable<double> col_vals = BlueDataUtility.get_colvalues<double>(rows, in_col_name);
+            double comp_res = compute_op(col_vals, op);
+            or.setValue(out_col_name, comp_res);
+         }
+         return or;
+      }
+
+      // rolling_rows are the rolling window of rows. the 0th row is the one thats gettin
+      // calculated. rows 1 - n-1 are its window rows. the schema is based on the operations.
+
+      public IBlueRow rolling_calc(List<IBlueRow> rolling_rows) {
+         if (rolling_rows.Count() == 0) {
+            return null;
+         }
+         IBlueRow ir = rolling_rows.ElementAt(0);
+         IBlueTableSchema in_schema = ir.schema;
+         IBlueTableSchema out_schema = in_schema;
+         IDictionary<string, Tuple<string, MultiRowOp>> out_col_map =
+            new Dictionary<string, Tuple<string, MultiRowOp>>();
+         foreach (var t in operations) {
+            string out_col_name = $"{t.Item1}_{t.Item2}".ToLower();
+            out_schema = out_schema.add_column(TypeCode.Double, out_col_name);
+            out_col_map[out_col_name] = t;
+         }
+         MutableBlueRow or = new MutableBlueRow(out_schema, ir);
+         foreach (var kv in out_col_map) {
+            string out_col_name = kv.Key;
+            string in_col_name = kv.Value.Item1;
+            MultiRowOp op = kv.Value.Item2;
+            IEnumerable<double> col_vals = BlueDataUtility.get_colvalues<double>(rolling_rows, in_col_name);
+            double comp_res = compute_op(col_vals, op);
+            or.setValue(out_col_name, comp_res);
+         }
+         return or;
+      }
+
+      public static double compute_op(IEnumerable<double> col_vals, MultiRowOp op) {
+         switch (op) {
+            case MultiRowOp.Max:
+               return col_vals.Max<double>();
+            case MultiRowOp.Min:
+               return col_vals.Min<double>();
+            case MultiRowOp.Mean:
+               return col_vals.Sum() / col_vals.Count();
+            case MultiRowOp.Sum:
+               return col_vals.Sum();
+            case MultiRowOp.First:
+               return col_vals.First();
+            case MultiRowOp.Last:
+               return col_vals.Last();
+            case MultiRowOp.StdDev:
+               return MathNet.Numerics.Statistics.Statistics.StandardDeviation(col_vals);
+            default:
+               // TODO
+               Log.assert(false);
+               return Double.NaN;
+         }
+      }
+
+   }
+}

src/DataStreamSuite/DataStream/BlueOrb/ProjectStream.cs

@@ -0,0 +1,207 @@
+namespace bts.BlueOrb {
+   using BCG;
+
+   using ProtoBuf;
+
+   using System;
+   using System.Collections;
+   using System.Collections.Generic;
+   using System.Linq;
+
+   #region mappings
+   internal class ProjectData {
+      // schema from the base stream
+      public readonly IBlueTableSchema in_schema;
+      // this is the projected schema
+      public readonly IBlueTableSchema out_schema;
+      // projected cols to the input cols, for cols who's names are changed
+      public readonly IDictionary<string, string> mapped_to_orig_col_map = new Dictionary<string, string>();
+      public ProjectData(IBlueTableSchema in_schema_, IBlueTableSchema out_schema_,
+                  IDictionary<string, string> mapped_to_orig_col_map_) {
+         in_schema = in_schema_;
+         out_schema = out_schema_;
+         mapped_to_orig_col_map = mapped_to_orig_col_map_;
+      }
+   }
+   #endregion
+
+
+#region Row with mapipings build in
+   internal class NewProjectRow : IBlueRow {
+      #region IBlueRow members
+      private ProjectData _project_data;
+      private IBlueRow _row;
+      public NewProjectRow(ProjectData project_data, IBlueRow row) {
+         _project_data = project_data;
+         _row = row;
+      }
+      public IBlueTableSchema schema { get { return _project_data.out_schema; } }
+      #endregion
+
+      #region ICollection members
+      public int Count { get { return schema.columnNames.Length; } }
+
+      public bool IsSynchronized { get { return false; } }
+
+      public object SyncRoot { get { return this; } }
+
+      public void CopyTo(Array array, int index) {
+         int i = index;
+         foreach (object o in this as IEnumerable)
+            array.SetValue(o, i++);
+      }
+      #endregion
+
+      #region IDictionary members
+      public bool IsFixedSize { get { return true; } }
+
+      public bool IsReadOnly { get { return true; } }
+
+      public object this[object key] {
+         get {
+            if (key is string)
+               return this[(string)key];
+            return this[(int)key];
+         }
+         set { throw new NotSupportedException("The IBlueRow is readonly"); }
+      }
+
+      public object this[int index] {
+         get {
+            return this[schema.columnInfo[index].colName];
+         }
+      }
+
+      public object this[string col_name] {
+         get {
+            string mapped_to_col = null;
+            if (_project_data.mapped_to_orig_col_map.TryGetValue(col_name, out mapped_to_col)) {
+               return _row[mapped_to_col];
+            }
+            return _row[col_name];
+         }
+      }
+
+      public ICollection Keys { get { return schema.columnNames; } }
+
+      public ICollection Values {
+         get {
+            object[] retVal = new object[schema.columnNames.Length];
+            CopyTo(retVal, 0);
+            return retVal;
+         }
+      }
+
+      public void Add(object key, object value) { throw new NotSupportedException("The IBlueRow is readonly"); }
+
+      public void Clear() { throw new NotSupportedException("The IBlueRow is readonly"); }
+
+      public bool Contains(object key) { return schema[(string)key] != null; }
+
+      /// <summary>
+      /// Note, the default enumerator is the IDictionary's enumerator, so if user needs to
+      /// iterate through the values, typecast this into an IEnumerable
+      /// </summary>
+      /// <returns></returns>
+      public IDictionaryEnumerator GetEnumerator() { return new BlueRowDictionaryEnumerator(this); }
+
+      IEnumerator IEnumerable.GetEnumerator() { return new BlueRowEnumerator(this); }
+
+      public void Remove(object key) { throw new NotSupportedException("The IBlueRow is readonly"); }
+      #endregion
+   }
+   #endregion
+
+   [DataStreamUserRespresentation("ProjectStream")]
+   public class NewProjectStream : ViewStream, IStreamMapper {
+
+      private ViewStream _stream;
+      private string _project_expression;
+      private ProjectData _project_data = null;
+      //public ProjectData 
+      public NewProjectStream(ViewStream vs, string project_expression) {
+         _stream = vs;
+         _project_expression = project_expression;
+         _stream = new MapStream(vs, this);
+         mySetRealSource(_stream);
+      }
+
+      public IBlueTableSchema getOutputSchema(IBlueTableSchema input_schema) {
+         _project_data = _create_project_data(input_schema, _project_expression);
+         return _project_data.out_schema;
+      }
+
+      public IBlueRow map(IBlueRow row) {
+         return new NewProjectRow(_project_data, row);
+      }
+
+      private static ProjectData _create_project_data(IBlueTableSchema input_schema,
+                                                      string project_expression) {
+         project_expression = project_expression.Trim();
+         string[] project_toks = project_expression.Split(' ');
+         HashSet<string> removed_cols = new HashSet<string>();
+         Dictionary<string, string> mapped_to_orig_map = new Dictionary<string, string>();
+         Dictionary<string, string> orig_to_mapped = new Dictionary<string, string>();
+         // by default every col will show up, so if we have the projection as (a:a1)
+         // it will convert a to a1, 
+         // examples: if a schema has cols a b c
+         // project(parent_str, "a:aa ") -> aa b c
+         // project(parent_str, "a:aa !c") -> aa b
+         // project(parent_str, "a:aa !") -> aa
+         // project(parent_str, "a:aa b")) -> aa b
+         bool hide_all = false;
+         foreach (string p_tok in project_toks) {
+            if (p_tok.StartsWith("!")) {
+               if (p_tok.Length == 1) {
+                  // if it is a "blanket "!"=> hide all cols except the one that
+                  // are stated explicitly
+                  hide_all = true;
+               } else {
+                  string removed_col = p_tok.Substring(1);
+                  removed_cols.Add(removed_col);
+               }
+            } else if (p_tok.Contains(":")) {
+               string[] map_toks = p_tok.Split(":");
+               string orig_col = map_toks[0];
+               string mapped_col = map_toks[1];
+               mapped_to_orig_map.Add(mapped_col, orig_col);
+               orig_to_mapped.Add(orig_col, mapped_col);
+            } else {
+               if (input_schema.columnNames.Contains(p_tok)) {
+                  string orig_col = p_tok;
+                  string mapped_col = orig_col;
+                  orig_to_mapped.Add(orig_col, mapped_col);
+               }
+            }
+         }
+
+         // make the new schema
+         List<ColumnInfo> new_cols = new List<ColumnInfo>();
+         foreach (ColumnInfo ci in input_schema.columnInfo) {
+            if (ci.colName == "id") {
+               ColumnInfo nc = new ColumnInfo(ci.colName, new_cols.Count, ci.colType);
+               new_cols.Add(nc);
+               continue;
+            }
+            if (removed_cols.Contains(ci.colName)) {
+               continue;
+            } else if (orig_to_mapped.ContainsKey(ci.colName)) {
+               string mapped_name = orig_to_mapped[ci.colName];
+               ColumnInfo nc = new ColumnInfo(mapped_name, new_cols.Count, ci.colType);
+               new_cols.Add(nc);
+            } else {
+               if (hide_all)
+                  continue;
+               //if (ci.colName == "id" || removed_cols.Count > 0) {
+               // unless columns are  explicitely hidden, all columns show up
+               ColumnInfo nc = new ColumnInfo(ci.colName, new_cols.Count, ci.colType);
+               new_cols.Add(nc);
+               //}
+            }
+         }
+         IBlueTableSchema new_schema = new BlueTableSchema(input_schema.tableName, new_cols.ToArray());
+         return new ProjectData(input_schema, new_schema, mapped_to_orig_map);
+      }
+
+   }
+}

src/DataStreamSuite/DataStream/BlueOrb/SchemaGuesserStream.cs

@@ -0,0 +1,30 @@
+using BCG;
+
+namespace bts.BlueOrb {
+   public class SchemaGuesserStream : ViewStream {
+
+      private IBlueRowCollection _rows = null;
+      private ViewStream _vs;
+      public SchemaGuesserStream(ViewStream vs) {
+         _vs = vs;
+      }
+
+      protected override void myOnFirstSubscriber() {
+         _vs.onChange += _on_data;
+      }
+      protected override void myOnLastSubscriber() {
+         _vs.onChange -= _on_data;
+      }
+
+      private void _on_data(object obj) {
+         DataPacket dp = obj as DataPacket;
+         if (_rows == null) {
+            _rows = BlueDataUtility.get_typed_table(dp.newRows);
+         }
+         var dp_out = BlueDataUtility.build_dp_inc(_rows, myLastValue as DataPacket);
+         myFireEvent(dp_out);
+
+      }
+
+   }
+}

src/DataStreamSuite/DataStream/BlueOrb/SingleRowMathStream.cs

@@ -0,0 +1,67 @@
+using Bts.ArithmeticExpressions;
+
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+using BCG;
+
+namespace bts.BlueOrb {
+
+   [DataStreamUserRespresentation("SingleRowMathStream")]
+   public class MathStream : ViewStream, IStreamMapper {
+      private IArithmeticProgram myExp;
+      private BooleanExpressionFilter _bool;
+      List<string> myExtraColumns = new List<string>();
+      BlueTableSchema mySchema;
+
+
+      public MathStream(ViewStream vs, string mathExpression)
+         : this(vs, mathExpression, string.Empty) {
+      }
+
+      public MathStream(ViewStream vs, string mathExpression, string boolean_expression) {
+         myExp = ArithmeticProgramParser.Parse(mathExpression) ??
+            throw new Exception($"Failed to parse ArithmeticExpression '{mathExpression}'");
+         myExtraColumns = myExp.OutputIdentifiers.ToList();
+         if (!string.IsNullOrEmpty(boolean_expression))
+            _bool = new BooleanExpressionFilter(boolean_expression);
+         mySetRealSource(new MapStream(vs, this));
+      }
+
+      #region IStreamMapper Members
+
+      public IBlueTableSchema getOutputSchema(IBlueTableSchema inputSchema) {
+         if (mySchema == null) {
+            Dictionary<string, ColumnInfo> colIdx = new Dictionary<string, ColumnInfo>();
+            foreach (ColumnInfo ci in inputSchema.columnInfo)
+               colIdx[ci.colName] = ci;
+
+            foreach (string ncn in myExtraColumns) {
+               ColumnInfo newC;
+               if (colIdx.ContainsKey(ncn))
+                  newC = new ColumnInfo(colIdx[ncn].colName, colIdx[ncn].colNo, typeof(double));
+               else
+                  newC = new ColumnInfo(ncn, colIdx.Count, typeof(double));
+               colIdx[ncn] = newC;
+            }
+            mySchema = new BlueTableSchema(colIdx.Values.ToArray(), "math(" + inputSchema.tableName + ")");
+         }
+         return mySchema;
+      }
+
+      #endregion
+
+      #region IDynamicStreamMapper Members
+
+      public IBlueRow map(IBlueRow row) {
+         MutableBlueRow result = new MutableBlueRow(mySchema, row);
+         if (_bool == null || _bool.filter(row)) {
+            myExp.Evaluate(result.AsDictionary());
+         }
+         return result;
+      }
+
+      #endregion
+   }
+}

src/DataStreamSuite/DataStream/BlueOrb/TopStream.cs

@@ -0,0 +1,11 @@
+namespace bts.BlueOrb {
+   using BCG;
+   [DataStreamUserRespresentation("TopStream")]
+   public class TopStream : ViewStream { 
+      public TopStream(ViewStream vs, int n_rows, string sort_col) {
+         ViewStream rs = new BCG.TopStream (vs, n_rows, sort_col);
+         mySetRealSource(rs);
+      }
+   }
+
+}

src/DataStreamSuite/DataStream/BlueOrb/UnionStream.cs

@@ -0,0 +1,261 @@
+using System;
+using System.Collections.Generic;
+
+using mbts;
+using BCG;
+
+namespace bts.BlueOrb {
+   internal class MutableAppendTable : IBlueRowCollection {
+      private class MyRow : IBlueRow {
+         private IBlueRow myRow;
+         private int myID;
+
+         public MyRow(IBlueRow r, int id) {
+            myRow = r;
+            myID = id;
+         }
+
+         #region IBlueRow Members
+
+         public IBlueTableSchema schema { get { return myRow.schema; } }
+
+         public object this[int index] {
+            get {
+               if (myRow.schema[index].colName == "id")
+                  return myID;
+               else
+                  return myRow[index];
+            }
+         }
+
+         public object this[string colName] { get { return colName == "id" ? (object)myID : myRow[colName]; } }
+
+         #endregion
+
+         #region IDictionary Members
+
+         public void Add(object key, object value) { throw new Exception("Row is read only."); }
+
+         public void Clear() { throw new Exception("Row is read only."); }
+
+         public bool Contains(object key) { return myRow.Contains(key); }
+
+         public System.Collections.IDictionaryEnumerator GetEnumerator() { return new BlueRowDictionaryEnumerator(this); }
+
+         public bool IsFixedSize { get { return true; } }
+
+         public bool IsReadOnly { get { return true; } }
+
+         public System.Collections.ICollection Keys { get { return myRow.Keys; } }
+
+         public void Remove(object key) { throw new Exception("The method or operation is not implemented."); }
+
+         public System.Collections.ICollection Values {
+            get {
+               object[] vals = new object[myRow.Count];
+               CopyTo(vals, 0);
+               return vals;
+            }
+         }
+
+         public object this[object key] {
+            get { return key is string ? this[key as string] : this[(int)key]; }
+            set { throw new Exception("The method or operation is not implemented."); }
+         }
+
+         #endregion
+
+         #region ICollection Members
+
+         public void CopyTo(Array array, int index) {
+            myRow.CopyTo(array, index);
+            array.SetValue(myID, index + myRow.schema["id"].colNo);
+         }
+
+         public int Count { get { return myRow.Count; } }
+
+         public bool IsSynchronized { get { return myRow.IsSynchronized; } }
+
+         public object SyncRoot { get { return myRow.SyncRoot; } }
+
+         #endregion
+
+         #region IEnumerable Members
+
+         System.Collections.IEnumerator System.Collections.IEnumerable.GetEnumerator() { return new BlueRowEnumerator(this); }
+
+         #endregion
+      }
+
+      private BlueTable myRows;
+      private int myID = 0;
+      /// <summary>
+      /// maps the table no to a dictionary containing the
+      /// local IDs to a row in the final big table
+      /// </summary>
+      private Dictionary<int, Dictionary<int, IBlueRow>> myIDMap = new Dictionary<int, Dictionary<int, IBlueRow>>();
+
+      public MutableAppendTable(IBlueTableSchema schema) { myRows = new BlueTable(schema); }
+
+      public IBlueRow addRow(IBlueRow r, int tableNo) {
+         if (!myIDMap.ContainsKey(tableNo))
+            myIDMap.Add(tableNo, new Dictionary<int, IBlueRow>());
+         IBlueRow nr = new MyRow(r, myID++); // BlueDataUtility.getNewReplacedRow(r, myID++, "id");
+         myRows.addRow(nr);
+         myIDMap[tableNo].Add((int)r["id"], nr);
+         return nr;
+      }
+
+      public IBlueRow removeRow(IBlueRow r, int tableNo) {
+         IBlueRow dr = myIDMap[tableNo][(int)r["id"]];
+         myRows.deleteRow(dr);
+         myIDMap[tableNo].Remove((int)r["id"]);
+         return dr;
+      }
+
+      public IBlueRowCollection removeTable(int tableNo) {
+         if (!myIDMap.ContainsKey(tableNo))
+            return null;
+         else {
+            BlueTable rv = new BlueTable(myRows.schema);
+            foreach (IBlueRow r in myIDMap[tableNo].Values)
+               rv.addRow(r);
+            myRows.deleteRows(rv);
+            myIDMap.Remove(tableNo);
+            return rv;
+         }
+      }
+
+      public void clear() {
+         myRows.clear();
+         myIDMap.Clear();
+      }
+
+      #region IBlueRowCollection Members
+
+      public IBlueRow this[int id] { get { return myRows[id]; } }
+
+      public IBlueTableSchema schema { get { return myRows.schema; } }
+
+      public bool containsID(int id) { return myRows.containsID(id); }
+
+      #endregion
+
+      #region ICollection Members
+
+      public void CopyTo(Array array, int index) { myRows.CopyTo(array, index); }
+
+      public int Count { get { return myRows.Count; } }
+
+      public bool IsSynchronized { get { return myRows.IsSynchronized; } }
+
+      public object SyncRoot { get { return myRows.SyncRoot; } }
+
+      #endregion
+
+      #region IEnumerable Members
+
+      public System.Collections.IEnumerator GetEnumerator() { return myRows.GetEnumerator(); }
+
+      #endregion
+   }
+   /// <summary>
+   /// MutableStream can append multiple streams
+   /// as well as give the user the flexibility
+   /// to append (or remove) more streams at
+   /// runtime
+   /// </summary>
+   }
+namespace bts.BlueOrb { 
+   [DataStreamUserRespresentation("UnionStream")]
+   public class MutableAppendStream : ViewStream {
+      private bool mySubscriptionActive = false;
+      private MutableAppendTable myTable;
+      private int myMovingIndex;
+      private Dictionary<int, Pair<ViewStream, ViewStream>> myStreamIndex = new Dictionary<int, Pair<ViewStream, ViewStream>>();
+      private bool _allign_col_orders = true;
+
+      public MutableAppendStream(params ViewStream[] streams)
+         : this(new List<ViewStream>(streams)) { }
+
+      public MutableAppendStream(bool allign_col_order, params ViewStream[] streams)
+         : this(allign_col_order, new List<ViewStream>(streams)) { }
+
+      public MutableAppendStream(ICollection<ViewStream> streams)
+         : this(true, streams) { }
+
+      public MutableAppendStream(bool allign_col_orders, ICollection<ViewStream> streams) {
+         _allign_col_orders = allign_col_orders;
+         foreach (ViewStream vs in streams) {
+            myMovingIndex++;
+            ViewStream pj = new ChangeTableNameStream(vs, myMovingIndex.ToString());
+            if (_allign_col_orders)
+               pj = new ProjectStream(pj, "!");
+            myStreamIndex.Add(myMovingIndex, new Pair<ViewStream, ViewStream>(vs, pj));
+         }
+      }
+
+      public void addStream(ViewStream vs) {
+         myMovingIndex++;
+         ///the change name is done to identify the callbacks, and the project to
+         ///make all the schemas col orders the same
+         ViewStream pj = new ChangeTableNameStream(vs, myMovingIndex.ToString());
+         if (_allign_col_orders)
+            pj = new ProjectStream(pj, "!");
+         myStreamIndex.Add(myMovingIndex, new Pair<ViewStream, ViewStream>(vs, pj));
+         if (mySubscriptionActive)
+            pj.onChange += new ChangeDelegate(myHandleStreamChanges);
+      }
+
+      public void removeStream(ViewStream vs) {
+         int index = -1;
+         foreach (KeyValuePair<int, Pair<ViewStream, ViewStream>> p in myStreamIndex) {
+            if (p.Value.first == vs) {
+               index = p.Key;
+               if (mySubscriptionActive) {
+                  p.Value.second.onChange -= new ChangeDelegate(myHandleStreamChanges);
+                  if (myTable != null) {
+                     IBlueRowCollection delRows = myTable.removeTable(p.Key);
+                     if (delRows != null)
+                        myFireEvent(new DataPacket(myTable, new BlueTable(myTable.schema), delRows));
+                  }
+               }
+               break;
+            }
+            if (index > 0)
+               myStreamIndex.Remove(index);
+         }
+      }
+
+      protected override void myOnFirstSubscriber() {
+         mySubscriptionActive = true;
+         foreach (KeyValuePair<int, Pair<ViewStream, ViewStream>> p in myStreamIndex)
+            p.Value.second.onChange += new ChangeDelegate(myHandleStreamChanges);
+      }
+
+      protected override void myOnLastSubscriber() {
+         mySubscriptionActive = false;
+         foreach (KeyValuePair<int, Pair<ViewStream, ViewStream>> p in myStreamIndex)
+            p.Value.second.onChange -= new ChangeDelegate(myHandleStreamChanges);
+         if (myTable != null)
+            myTable.clear();
+      }
+
+      private void myHandleStreamChanges(object newValue) {
+         DataPacket dp = newValue as DataPacket;
+         myTable = myTable ?? new MutableAppendTable(new BlueTableSchema("MutableAppend", dp.schema.columnInfo));
+         BlueTable adds = new BlueTable(myTable.schema),
+                     dels = new BlueTable(myTable.schema);
+         int index = Int32.Parse(dp.schema.tableName);
+         foreach (IBlueRow r in dp.deletedRows) {
+            IBlueRow dr = myTable.removeRow(r, index);
+            dels.addRow(dr);
+         }
+         foreach (IBlueRow r in dp.addedRows) {
+            IBlueRow nr = myTable.addRow(r, index);
+            adds.addRow(nr);
+         }
+         myFireEvent(new DataPacket(myTable, adds, dels));
+      }
+   }
+}

src/DataStreamSuite/DataStream/DataStream.csproj

@@ -1,7 +1,7 @@
 <Project Sdk="Microsoft.NET.Sdk">
 
   <PropertyGroup>
-    <TargetFramework>net7.0</TargetFramework>
+    <TargetFramework>net8.0</TargetFramework>
     <Nullable>enable</Nullable>
   </PropertyGroup>
 
@@ -18,11 +18,13 @@
 
   <ItemGroup>
     <PackageReference Include="HtmlAgilityPack" Version="1.11.57" />
+    <PackageReference Include="IronPython" Version="3.4.2" />
     <PackageReference Include="Microsoft.Data.Analysis" Version="0.21.1" />
     <PackageReference Include="MongoDB.Bson" Version="2.23.1" />
     <PackageReference Include="MongoDB.Driver" Version="2.23.1" />
     <PackageReference Include="Parquet.Net" Version="5.0.1" />
     <PackageReference Include="System.Data.SQLite" Version="1.0.118" />
+    <PackageReference Include="MathNet.Numerics" Version="5.0.0" />
   </ItemGroup>
 
 </Project>

src/DataStreamSuite/DataStream/Primitives/Count/CountStream.cs

@@ -1,52 +1,47 @@
 using System;
 
-namespace BCG
-{
-   [DataStreamUserRespresentation ("count")]
-	public class CountStream : ViewStream
-	{
+namespace bts.BlueOrb {
+   using BCG;
+   [DataStreamUserRespresentation("CountStream")]
+   public class CountStream : ViewStream {
       private DataPacket myLastFiredDataPacket;
-      private IBlueTableSchema mySchema = new BlueTableSchema ("count", 
-         new ColumnInfo ("id", 0, typeof(int)),
-         new ColumnInfo ("count", 1, typeof(int)));
+      private IBlueTableSchema mySchema = new BlueTableSchema("count",
+         new ColumnInfo("id", 0, typeof(int)),
+         new ColumnInfo("count", 1, typeof(int)));
       private int myId = 0;
       private ViewStream myVS;
 
-      public CountStream (ViewStream vs)
-      {
+      public CountStream(ViewStream vs) {
          myVS = vs;
       }
 
-      public CountStream (string tableName)
-         : this (new TableStream (tableName))
-      {}
+      public CountStream(string tableName)
+         : this(new TableStream(tableName)) { }
 
-      protected override void myOnFirstSubscriber () {
+      protected override void myOnFirstSubscriber() {
          myVS.onChange += new ChangeDelegate(myHandleChanges);
       }
-      protected override void myOnLastSubscriber () {
+      protected override void myOnLastSubscriber() {
          myVS.onChange -= new ChangeDelegate(myHandleChanges);
       }
 
 
-      private void myHandleChanges(object newValue)
-      {
-         myFireEvent (
+      private void myHandleChanges(object newValue) {
+         myFireEvent(
             myLastFiredDataPacket =
                myLastFiredDataPacket == null ?
-                  new DataPacket (myGetRows (newValue, ++myId), myGetRows (newValue, myId), 
-                  new BlueTable (mySchema))
+                  new DataPacket(myGetRows(newValue, ++myId), myGetRows(newValue, myId),
+                  new BlueTable(mySchema))
                   :
-                  new DataPacket (myGetRows (newValue, ++myId), myGetRows (newValue, myId), 
+                  new DataPacket(myGetRows(newValue, ++myId), myGetRows(newValue, myId),
                   myLastFiredDataPacket.newRows)
             );
       }
 
-      private IBlueRowCollection myGetRows (object dp, int id)
-      {
-         BlueTable retVal = new BlueTable (mySchema);
+      private IBlueRowCollection myGetRows(object dp, int id) {
+         BlueTable retVal = new BlueTable(mySchema);
          retVal.addRow
-            (new BlueRow (mySchema, id, (dp as DataPacket).newRows.Count));
+            (new BlueRow(mySchema, id, (dp as DataPacket).newRows.Count));
          return retVal;
       }
 

src/DataStreamSuite/DataStream/Primitives/Filter/FilterStream.cs

@@ -5,25 +5,20 @@ using mbts.config;
 using System;
 using System.Text.RegularExpressions;
 
-namespace BCG.StreamWorld.Addition
-{
-   [DataStreamUserRespresentation (false)]
-   public class FilterStream : ViewStream
-   {
+namespace BCG.StreamWorld.Addition {
+   [DataStreamUserRespresentation(false)]
+   public class FilterStream : ViewStream {
       private ViewStream myBaseStream;
       private BlueTable myBaseTable, myStateTable;
       private IStreamFilter myFilter;
 
       public FilterStream(ViewStream vs)
-         : this(vs, "1=1")
-      { }
+         : this(vs, "1=1") { }
 
       public FilterStream(ViewStream vs, string filterExp)
-         : this(vs, new BooleanExpressionFilter(filterExp))
-      { }
+         : this(vs, new BooleanExpressionFilter(filterExp)) { }
 
-      public FilterStream(ViewStream vs, IStreamFilter filter)
-      {
+      public FilterStream(ViewStream vs, IStreamFilter filter) {
          myBaseStream = vs;
          myFilter = filter;
          if (filter is IDynamicStreamFilter)
@@ -31,43 +26,35 @@ namespace BCG.StreamWorld.Addition
       }
 
 
-      protected override void myOnFirstSubscriber()
-      {
+      protected override void myOnFirstSubscriber() {
          myBaseStream.onChange += new ChangeDelegate(myHandleChanges);
       }
 
-      protected override void myOnLastSubscriber()
-      {
+      protected override void myOnLastSubscriber() {
          myBaseStream.onChange -= new ChangeDelegate(myHandleChanges);
          myBaseTable.clear();
          myStateTable.clear();
       }
 
-      private void myHandleChanges(object newValue)
-      {
+      private void myHandleChanges(object newValue) {
          DataPacket dp = newValue as DataPacket;
-         if (myStateTable == null)
-         {
+         if (myStateTable == null) {
             bts.Log.assert(myBaseTable == null);
             myStateTable = new BlueTable(dp.schema);
             myBaseTable = new BlueTable(dp.schema);
          }
          BlueTable dels = new BlueTable(dp.schema),
                      adds = new BlueTable(dp.schema);
-         foreach (IBlueRow r in dp.deletedRows)
-         {
+         foreach (IBlueRow r in dp.deletedRows) {
             myBaseTable.deleteRow(r);
-            if (myStateTable.containsID((int)r["id"]))
-            {
+            if (myStateTable.containsID((int)r["id"])) {
                dels.addRow(r);
                myStateTable.deleteRow(r);
             }
          }
-         foreach (IBlueRow r in dp.addedRows)
-         {
+         foreach (IBlueRow r in dp.addedRows) {
             myBaseTable.addRow(r);
-            if (myFilter.filter(r))
-            {
+            if (myFilter.filter(r)) {
                adds.addRow(r);
                myStateTable.addRow(r);
             }
@@ -76,13 +63,10 @@ namespace BCG.StreamWorld.Addition
             myFireEvent(new DataPacket(myStateTable, adds, dels));
       }
 
-      public IStreamFilter filter
-      {
-         set
-         {
+      public IStreamFilter filter {
+         set {
             myFilter = value;
-            if (myBaseTable != null)
-            {
+            if (myBaseTable != null) {
                IBlueRowCollection oldState = new BlueTable(myStateTable);
                myStateTable.clear();
                foreach (IBlueRow r in myBaseTable)

src/DataStreamSuite/DataStream/Primitives/GroupBy/GroupByBundle.cs

@@ -4,42 +4,46 @@ namespace bts.SilverPine {
    using System;
    using System.Collections.Generic;
    using System.Linq;
+   //using MathNet.Numerics.Statistics;
 
-   using ZstdSharp.Unsafe;
 
-   public enum GroupByBundleOp {
+   public enum MultiRowOp {
       Mean,
       Median,
       Mode,
+      Range,
       StdDev,
       Max,
       Min,
       First,
       Last,
-      Sum
+      Sum,
+      EdgeDiff,
+      EdgeDiffPercentage,
+      //IsEdgeDiff,
    }
-   public class GroupByBundle {
-      public readonly List<Tuple<string, GroupByBundleOp>> operations;
+   public class MultiRowOpHelper {
+      public readonly List<Tuple<string, MultiRowOp>> operations;
 
-      public static GroupByBundle parse(string text) {
+      public static MultiRowOpHelper parse(string text) {
          // text is inp_col: op, inp_col: op, inp_col:op
-         List<Tuple<string, GroupByBundleOp>> rv = new List<Tuple<string, GroupByBundleOp>>();
+         List<Tuple<string, MultiRowOp>> rv = new List<Tuple<string, MultiRowOp>>();
          string[] calcs = text.Split(',');
          foreach (string c in calcs) {
             string[] toks = c.Split(':');
             string inp_col = toks[0].Trim();
             string op_txt = toks[1].Trim();
-            GroupByBundleOp op = Enum.Parse<GroupByBundleOp>(op_txt, true);
+            MultiRowOp op = Enum.Parse<MultiRowOp>(op_txt, true);
             rv.Add(Tuple.Create(inp_col, op));
          }
-         return new GroupByBundle(rv);
+         return new MultiRowOpHelper(rv);
       }
 
-      public GroupByBundle(List<Tuple<string, GroupByBundleOp>> operations_) {
+      public MultiRowOpHelper(List<Tuple<string, MultiRowOp>> operations_) {
          operations = operations_;
       }
 
-      public static IBlueRow group_by(List<IBlueRow> rows, GroupByBundle parameters) {
+      public static IBlueRow group_by(List<IBlueRow> rows, MultiRowOpHelper parameters) {
          if (rows.Count() == 0) {
             return null;
          }
@@ -47,8 +51,8 @@ namespace bts.SilverPine {
          IBlueRow ir = rows.ElementAt(0);
          IBlueTableSchema in_schema = ir.schema;
          IBlueTableSchema out_schema = in_schema;
-         IDictionary<string, Tuple<string, GroupByBundleOp>> out_col_map =
-            new Dictionary<string, Tuple<string, GroupByBundleOp>>();
+         IDictionary<string, Tuple<string, MultiRowOp>> out_col_map =
+            new Dictionary<string, Tuple<string, MultiRowOp>>();
          foreach (var t in parameters.operations) {
             string out_col_name = $"{t.Item1}_{t.Item2}".ToLower();
             out_schema = out_schema.add_column(TypeCode.Double, out_col_name);
@@ -58,7 +62,7 @@ namespace bts.SilverPine {
          foreach (var kv in out_col_map) {
             string out_col_name = kv.Key;
             string in_col_name = kv.Value.Item1;
-            GroupByBundleOp op = kv.Value.Item2;
+            MultiRowOp op = kv.Value.Item2;
             IEnumerable<double> col_vals = BlueDataUtility.get_colvalues<double>(rows, in_col_name);
             double comp_res = compute_op(col_vals, op);
             or.setValue(out_col_name, comp_res);
@@ -66,20 +70,51 @@ namespace bts.SilverPine {
          return or;
       }
 
-      public static double compute_op(IEnumerable<double> col_vals, GroupByBundleOp op) {
+      // rolling_rows are the rolling window of rows. the 0th row is the one thats gettin
+      // calculated. rows 1 - n-1 are its window rows. the schema is based on the operations.
+
+      public IBlueRow rolling_calc(List<IBlueRow> rolling_rows) {
+         if (rolling_rows.Count() == 0) {
+            return null;
+         }
+         IBlueRow ir = rolling_rows.ElementAt(0);
+         IBlueTableSchema in_schema = ir.schema;
+         IBlueTableSchema out_schema = in_schema;
+         IDictionary<string, Tuple<string, MultiRowOp>> out_col_map =
+            new Dictionary<string, Tuple<string, MultiRowOp>>();
+         foreach (var t in operations) {
+            string out_col_name = $"{t.Item1}_{t.Item2}".ToLower();
+            out_schema = out_schema.add_column(TypeCode.Double, out_col_name);
+            out_col_map[out_col_name] = t;
+         }
+         MutableBlueRow or = new MutableBlueRow(out_schema, ir);
+         foreach (var kv in out_col_map) {
+            string out_col_name = kv.Key;
+            string in_col_name = kv.Value.Item1;
+            MultiRowOp op = kv.Value.Item2;
+            IEnumerable<double> col_vals = BlueDataUtility.get_colvalues<double>(rolling_rows, in_col_name);
+            double comp_res = compute_op(col_vals, op);
+            or.setValue(out_col_name, comp_res);
+         }
+         return or;
+      }
+
+      public static double compute_op(IEnumerable<double> col_vals, MultiRowOp op) {
          switch (op) {
-            case GroupByBundleOp.Max:
+            case MultiRowOp.Max:
                return col_vals.Max<double>();
-            case GroupByBundleOp.Min:
+            case MultiRowOp.Min:
                return col_vals.Min<double>();
-            case GroupByBundleOp.Mean:
+            case MultiRowOp.Mean:
                return col_vals.Sum() / col_vals.Count();
-            case GroupByBundleOp.Sum:
+            case MultiRowOp.Sum:
                return col_vals.Sum();
-            case GroupByBundleOp.First:
+            case MultiRowOp.First:
                return col_vals.First();
-            case GroupByBundleOp.Last:
+            case MultiRowOp.Last:
                return col_vals.Last();
+            case MultiRowOp.StdDev:
+               return MathNet.Numerics.Statistics.Statistics.StandardDeviation(col_vals);
             default:
                // TODO
                Log.assert(false);

src/DataStreamSuite/DataStream/Primitives/GroupBy/GroupByStream.cs

@@ -5,7 +5,7 @@ using System.Collections.Generic;
 using System.Linq;
 
 namespace BCG {
-   public class GroupByStream : MethodDrivenStream {
+   internal class GroupByStream : MethodDrivenStream {
       private class MyFuncGroupByEvaluator : IGroupByEvaluator {
          private string[] _groupby_cols;
          private Func<DataPacket, IBlueRow> _groupby_func;

src/DataStreamSuite/DataStream/Primitives/GroupBy/NonIncremental/AggregationSteam.cs

@@ -0,0 +1,20 @@
+namespace bts.SilverPine {
+   using BCG;
+
+   using System;
+
+   [DataStreamUserRespresentation("AggregationStream")]
+   public class AgregationStream : ViewStream {
+
+      public AgregationStream(ViewStream parent_str, string operation_txt) {
+         ViewStream rs = new NonIncGroupByStream(parent_str, String.Empty, operation_txt);
+         mySetRealSource(rs);
+      }
+
+      public AgregationStream(ViewStream parent_str, string groupby_col, string operation_txt) {
+         ViewStream rs = new NonIncGroupByStream(parent_str, groupby_col, operation_txt);
+         mySetRealSource(rs);
+      }
+   }
+
+}

src/DataStreamSuite/DataStream/Primitives/GroupBy/NonIncremental/NonIncGroupByStream.cs

@@ -8,7 +8,7 @@ using bts;
 using bts.SilverPine;
 
 namespace BCG {
-   [DataStreamUserRespresentation("aggregate")]
+   [DataStreamUserRespresentation("AggregateStream")]
    public class NonIncGroupByStream : ViewStream {
       private class DelNonIncGroupByEvaluator : INonIncGroupByEvaluator {
          Func<string[]> _groupby_cols_func;
@@ -53,15 +53,18 @@ namespace BCG {
                                  Func<List<IBlueRow>, List<IBlueRow>> evaluate_func)
                                  : this(main_stream, new DelNonIncGroupByEvaluator(groupby_cols_func, evaluate_func)) { }
 
-      public NonIncGroupByStream(ViewStream parent_str, string groupby_col, string standard_group_by_text, string optional_sort_col = null) {
-         GroupByBundle std_group_by = GroupByBundle.parse(standard_group_by_text);
+      public NonIncGroupByStream(ViewStream parent_str, string groupby_col, string standard_group_by_text)
+      : this(parent_str, groupby_col, standard_group_by_text, String.Empty) { }
+
+      public NonIncGroupByStream(ViewStream parent_str, string groupby_col, string standard_group_by_text, string optional_sort_col) {
+         MultiRowOpHelper std_group_by = MultiRowOpHelper.parse(standard_group_by_text);
          Func<string[]> groupby_cols_func = () => {
             return String.IsNullOrEmpty(groupby_col) ? null : new string[] { groupby_col };
          };
          Func<List<IBlueRow>, List<IBlueRow>> evaluate_func = (group_rows) => {
             if (!String.IsNullOrEmpty(optional_sort_col))
                group_rows.Sort((r1, r2) => BlueDataUtility.compare_rows(r2, r1, optional_sort_col));
-            IBlueRow rv = GroupByBundle.group_by(group_rows, std_group_by);
+            IBlueRow rv = MultiRowOpHelper.group_by(group_rows, std_group_by);
             return new IBlueRow[] { rv }.ToList();
          };
          ViewStream rs = new NonIncGroupByStream(parent_str, groupby_cols_func, evaluate_func);

src/DataStreamSuite/DataStream/Primitives/Join/NonIncremental/NonIncJoinBookkeeper.cs

@@ -74,7 +74,7 @@ namespace BCG {
       }
 
       public void _on_stream_cb(object dp_obj) {
-         Log.Gen.info($"NonIncJoinBookkeeper._on_stream_cb dp_obj: {dp_obj}");
+         //Log.Gen.info($"NonIncJoinBookkeeper._on_stream_cb dp_obj: {dp_obj}");
          var timer = Stopwatch.StartNew();
          DataPacket dp = (DataPacket)dp_obj;
          _fill_missing_data(dp);

src/DataStreamSuite/DataStream/Primitives/Join/NonIncremental/NonIncJoinStream.cs

@@ -50,7 +50,7 @@ namespace BCG {
          int data_col_no = 1;
          for (int jp = 0; jp < _join_specs.Length; jp++) {
             NonIncJoinBookkeeper s = new NonIncJoinBookkeeper(_join_specs[jp], jp, data_col_no, _row_hasher);
-            Log.Gen.info($"NonIncJoinStream.myOnFirstSubscriber jp: {jp} spec: {_join_specs[jp]}");
+            //Log.Gen.info($"NonIncJoinStream.myOnFirstSubscriber jp: {jp} spec: {_join_specs[jp]}");
             data_col_no += _join_specs[jp].data_col_names.Length;
             s.on_new_data += _stream_callback;
             _book_keeping_data.Add(s);
@@ -71,7 +71,7 @@ namespace BCG {
       }
 
       private void _stream_callback() {
-         Log.Gen.info("NonIncJoinStream._stream_callback received callback");
+         //Log.Gen.info("NonIncJoinStream._stream_callback received callback");
          _is_dirty = true;
          doWork();
       }
@@ -99,7 +99,7 @@ namespace BCG {
 
       private int _n_total_fires = 0, _n_expensive_fires = 0;
       private void _form_and_fire() {
-         Log.Gen.info($"NonIncJoinStream._form_and_fire _is_ready(): {_is_ready()}");
+         //Log.Gen.info($"NonIncJoinStream._form_and_fire _is_ready(): {_is_ready()}");
          if (_is_ready()) {
             _construct_schema();
             var timer = Stopwatch.StartNew();
@@ -126,11 +126,11 @@ namespace BCG {
             _n_total_fires++;
             //if (timer.ElapsedMilliseconds >= 250) {
             _n_expensive_fires++;
-            string log_msg = $"Join and map time for {_book_keeping_data.Count} "
-               + $"tables {timer.Elapsed.TotalMilliseconds} msecs, ratio {_n_expensive_fires}/{_n_total_fires}";
-            Log.Gen.info(log_msg);
+            //string log_msg = $"Join and map time for {_book_keeping_data.Count} "
+            //   + $"tables {timer.Elapsed.TotalMilliseconds} msecs, ratio {_n_expensive_fires}/{_n_total_fires}";
+            //Log.Gen.info(log_msg);
             //}
-            var dp_out = BlueDataUtility.build_dp_inc(dp_output_rows, myLastValue as DataPacket);
+            var dp_out = BlueDataUtility.build_dp_inc(dp_output_rows, myLastValue as DataPacket, true);
             myFireEvent(dp_out);
             //myFireEvent(new DataPacket(dp_output_rows, dp_output_rows, myLastValue == null ? new BlueTable(_output_schema) : (myLastValue as DataPacket).newRows));
          }
@@ -152,7 +152,7 @@ namespace BCG {
       }
 
       public void doWork() {
-         Log.Gen.info($"NonIncJoinStream.doWork _last_fire_time: {_last_fire_time} _is_dirty: {_is_dirty}");
+         //Log.Gen.info($"NonIncJoinStream.doWork _last_fire_time: {_last_fire_time} _is_dirty: {_is_dirty}");
          if (DateTime.Now - _last_fire_time >= _throttle_period && _is_dirty) {
             _is_dirty = false;
             _last_fire_time = DateTime.Now;

src/DataStreamSuite/DataStream/Primitives/Map/GlobalMapStream.cs

@@ -69,4 +69,6 @@ namespace bts {
          myFireEvent(dp_out);
       }
    }
+
+
 }

src/DataStreamSuite/DataStream/Primitives/Map/MapStream.cs

@@ -13,9 +13,8 @@ namespace BCG {
    /// desired row ie newRow = f(row), where f is the mapping fucntion. Note that the id 
    /// is preserved across the rows, ie row["id"] = f(row)["id"]
    /// </summary>
-   [DataStreamUserRespresentation (false)]
-   public class MapStream : MethodDrivenStream
-   {
+   [DataStreamUserRespresentation(false)]
+   public class MapStream : MethodDrivenStream {
       protected IStreamMapper myMapper;
       protected IChangeableStreamMapper myChangeableMapper = null;
       private bool myUseNewMap = Configuration.get<DataStreamConfig>().UseNewMap;

src/DataStreamSuite/DataStream/Primitives/Map/RollingComputationStream

@@ -0,0 +1,14 @@
+namespace bts.SilverPine
+{
+   using BCG;
+
+   [DataStreamUserRespresentation("RollingComputationStream")]
+   public class RollingComputationStream : ViewStream
+   {
+      private ViewStream _parent_str;
+      public RollingComputationStream(ViewStream parent_str, string operation_txt, string roll_column, uint window_sz)
+      {
+
+      }
+   }
+}

src/DataStreamSuite/DataStream/Primitives/Map/RollingComputationStream.cs

@@ -0,0 +1,10 @@
+namespace bts.SilverPine {
+   using BCG;
+
+
+   [DataStreamUserRespresentation("RollingComputationStream")]
+   public class RollingComputationStream : ViewStream {
+      public RollingComputationStream(ViewStream parent_str, string operation_txt, string roll_column, uint window_sz) {
+      }
+   }
+}

src/DataStreamSuite/DataStream/Primitives/Project/New/NewProjectStream.cs

@@ -108,6 +108,7 @@ namespace bts {
 
    }
 
+   [DataStreamUserRespresentation("Project")]
    public class NewProjectStream : ViewStream, IStreamMapper {
 
       private ViewStream _stream;

src/DataStreamSuite/DataStream/Primitives/Time/DateTimeExpanstionStream.cs

@@ -4,7 +4,7 @@ namespace bts.SilverPine {
    using BCG;
 
    using System;
-
+   [DataStreamUserRespresentation("DateTimeExpanderStream")]
    public class DateTimeExpanstionStream : ViewStream {
       public DateTimeExpanstionStream(ViewStream parent_stream, string inp_col) {
          Func<IBlueTableSchema, IBlueTableSchema> get_schema = (s) => {
@@ -24,11 +24,11 @@ namespace bts.SilverPine {
             IBlueTableSchema s = get_schema(r.schema);
             MutableBlueRow rv = new MutableBlueRow(s, r);
             rv.setValue($"{inp_col}_date", dt.Date);
-            rv.setValue($"{inp_col}_day", dt.DayOfWeek.ToString());
+            rv.setValue($"{inp_col}_day_of_week", dt.DayOfWeek.ToString());
             rv.setValue($"{inp_col}_day_of_month", dt.Day);
             rv.setValue($"{inp_col}_month", dt.Date.Month);
             rv.setValue($"{inp_col}_hour", dt.Hour);
-            rv.setValue($"{inp_col}_min", dt.Minute);
+            rv.setValue($"{inp_col}_minute", dt.Minute);
             rv.setValue($"{inp_col}_seconds_of_day", (dt - dt.Date).TotalSeconds);
             return rv;
          });

src/DataStreamSuite/DataStream/Primitives/Time/SecondStream.cs

@@ -37,4 +37,42 @@ namespace BCG
       }
          
    }
+
+   [DataStreamUserRespresentation("throttle")]
+   public class ThrottleStream : ViewStream{
+      private readonly ViewStream _main_stream;
+      private readonly TimeSpan _ts;
+      private readonly ViewStreamDatabase _vsd;
+      private DateTime _last_fire_time = DateTime.MinValue;
+
+      public ThrottleStream (ViewStream main_stream, int throttle_seconds)
+      :
+      this(main_stream, TimeSpan.FromSeconds(throttle_seconds), ViewStreamDatabase.instance)
+      {
+      }
+
+      public ThrottleStream (ViewStream main_stream, TimeSpan ts, ViewStreamDatabase vsd){
+         _main_stream = main_stream;
+         _ts = ts;
+         _vsd = vsd;
+      }
+
+       protected override void myOnFirstSubscriber() {
+         _main_stream.onChange += _handle_base_stream_update;
+      }
+
+      protected override void myOnLastSubscriber() {
+         _main_stream.onChange -= _handle_base_stream_update;
+      }
+
+      private void _handle_base_stream_update(object dp_obj){
+         if (DateTime.Now - _last_fire_time >= _ts){
+            DataPacket dp_in = dp_obj as DataPacket;
+            _last_fire_time = DateTime.Now;
+            DataPacket dp = BlueDataUtility.build_dp_inc(dp_in.newRows, myLastValue as DataPacket);
+            myFireEvent(dp);
+         }
+      }
+ }
+
 }

src/DataStreamSuite/DataStream/Primitives/Time/ThrottleStream

@@ -0,0 +1,40 @@
+namespace bts.SilverPine{
+   using BGC;
+   using System;
+   [DataStreamUserRespresentation("throttle")]
+   public class ThrottleStream : ViewStream{
+      private readonly ViewStream _main_stream;
+      private readonly TimeSpan _ts;
+      private readonly ViewStreamDatabase _vsd;
+      private DateTime _last_fire_time = DateTime.MinValue;
+
+      public ThrottleStream (ViewStream main_stream, int throttle_seconds)
+      :
+      this(main_stream, TimeSpan.FromSeconds(throttle_seconds), ViewStreamDatabase.instance)
+      {
+      }
+
+      public ThrottleStream (ViewStream main_stream, TimeSpan ts, ViewStreamDatabase vsd){
+         _main_stream = main_stream;
+         _ts = ts;
+         _vsd;
+      }
+
+       protected override void myOnFirstSubscriber() {
+         _main_stream.onChange += _handle_base_stream_update;
+      }
+
+      protected override void myOnLastSubscriber() {
+         _main_stream.onChange -= _handle_base_stream_update;
+      }
+
+      private void _handle_base_stream_update(object dp_obj){
+         if (DateTime.Now - _last_fire_time >= _ts){
+            DataPacket dp_in = dp_obj;
+            _last_fire_time = DateTime.Now;
+            DataPacket dp = BlueDataUtility.build_dp_inc(dp_in.newRow, myLastValue as DataPacket);
+            myFireEvent(dp);
+         }
+      }
+ }
+}

src/DataStreamSuite/DataStream/Primitives/Top/TopStream.cs

@@ -40,4 +40,4 @@ namespace BCG {
             myFireSwitch = myLastFiredPacket.addedRows.Count + myLastFiredPacket.deletedRows.Count == 0 ? false : true;
         }
     }
-}
+}

src/DataStreamSuite/DataStream/Primitives/Union/MutableAppendStream.cs

@@ -1,18 +1,15 @@
 using System;
 using System.Collections.Generic;
+
 using mbts;
 
-namespace BCG
-{
-   internal class MutableAppendTable : IBlueRowCollection
-   {
-      private class MyRow : IBlueRow
-      {
+namespace BCG {
+   internal class MutableAppendTable : IBlueRowCollection {
+      private class MyRow : IBlueRow {
          private IBlueRow myRow;
          private int myID;
 
-         public MyRow(IBlueRow r, int id)
-         {
+         public MyRow(IBlueRow r, int id) {
             myRow = r;
             myID = id;
          }
@@ -21,10 +18,8 @@ namespace BCG
 
          public IBlueTableSchema schema { get { return myRow.schema; } }
 
-         public object this[int index]
-         {
-            get
-            {
+         public object this[int index] {
+            get {
                if (myRow.schema[index].colName == "id")
                   return myID;
                else
@@ -52,20 +47,17 @@ namespace BCG
 
          public System.Collections.ICollection Keys { get { return myRow.Keys; } }
 
-         public void Remove(object key) {throw new Exception("The method or operation is not implemented.");}
+         public void Remove(object key) { throw new Exception("The method or operation is not implemented."); }
 
-         public System.Collections.ICollection Values
-         {
-            get
-            {
+         public System.Collections.ICollection Values {
+            get {
                object[] vals = new object[myRow.Count];
                CopyTo(vals, 0);
                return vals;
             }
          }
 
-         public object this[object key]
-         {
+         public object this[object key] {
             get { return key is string ? this[key as string] : this[(int)key]; }
             set { throw new Exception("The method or operation is not implemented."); }
          }
@@ -74,8 +66,7 @@ namespace BCG
 
          #region ICollection Members
 
-         public void CopyTo(Array array, int index)
-         {
+         public void CopyTo(Array array, int index) {
             myRow.CopyTo(array, index);
             array.SetValue(myID, index + myRow.schema["id"].colNo);
          }
@@ -90,8 +81,7 @@ namespace BCG
 
          #region IEnumerable Members
 
-         System.Collections.IEnumerator System.Collections.IEnumerable.GetEnumerator()
-         { return new BlueRowEnumerator(this); }
+         System.Collections.IEnumerator System.Collections.IEnumerable.GetEnumerator() { return new BlueRowEnumerator(this); }
 
          #endregion
       }
@@ -106,8 +96,7 @@ namespace BCG
 
       public MutableAppendTable(IBlueTableSchema schema) { myRows = new BlueTable(schema); }
 
-      public IBlueRow addRow(IBlueRow r, int tableNo)
-      {
+      public IBlueRow addRow(IBlueRow r, int tableNo) {
          if (!myIDMap.ContainsKey(tableNo))
             myIDMap.Add(tableNo, new Dictionary<int, IBlueRow>());
          IBlueRow nr = new MyRow(r, myID++); // BlueDataUtility.getNewReplacedRow(r, myID++, "id");
@@ -116,20 +105,17 @@ namespace BCG
          return nr;
       }
 
-      public IBlueRow removeRow(IBlueRow r, int tableNo)
-      {
+      public IBlueRow removeRow(IBlueRow r, int tableNo) {
          IBlueRow dr = myIDMap[tableNo][(int)r["id"]];
          myRows.deleteRow(dr);
          myIDMap[tableNo].Remove((int)r["id"]);
          return dr;
       }
 
-      public IBlueRowCollection removeTable(int tableNo)
-      {
+      public IBlueRowCollection removeTable(int tableNo) {
          if (!myIDMap.ContainsKey(tableNo))
             return null;
-         else
-         {
+         else {
             BlueTable rv = new BlueTable(myRows.schema);
             foreach (IBlueRow r in myIDMap[tableNo].Values)
                rv.addRow(r);
@@ -139,8 +125,7 @@ namespace BCG
          }
       }
 
-      public void clear()
-      {
+      public void clear() {
          myRows.clear();
          myIDMap.Clear();
       }
@@ -151,7 +136,7 @@ namespace BCG
 
       public IBlueTableSchema schema { get { return myRows.schema; } }
 
-      public bool containsID(int id) {return myRows.containsID(id);}
+      public bool containsID(int id) { return myRows.containsID(id); }
 
       #endregion
 
@@ -169,7 +154,7 @@ namespace BCG
 
       #region IEnumerable Members
 
-      public System.Collections.IEnumerator GetEnumerator() {return myRows.GetEnumerator();}
+      public System.Collections.IEnumerator GetEnumerator() { return myRows.GetEnumerator(); }
 
       #endregion
    }
@@ -179,8 +164,8 @@ namespace BCG
    /// to append (or remove) more streams at
    /// runtime
    /// </summary>
-   public class MutableAppendStream : ViewStream
-   {
+   [DataStreamUserRespresentation("UnionStream")]
+   public class MutableAppendStream : ViewStream {
       private bool mySubscriptionActive = false;
       private MutableAppendTable myTable;
       private int myMovingIndex;
@@ -188,22 +173,17 @@ namespace BCG
       private bool _allign_col_orders = true;
 
       public MutableAppendStream(params ViewStream[] streams)
-         : this(new List<ViewStream>(streams))
-      { }
+         : this(new List<ViewStream>(streams)) { }
 
       public MutableAppendStream(bool allign_col_order, params ViewStream[] streams)
-         : this(allign_col_order, new List<ViewStream>(streams))
-      { }
+         : this(allign_col_order, new List<ViewStream>(streams)) { }
 
       public MutableAppendStream(ICollection<ViewStream> streams)
-         : this(true, streams)
-      { }
+         : this(true, streams) { }
 
-      public MutableAppendStream(bool allign_col_orders, ICollection<ViewStream> streams)
-      {
+      public MutableAppendStream(bool allign_col_orders, ICollection<ViewStream> streams) {
          _allign_col_orders = allign_col_orders;
-         foreach (ViewStream vs in streams)
-         {
+         foreach (ViewStream vs in streams) {
             myMovingIndex++;
             ViewStream pj = new ChangeTableNameStream(vs, myMovingIndex.ToString());
             if (_allign_col_orders)
@@ -212,8 +192,7 @@ namespace BCG
          }
       }
 
-      public void addStream(ViewStream vs)
-      {
+      public void addStream(ViewStream vs) {
          myMovingIndex++;
          ///the change name is done to identify the callbacks, and the project to
          ///make all the schemas col orders the same
@@ -225,19 +204,14 @@ namespace BCG
             pj.onChange += new ChangeDelegate(myHandleStreamChanges);
       }
 
-      public void removeStream(ViewStream vs)
-      {
+      public void removeStream(ViewStream vs) {
          int index = -1;
-         foreach (KeyValuePair<int, Pair<ViewStream, ViewStream>> p in myStreamIndex)
-         {
-            if (p.Value.first == vs)
-            {
+         foreach (KeyValuePair<int, Pair<ViewStream, ViewStream>> p in myStreamIndex) {
+            if (p.Value.first == vs) {
                index = p.Key;
-               if (mySubscriptionActive)
-               {
+               if (mySubscriptionActive) {
                   p.Value.second.onChange -= new ChangeDelegate(myHandleStreamChanges);
-                  if (myTable != null)
-                  {
+                  if (myTable != null) {
                      IBlueRowCollection delRows = myTable.removeTable(p.Key);
                      if (delRows != null)
                         myFireEvent(new DataPacket(myTable, new BlueTable(myTable.schema), delRows));
@@ -250,15 +224,13 @@ namespace BCG
          }
       }
 
-      protected override void myOnFirstSubscriber()
-      {
+      protected override void myOnFirstSubscriber() {
          mySubscriptionActive = true;
          foreach (KeyValuePair<int, Pair<ViewStream, ViewStream>> p in myStreamIndex)
             p.Value.second.onChange += new ChangeDelegate(myHandleStreamChanges);
       }
 
-      protected override void myOnLastSubscriber()
-      {
+      protected override void myOnLastSubscriber() {
          mySubscriptionActive = false;
          foreach (KeyValuePair<int, Pair<ViewStream, ViewStream>> p in myStreamIndex)
             p.Value.second.onChange -= new ChangeDelegate(myHandleStreamChanges);
@@ -266,20 +238,17 @@ namespace BCG
             myTable.clear();
       }
 
-      private void myHandleStreamChanges(object newValue)
-      {         
+      private void myHandleStreamChanges(object newValue) {
          DataPacket dp = newValue as DataPacket;
          myTable = myTable ?? new MutableAppendTable(new BlueTableSchema("MutableAppend", dp.schema.columnInfo));
          BlueTable adds = new BlueTable(myTable.schema),
                      dels = new BlueTable(myTable.schema);
          int index = Int32.Parse(dp.schema.tableName);
-         foreach (IBlueRow r in dp.deletedRows)
-         {
+         foreach (IBlueRow r in dp.deletedRows) {
             IBlueRow dr = myTable.removeRow(r, index);
             dels.addRow(dr);
          }
-         foreach (IBlueRow r in dp.addedRows)
-         {
+         foreach (IBlueRow r in dp.addedRows) {
             IBlueRow nr = myTable.addRow(r, index);
             adds.addRow(nr);
          }

src/DataStreamSuite/DataStreamCore/BlueData/BlueDataUtility.cs

@@ -280,6 +280,28 @@ namespace BCG {
 
       #region json
 
+      public static void write_to_json_file(this IBlueRowCollection table, string file_path) { 
+         write_dict_to_json_file(file_path, table.to_namevalue_pairs());
+      }
+
+      public static void write_dict_to_json_file(string file_path, List<Dictionary<string, string>> data) {
+         var options = new JsonSerializerOptions { WriteIndented = true };
+         string json = System.Text.Json.JsonSerializer.Serialize(data, options);
+         File.WriteAllText(file_path, json);
+      }
+
+      public static List<Dictionary<string, string>> ReadFromFile(string filePath) {
+         string json = File.ReadAllText(filePath);
+         return System.Text.Json.JsonSerializer.Deserialize<List<Dictionary<string, string>>>(json);
+      }
+
+      public static void write_table_to_json(IBlueRowCollection table) {
+         List<Dictionary<string, string>> rows = new List<Dictionary<string, string>>();
+         foreach (IBlueRow row in table) {
+            row.to_json();
+         }
+      }
+
       public static Dictionary<string, string> convert_json_to_dictionary(string json_string) {
          Dictionary<string, string> dictionary = JsonConvert.DeserializeObject<Dictionary<string, string>>(json_string);
          return dictionary;
@@ -348,6 +370,14 @@ namespace BCG {
          }
          return rv;
       }
+
+      public static List<Dictionary<string,string>> to_namevalue_pairs(this IBlueRowCollection table) {
+         List<Dictionary<string, string>> rv = new List<Dictionary<string, string>>();
+         foreach (IBlueRow r in table) {
+            rv.Add(r.to_dict());
+         }
+         return rv;
+      }
       #region Schema types
       // these are the only types allowed in schemas
       private static IReadOnlyDictionary<TypeCode, Type> _typecode_to_type = new Dictionary<TypeCode, Type>
@@ -1225,6 +1255,20 @@ namespace BCG {
          return rv;
       }
 
+      public static Dictionary<string, string> to_dict(this IBlueRow r) {
+         Dictionary<string, string> map = new Dictionary<string, string>();
+         foreach (ColumnInfo c in r.schema.columnInfo) {
+            map.Add(c.colName, r[c.colName].ToString());
+         }
+         return map;
+      }
+
+      public static string to_json(this IBlueRow r) {
+         Dictionary<string, string> map = r.to_dict();
+         string json = System.Text.Json.JsonSerializer.Serialize(map);
+         return json;
+      }
+
       public static void compute_differences_fast(IBlueRowCollection new_rows, IBlueRowCollection old_rows,
                                                   out IBlueRowCollection out_d_rows, out IBlueRowCollection out_a_rows) {
          if (old_rows == null || old_rows.Count == 0) {

src/DataStreamSuite/DataStreamCore/DataStreamCore.csproj

@@ -2,7 +2,7 @@
 
   <PropertyGroup>
     <AllowUnsafeBlocks>true</AllowUnsafeBlocks>
-    <TargetFramework>net7.0</TargetFramework>
+    <TargetFramework>net8.0</TargetFramework>
     <Nullable>disable</Nullable>
   </PropertyGroup>
 
@@ -19,8 +19,8 @@
   </ItemGroup>
 
   <ItemGroup>
-    <Proto Include="../../lib/config/protos/data_stream_config.proto" CSName="DataStreamConfig">
-      <OutDir>$(ProtoOutputPath)/lib/config/protos</OutDir>
+    <Proto Include="../../lib_spark/spark/config/protos/data_stream_config.proto" CSName="DataStreamConfig">
+      <OutDir>$(ProtoOutputPath)/lib_spark/spark/config/protos</OutDir>
     </Proto>
   </ItemGroup>
 

src/DataStreamSuite/DataStreamCore/DynamicLoading/PrimitiveManager.cs

@@ -47,10 +47,10 @@ namespace BCG {
       /// </summary>
       /// <param name="assemblyName"></param>
       public void loadAssembly(string assemblyName) {
-         Func<Type, bool> is_silverpine_only = (t) => {
-            if (!assemblyName.Contains("SharedStreams.dll"))
-               return true;
-            return t.Namespace == "bts.SilverPine";
+         Func<Type, bool> is_blue_orb = (t) => {
+            //if (!assemblyName.Contains("SharedStreams.dll"))
+            //   return true;
+            return t.Namespace == "bts.BlueOrb";// || t.Namespace == "BCG";
          };
          bts.Log.Gen.info($"PrimitiveManager.loadAssembly assemblyName: {assemblyName}");
          if (myAssemblyNames.Contains(assemblyName))
@@ -61,7 +61,7 @@ namespace BCG {
             Assembly.LoadFrom(assemblyName);
 
          foreach (Type type in loadedAssembly.GetTypes())
-            if (!type.IsInterface && is_silverpine_only(type)
+            if (!type.IsInterface && is_blue_orb(type)
                && !type.IsAbstract
                && type.IsSubclassOf(typeof(ViewStream))
                && type.IsPublic) {
@@ -77,7 +77,7 @@ namespace BCG {
                      break;
                   }
                if (isVisible) {
-                  Log.assert_d(myTypeLookupTable[userRep] == null, type.FullName);
+                 // Log.assert_d(myTypeLookupTable[userRep] == null, type.FullName);
                   myTypeLookupTable[userRep] = type;
                }
             }

src/DataStreamSuite/DataStreamExpressions/DataStreamExpressions.csproj

@@ -1,7 +1,7 @@
 <Project Sdk="Microsoft.NET.Sdk">
 
   <PropertyGroup>
-    <TargetFramework>net7.0</TargetFramework>
+    <TargetFramework>net8.0</TargetFramework>
     <ImplicitUsings>enable</ImplicitUsings>
     <Nullable>enable</Nullable>
   </PropertyGroup>

src/DataStreamSuite/DataStreamPublisher/DataStreamPublisher.csproj

@@ -2,7 +2,7 @@
 
   <PropertyGroup>
     <OutputType>Exe</OutputType>
-    <TargetFramework>net7.0</TargetFramework>
+    <TargetFramework>net8.0</TargetFramework>
     <Nullable>enable</Nullable>
     <AllowUnsafeBlocks>true</AllowUnsafeBlocks>
   </PropertyGroup>

src/DataStreamSuite/DataStreamQueryAnalyzer/DataStreamQueryAnalyzer.csproj

@@ -2,7 +2,7 @@
 
   <PropertyGroup>
     <OutputType>Exe</OutputType>
-    <TargetFramework>net7.0-windows</TargetFramework>
+    <TargetFramework>net8.0-windows</TargetFramework>
     <Nullable>enable</Nullable>
     <UseWindowsForms>true</UseWindowsForms>
   </PropertyGroup>

src/DataStreamSuite/DataStreamSuite.sln

@@ -36,6 +36,8 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Bts.ColumnExpressions", "Bt
 EndProject
 Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Bts.ColumnSortExpressions", "Bts.ColumnSortExpressions\Bts.ColumnSortExpressions.csproj", "{B038CB72-4E81-4AF1-976F-ED8655C2C3D6}"
 EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "DataStreamQueryAnalyzer", "DataStreamQueryAnalyzer\DataStreamQueryAnalyzer.csproj", "{BFA5BA5C-927A-41A9-A696-1E2F4B5FFD15}"
+EndProject
 Global
 	GlobalSection(SolutionConfigurationPlatforms) = preSolution
 		Debug|Any CPU = Debug|Any CPU
@@ -113,5 +115,9 @@ Global
 		{B038CB72-4E81-4AF1-976F-ED8655C2C3D6}.Debug|Any CPU.Build.0 = Debug|Any CPU
 		{B038CB72-4E81-4AF1-976F-ED8655C2C3D6}.Release|Any CPU.ActiveCfg = Release|Any CPU
 		{B038CB72-4E81-4AF1-976F-ED8655C2C3D6}.Release|Any CPU.Build.0 = Release|Any CPU
+		{BFA5BA5C-927A-41A9-A696-1E2F4B5FFD15}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{BFA5BA5C-927A-41A9-A696-1E2F4B5FFD15}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{BFA5BA5C-927A-41A9-A696-1E2F4B5FFD15}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{BFA5BA5C-927A-41A9-A696-1E2F4B5FFD15}.Release|Any CPU.Build.0 = Release|Any CPU
 	EndGlobalSection
 EndGlobal

src/DataStreamSuite/DataStreamTests/DataStreamTests.csproj

@@ -1,7 +1,7 @@
 <Project Sdk="Microsoft.NET.Sdk">
 
   <PropertyGroup>
-    <TargetFramework>net7.0</TargetFramework>
+    <TargetFramework>net8.0</TargetFramework>
     <ImplicitUsings>enable</ImplicitUsings>
     <Nullable>enable</Nullable>
 

src/DataStreamSuite/ExpressionCore/ExpressionCore.csproj

@@ -1,7 +1,7 @@
 <Project Sdk="Microsoft.NET.Sdk">
 
   <PropertyGroup>
-    <TargetFramework>net7.0</TargetFramework>
+    <TargetFramework>net8.0</TargetFramework>
     <ImplicitUsings>enable</ImplicitUsings>
     <Nullable>enable</Nullable>
   </PropertyGroup>

src/DataStreamSuite/GuiUtilities/GuiUtilities.csproj

@@ -1,7 +1,7 @@
 <Project Sdk="Microsoft.NET.Sdk">
 
   <PropertyGroup>
-    <TargetFramework>net7.0-windows</TargetFramework>
+    <TargetFramework>net8.0-windows</TargetFramework>
     <ImplicitUsings>enable</ImplicitUsings>
     <Nullable>enable</Nullable>
     <UseWindowsForms>true</UseWindowsForms>

src/DataStreamSuite/PubSub/Client/PublishingUtilities/IPublisher.cs

@@ -6,12 +6,18 @@ namespace BCG
 	public interface IPublisher
 	{
       void publish( IPubItem pi );
+#pragma warning disable SYSLIB0011 // Type or member is obsolete
       void publish( IPubItem pi, IFormatter f );
+#pragma warning restore SYSLIB0011 // Type or member is obsolete
       void publish( IPubItem[] pi );
+#pragma warning disable SYSLIB0011 // Type or member is obsolete
       void publish( IPubItem[] pi, IFormatter f );
+#pragma warning restore SYSLIB0011 // Type or member is obsolete
    }
 
+#pragma warning disable SYSLIB0011 // Type or member is obsolete
    public interface IPublisher<T> where T : IFormatter
+#pragma warning restore SYSLIB0011 // Type or member is obsolete
    {
       void publish (IPubItem pi);
       void publish (IPubItem[] pi);

src/DataStreamSuite/PubSub/Misc/GenericFormatter.cs

@@ -5,23 +5,21 @@ using System.Runtime.Serialization.Formatters.Binary;
 
 namespace BCG
 {
+#pragma warning disable SYSLIB0011 // Type or member is obsolete
    public class GenericFormatter<F> : IGenericFormatter where F : IFormatter, new ()
    {
       IFormatter myFormatter = new F ();
 
       public T Deserialize<T> (Stream serializationStream)
       {
-         #pragma warning disable 
          return (T) myFormatter.Deserialize (serializationStream);
-         #pragma warning restore SYSLIB0011
       }
       public void Serialize<T> (Stream serializationStream, T graph)
       {
-         #pragma warning disable 
          myFormatter.Serialize (serializationStream, graph);
-         #pragma warning restore SYSLIB0011
       }
    }
 
    public class GenericBinaryFormatter : GenericFormatter<BinaryFormatter> { }
+#pragma warning restore SYSLIB0011 // Type or member is obsolete
 }

src/DataStreamSuite/PubSub/PubSub.csproj

@@ -6,7 +6,7 @@
 
   <PropertyGroup>
       <AllowUnsafeBlocks>true</AllowUnsafeBlocks>
-    <TargetFramework>net7.0</TargetFramework>
+    <TargetFramework>net8.0</TargetFramework>
     <Nullable>enable</Nullable>
   </PropertyGroup>
 

src/DataStreamSuite/PubSubServer/PubSubServer.csproj

@@ -2,7 +2,7 @@
 
   <PropertyGroup>
     <OutputType>Exe</OutputType>
-    <TargetFramework>net7.0</TargetFramework>
+    <TargetFramework>net8.0</TargetFramework>
     <Nullable>enable</Nullable>
   </PropertyGroup>
 

src/DataStreamSuite/SharedStreams/BlueOrb/MarketData/Historical/HistoricalBarsStream.cs

@@ -1,4 +1,5 @@
-namespace bts.SilverPine {
+namespace bts.BlueOrb {
+   using bts.SilverPine;
    using BCG;
 
    using System.Runtime.CompilerServices;
@@ -45,7 +46,7 @@ namespace bts.SilverPine {
    }
 
    [DataStreamUserRespresentation("FuturesHistorical1DayBars")]
-   public class FuturesHistorical1DayBarsStream : ViewStream {
+   internal class FuturesHistorical1DayBarsStream : ViewStream {
 
       // this gets the raw bars for a symbol (perhaps make it into a raw bars stream?)
       public FuturesHistorical1DayBarsStream(string symbol) :
@@ -103,7 +104,7 @@ namespace bts.SilverPine {
    }
 
    [DataStreamUserRespresentation("FuturesHistorical1MinBars")]
-   public class FuturesHistorical1MinuteBarsStream : ViewStream {
+   internal class FuturesHistorical1MinuteBarsStream : ViewStream {
       // same as the above but for one min
       // this gets the raw bars for a symbol
       public FuturesHistorical1MinuteBarsStream(string symbol) :
@@ -156,6 +157,14 @@ namespace bts.SilverPine {
          ViewStream rs = new FutureHistoricalBarsStream(fut_contract, bar_size.ToString(), dt_range_beg, dt_range_end);
          mySetRealSource(rs);
       }
+      public FutureHistoricalBarsStream(string fut_contract, string bar_size)
+      : this(fut_contract, bar_size, DateTime.MinValue, DateTime.MaxValue) { }
+
+      public FutureHistoricalBarsStream(string fut_contract, string bar_size,
+                                        string dt_range_beg, string dt_range_end)
+                                        : this(fut_contract, bar_size, DateTime.Parse(dt_range_beg),
+                                        DateTime.Parse(dt_range_end)) { }
+
       public FutureHistoricalBarsStream(string fut_contract, string bar_size,
                                         DateTime dt_range_beg, DateTime dt_range_end)
                                         :
@@ -179,46 +188,6 @@ namespace bts.SilverPine {
       }
    }
 
-   [DataStreamUserRespresentation("FuturePriceDiff")]
-   public class FutureHistoricalBarsPriceDiffStream : ViewStream {
-
-      public FutureHistoricalBarsPriceDiffStream(string future_contract, string dt1, string dt2)
-      :
-      this(new ContractUtils.FutureContract(future_contract), DateTime.Parse(dt1), DateTime.Parse(dt2), "close", ViewStreamDatabase.instance) {
-      }
-
-
-      public FutureHistoricalBarsPriceDiffStream(string future_contract, DateTime dt1, DateTime dt2)
-      :
-      this(new ContractUtils.FutureContract(future_contract), dt1, dt2, "close", ViewStreamDatabase.instance) {
-      }
-
-      public FutureHistoricalBarsPriceDiffStream(string future_contract, DateTime dt1, DateTime dt2, string col)
-      :
-      this(new ContractUtils.FutureContract(future_contract), dt1, dt2, col, ViewStreamDatabase.instance) {
-      }
-
-
-      public FutureHistoricalBarsPriceDiffStream(ContractUtils.FutureContract fc, DateTime dt1, DateTime dt2, string col, ViewStreamDatabase vsd) {
-         BarSize bsize = dt2 - dt1 >= TimeSpan.FromDays(1.0) ? BarSize.ONE_DAY : BarSize.ONE_MIN;
-         ViewStream rs = new FutureHistoricalBarsStream(fc, BarSize.ONE_DAY, new DateTimeRange(dt1, dt2));
-
-         rs = new ColumnDifferenceStream(rs, r => {
-            DateTime r_dt = r.get_datetime("ts_event");
-            var rv = dt1.Equals(r_dt);
-            return rv;
-         },
-         //r => dt2.Equals(r.get_datetime("ts_event")), col, col);
-         r => {
-            DateTime r_dt = r.get_datetime("ts_event");
-            var rv = dt2.Equals(r_dt);
-            return rv;
-         }, col, col);
-         rs = new NewProjectStream(rs, "diff !");
-         mySetRealSource(rs);
-      }
-   }
-
    // takes the contract name and expands it into multuiple columns corresponding to the contracts.
    // the contract columns are a union, of all properties with some of them empty
    public class ContractExpanderStream : ViewStream {
@@ -239,3 +208,29 @@ namespace bts.SilverPine {
    }
 
 }
+
+namespace bts.BlueOrb {
+   using BCG;
+
+   using bts.api;
+
+   using Newtonsoft.Json.Serialization;
+
+   [DataStreamUserRespresentation("FuturesContinousContract")]
+   public class FuturesContinousContractStream : ViewStream {
+      // the argument is symbol, like "ES" etc and not the contract itself
+      public FuturesContinousContractStream(string symbol) : this(symbol, ViewStreamDatabase.instance) { }
+      public FuturesContinousContractStream(string symbol, ViewStreamDatabase db) {
+         symbol = symbol.EndsWith("*") ? symbol : symbol + "*";
+         ViewStream r = new FuturesHistorical1DayBarsStream(symbol);
+         r = new ContractExpanderStream(r, "symbol");
+         r = new FilterStream(r, "contract_type='Future'");
+         // finding the max volume across symbols, if we need to do this for open_interest, then join here 
+         // with the open interest stream on symbol/contract
+         r = new NonIncGroupByStream(r, "ts_event", "volume:Max");
+         r = new NewProjectStream(r, "ts_event symbol !");
+         r = new SortStream(r, "-ts_event");
+         mySetRealSource(r);
+      }
+   }
+}